├── .npmignore ├── tests.js ├── assets ├── QueryFlow-icon.png ├── QueryFlowTagLogo.png ├── QueryFlow-logo-white.png └── exampleController.exampleFunction.png ├── .gitignore ├── package.json ├── LICENSE ├── exampleController.mjs ├── QueryFlow.js └── README.md /.npmignore: -------------------------------------------------------------------------------- 1 | # .npmignore 2 | 3 | tests.js 4 | -------------------------------------------------------------------------------- /tests.js: -------------------------------------------------------------------------------- 1 | //Tests to verify query-flow-npm 2 | 3 | -------------------------------------------------------------------------------- /assets/QueryFlow-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/query-flow-npm/HEAD/assets/QueryFlow-icon.png -------------------------------------------------------------------------------- /assets/QueryFlowTagLogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/query-flow-npm/HEAD/assets/QueryFlowTagLogo.png -------------------------------------------------------------------------------- /assets/QueryFlow-logo-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/query-flow-npm/HEAD/assets/QueryFlow-logo-white.png -------------------------------------------------------------------------------- /assets/exampleController.exampleFunction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/query-flow-npm/HEAD/assets/exampleController.exampleFunction.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | *.lcov 25 | 26 | # nyc test coverage 27 | .nyc_output 28 | 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 30 | .grunt 31 | 32 | # Bower dependency directory (https://bower.io/) 33 | bower_components 34 | 35 | # node-waf configuration 36 | .lock-wscript 37 | 38 | # Dependency directories 39 | node_modules/ -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "queryflow.js", 3 | "author": "Ryan Campbell, Philip Brown, Vivek Patel, George Greer, Niko Amescua", 4 | "version": "1.0.4", 5 | "description": "Lightweight library for automatically caching slow SQL queries within the backend middleware.", 6 | "main": "QueryFlow.js", 7 | "scoped": false, 8 | "publishConfig": { 9 | "organization": "@query-flow", 10 | "access": "public" 11 | }, 12 | "dependencies": { 13 | "crypto-js": "^4.1.1" 14 | }, 15 | "type": "module", 16 | "scripts": { 17 | "test": "echo \"Error: no test specified\" && exit 1" 18 | }, 19 | "repository": { 20 | "type": "git", 21 | "url": "git+https://github.com/oslabs-beta/query-flow-npm.git" 22 | }, 23 | "keywords": [ 24 | "QueryFlow", 25 | "query-flow-npm", 26 | "query-flow", 27 | "queryflow", 28 | "SQL", 29 | "Redis", 30 | "cache", 31 | "memory" 32 | ], 33 | "license": "ISC", 34 | "homepage": "https://www.query-flow.com" 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 OSLabs Beta 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /exampleController.mjs: -------------------------------------------------------------------------------- 1 | import db from "../models/ourDBModel.mjs"; 2 | import redisModel from "../models/redisModel.mjs"; 3 | import QueryFlow from "queryflow.js"; 4 | 5 | const exampleController = {}; 6 | 7 | exampleController.example = async (req, res, next) => { 8 | const queryString = "SELECT * FROM users WHERE firstname = $1 AND lastname = $2"; 9 | const { firstName, lastName } = req.body; 10 | const values = [firstName, lastName]; 11 | const threshold = 3000; //Milliseconds 12 | const TTL = 30; //Seconds 13 | try { 14 | const result = await QueryFlow.autoCache({ 15 | redisModel, 16 | db, 17 | queryString, 18 | values, 19 | threshold, //OPTIONAL: Default value 3 seconds. 20 | TTL, //OPTIONAL: Default value 30 minutes. 21 | log:true, //OPTIONAL: Default value false. Turn console logs on and off. 22 | instanceLatency:true //OPTIONAL: Default value false. Switches measurement of time from measuring total latency to measuring total time within the primary database itself. 23 | }); 24 | res.locals.data = result.rows; 25 | return next(); 26 | } catch (error) { 27 | return next({ 28 | log: "Error handler caught error in middleware", 29 | status: 500, 30 | message: "Error handler caught error in middleware", 31 | }); 32 | } 33 | }; 34 | 35 | export default exampleController; -------------------------------------------------------------------------------- /QueryFlow.js: -------------------------------------------------------------------------------- 1 | import CryptoJS from "crypto-js"; 2 | 3 | const QueryFlow = {}; 4 | 5 | QueryFlow.autoCache = async ({redisModel, db, queryString, values, threshold = 3000, TTL = 1800, log = false, instanceLatency = false}) => { 6 | 7 | //Check querystring is 'Read' Type. If not, throw new error. 8 | const stringCheck = async () => { 9 | if(!queryString.toLowerCase().startsWith('select')){ 10 | console.error('Query string must be of read type only. I.E. SELECT'); 11 | const string = {text: queryString, values: values}; 12 | const result = await db(string); 13 | return result; 14 | } 15 | }; 16 | stringCheck(queryString); 17 | 18 | const string = {text: queryString, values: values}; 19 | 20 | //Create a unique key for redis storage 21 | let keyConcat = ''; 22 | for (let i = 0; i < values.length; i++){ 23 | keyConcat += values[i].toString() 24 | } 25 | keyConcat = keyConcat + queryString 26 | 27 | //Create the hash key for the data to be stored. 28 | const hash = CryptoJS.MD5(keyConcat).toString(); 29 | 30 | //Attempt to retrieve data from Redis instance. 31 | const getResultRedis = await redisModel.json.get(hash, { 32 | path: '.', 33 | }); 34 | 35 | //CACHE HIT 36 | if (getResultRedis){ 37 | if (log) console.log(`Returned data associated with key ${hash} from Redis`) 38 | return getResultRedis; 39 | 40 | //CACHE MISS 41 | } else if (!instanceLatency) { 42 | const startTime = process.hrtime(); 43 | const resultSQL = await db(string); 44 | const endTime = process.hrtime(startTime); 45 | const totalTimeHrTime = (endTime[0] * 1000 + endTime[1] / 1000000).toFixed(2); 46 | if (totalTimeHrTime > threshold) { 47 | const addToCache = async () => { 48 | try { 49 | if (log)console.log(`Set Redis with key ${hash} and data`); 50 | await redisModel.json.set(hash, '.', resultSQL); 51 | await redisModel.expire(hash, TTL); 52 | } catch (err) { 53 | console.error('Error setting JSON data in Redis Database:', err); 54 | } 55 | } 56 | addToCache(); 57 | } 58 | return resultSQL; 59 | } else { 60 | const appendedString = 'EXPLAIN (ANALYZE true, COSTS true, SETTINGS true, BUFFERS true, WAL true, SUMMARY true, FORMAT JSON)' + `${queryString}`; 61 | const query = {text: appendedString, values: values}; 62 | const data = await db(query) 63 | const totalTimeInstance = (Number(((data.rows[0]['QUERY PLAN'][0]['Planning Time']) + (data.rows[0]['QUERY PLAN'][0]['Execution Time'])))); 64 | const resultSQL = await db(string); 65 | if (totalTimeInstance > threshold) { 66 | const addToCache = async () => { 67 | try { 68 | if (log) console.log(`Set Redis with key ${hash} and data`); 69 | await redisModel.json.set(hash, '.', resultSQL); 70 | await redisModel.expire(hash, TTL); 71 | } catch (err) { 72 | console.error('Error setting JSON data in Redis Database:', err); 73 | } 74 | } 75 | addToCache(); 76 | } 77 | return resultSQL; 78 | } 79 | }; 80 | 81 | export default QueryFlow; -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # queryflow.js 2 | 3 | 4 |
5 | 6 |
7 | 8 | Logo 9 | 10 | 11 |

queryflow.js

12 | 13 |

14 | Lightweight middleware function that automatically caches SQL query results for increased performance. 15 |
16 |
17 | 18 | Analyze & Visualize » 19 |
20 |
21 | Report Bug 22 | · 23 | Request Feature 24 |

25 |
26 | 27 |
28 | 29 | 30 |
31 | Table of Contents 32 |
    33 |
  1. 34 | About The Project 35 |
    2. 36 |
  2. 37 | Getting Started 38 | 42 |
    3. 43 |
  3. Usage
    4. 44 |
  4. Contributing
    5. 45 |
  5. Contact
    6. 46 |
  6. Acknowledgments
    7. 47 |
  7. License
    8. 48 |
49 |
50 | 51 | 52 | 53 | ## About The Project 54 | 55 |
56 | 57 | Logo 58 | 59 |
60 | 61 |
62 | 63 | Before the development of this NPM package to automatically cache query results from a relational database, a [web application](https://www.query-flow.com) was made for developers to analyze and visualize the performance of SQL queries. With these insights into the performance of an application's backend queries, developers can implement data-backed thresholds with the queryflow.js NPM package, such that queries slower than the set threshold will be stored in a cache database. This **increases performance** of applications by **reducing each query's time** by up to _**94%**_. 64 | 65 | [NPM Package](https://www.npmjs.com/package/queryflow.js) 66 | 67 | 68 | 69 | ## Getting Started 70 | 71 | ### Prerequisites 72 | 73 | This package assumes that a Redis instance has been set up alongside a primary relational database. 74 | 75 | For more information about setting up a Redis database, visit [their docs](https://redis.io/docs/getting-started/). 76 | 77 | ### Installation 78 | 79 | To get started: 80 | 81 | ```sh 82 | npm install queryflow.js 83 | ``` 84 | 85 | 86 | 87 | ## Usage 88 | 89 | ### Example Implementation in Backend 90 | 91 | In the code below, the primary relational database model, Redis model and npm package are imported into a backend controller. Here, the developer defines the SQL query string, an array of values which will be bound to the query string, the threshold time in seconds for caching the results from the primary database, and the TTL of the data stored in the cache database. The autoCache method is invoked asynchronously within the controller function. The results can be assigned, and pass on through the response cycle. 92 | 93 | ```javascript 94 | import db from "../models/ourDBModel.mjs"; 95 | import redisModel from "../models/redisModel.mjs"; 96 | import QueryFlow from "queryflow.js"; 97 | 98 | const exampleController = {}; 99 | 100 | exampleController.example = async (req, res, next) => { 101 | const queryString = 102 | "SELECT * FROM users WHERE firstname = $1 AND lastname = $2"; 103 | const { firstName, lastName } = req.body; 104 | const values = [firstName, lastName]; 105 | const threshold = 3000; //Milliseconds 106 | const TTL = 30; //Seconds 107 | try { 108 | const result = await QueryFlow.autoCache({ 109 | redisModel, 110 | db, 111 | queryString, 112 | values, 113 | threshold, //OPTIONAL: Default value 3 seconds. 114 | TTL, //OPTIONAL: Default value 30 minutes. 115 | log: true, //OPTIONAL: Default value false. Turn console logs on and off. 116 | instanceLatency: true, //OPTIONAL: Default value false. Switches measurement of time from measuring total latency to measuring total time within the primary database itself. 117 | }); 118 | res.locals.data = result.rows; 119 | return next(); 120 | } catch (error) { 121 | return next({ 122 | log: "Error handler caught error in middleware", 123 | status: 500, 124 | message: "Error handler caught error in middleware", 125 | }); 126 | } 127 | }; 128 | 129 | export default exampleController; 130 | ``` 131 | 132 | ## Parameters 133 | 134 | - []() **redisModel:** Redis client. 135 | - []() **db:** Primary relational database. 136 | - []() **queryString:** SQL query string. 137 | - []() **values:** Array of values to be bound to SQL query string 138 | - []() **threshold:** OPTIONAL. Threshold in milliseconds (ms). Default value is 3 seconds. 139 | - []() **TTL:** OPTIONAL. Time to Live (TTL) in seconds (s). Default value is 30 mins. 140 | - []() **log:** OPTIONAL. Turns console.logs on and off. Default value is false. 141 | - []() **instanceLatency:** OPTIONAL. Switches from measuring total latency to measuring latency within the instance itself. Default value is false. 142 | 143 | We recommend using the _volatile-ttl_ eviction policy with this package to take advantage of the data's TTL, in the even that maximum memory is reached. 144 | 145 | ### Example Use Cases 146 | 147 | **Session Storage** 148 | 149 | With an automatic cache backend architecture, a session store could be used to cache user session data, thereby improving the speed and efficiency of accessing session data since data is kept in a fast-access cache instead of slower primary storage, such as a relational database. The cache can store frequently accessed session data, which reduces the load on the primary database and improves response times for the user. 150 | 151 | **Machine Learning** 152 | 153 | Machine learning models can often be quite large and take time to load from a primary database. If an application needs to make frequent predictions in real time, it may be too slow to load the model from disk each time a prediction is needed. This NPM package can be used to cache critical components of the model in memory for fast access, significantly reducing the prediction latency and increasing the overall speed of the application. 154 | 155 | **Frequently Fetched Data or Costly Queries** 156 | 157 | Storing fetch requests' result sets in memory decreases the processing time for web applications that need to return data from complex queries. Furthermore, data that is frequently used need not be retrieved from a slow primary relational database, it can be kept in memory for quick retrieval as it is often requested by the client. 158 | 159 | See the [open issues](https://github.com/oslabs-beta/query-flow-npm/issues) for a full list of proposed features (and known issues). 160 | 161 | 162 | 163 | ## Contributing 164 | 165 | Contributions are what make the open source community such an amazing place to learn, inspire, and create. Any contributions you make are **greatly appreciated**. 166 | 167 | If you have a suggestion that would make this better, please fork the repo and create a pull request. You can also simply open an issue with the tag "enhancement". 168 | Don't forget to give the project a star! Thanks again! 169 | 170 | 1. Fork the Project 171 | 2. Create your Feature Branch (`git checkout -b feature/AmazingFeature`) 172 | 3. Commit your Changes (`git commit -m 'Add some AmazingFeature'`) 173 | 4. Push to the Branch (`git push origin feature/AmazingFeature`) 174 | 5. Open a Pull Request 175 | 176 | 177 | 178 | ## Contact 179 | 180 | Email - queryflow58@gmail.com 181 | 182 | Twitter - [@Query_Flow](https://twitter.com/Query_Flow) 183 | 184 | LinkedIn - [Team Page](https://www.linkedin.com/company/query-flow/about/) 185 | 186 | QueryFlow Performance Visualizer and Analyzer: [https://www.query-flow.com/](https://www.query-flow.com/) 187 | 188 | ## Team 189 | 190 | 191 | 192 | - []() **Vivek Patel** - [GitHub](https://github.com/vkpatel007) - [LinkedIn](https://www.linkedin.com/in/vivekpatel607/) 193 | - []() **Niko Amescua** - [GitHub](https://github.com/NikoAmescua) - [LinkedIn](https://www.linkedin.com/in/nikoamescua/) 194 | - []() **Ryan Campbell** - [GitHub](https://github.com/cronullarc) - [LinkedIn](https://www.linkedin.com/in/ryancampbelladr/) 195 | - []() **Philip Brown** - [GitHub](https://github.com/starfishpanda) - [LinkedIn](https://www.linkedin.com/in/philiplbrown/) 196 | - []() **George Greer** - [GitHub](https://github.com/ggreer91) - [LinkedIn](https://www.linkedin.com/in/george-greer/) 197 | 198 | ## Acknowledgements 199 | 200 | The Team wholeheartedly thanks Chris Suzukida for his mentorship and support throughout the development of this project. 201 | 202 | 203 | 204 | ## License 205 | 206 | Distributed under the MIT License. See `LICENSE.txt` for more information. 207 | 208 |

(Back to Top)

209 | --------------------------------------------------------------------------------