├── .gitignore ├── .github └── FUNDING.yml ├── package.json ├── example2.js ├── LICENSE ├── example1.js ├── test.js ├── README.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | archive/ 3 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | tidelift: "npm/reactive-postgres" 2 | github: mitar 3 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "reactive-postgres", 3 | "version": "0.2.0", 4 | "description": "Reactive queries for PostgreSQL", 5 | "keywords": [ 6 | "reactivity", 7 | "postgresql", 8 | "postgres", 9 | "sql", 10 | "live", 11 | "query", 12 | "meteor", 13 | "observer", 14 | "select" 15 | ], 16 | "main": "index.js", 17 | "dependencies": { 18 | "await-lock": "~2.1.0", 19 | "pg": "~8.7.1", 20 | "@tozd/random-id": "~1.0.0" 21 | }, 22 | "devDependencies": { 23 | "through2": "~4.0.2" 24 | }, 25 | "engines": { 26 | "node": ">=0.10.0" 27 | }, 28 | "license": "BSD-3-Clause", 29 | "repository": { 30 | "type": "git", 31 | "url": "https://github.com/tozd/node-reactive-postgres.git" 32 | }, 33 | "funding": [ 34 | { 35 | "type": "github", 36 | "url": "https://github.com/sponsors/mitar" 37 | }, 38 | { 39 | "type": "tidelift", 40 | "url": "https://tidelift.com/subscription/pkg/npm-reactive-postgres" 41 | } 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /example2.js: -------------------------------------------------------------------------------- 1 | const {Manager} = require('reactive-postgres'); 2 | const through2 = require('through2'); 3 | 4 | const manager = new Manager({ 5 | connectionConfig: { 6 | user: 'dbuser', 7 | host: 'database.server.com', 8 | database: 'mydb', 9 | password: 'secretpassword', 10 | port: 3211, 11 | }, 12 | }); 13 | 14 | const jsonStream = through2.obj(function (chunk, encoding, callback) { 15 | this.push(JSON.stringify(chunk, null, 2) + '\n'); 16 | callback(); 17 | }); 18 | 19 | (async () => { 20 | await manager.start(); 21 | 22 | const handle = await manager.query(`SELECT * FROM posts`, { 23 | uniqueColumn: '_id', 24 | mode: 'changed', 25 | }); 26 | 27 | handle.on('error', (error) => { 28 | console.error("stream error", error); 29 | }); 30 | 31 | handle.on('close', () => { 32 | console.log("stream has closed"); 33 | }); 34 | 35 | handle.pipe(jsonStream).pipe(process.stdout); 36 | })().catch((error) => { 37 | console.error("async error", error); 38 | }); 39 | 40 | process.on('SIGINT', () => { 41 | manager.stop().catch((error) => { 42 | console.error("async error", error); 43 | }).finally(() => { 44 | process.exit(); 45 | }); 46 | }); 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019, TOZD 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of the PeerLibrary Project nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /example1.js: -------------------------------------------------------------------------------- 1 | const {Manager} = require('reactive-postgres'); 2 | 3 | const manager = new Manager({ 4 | connectionConfig: { 5 | user: 'dbuser', 6 | host: 'database.server.com', 7 | database: 'mydb', 8 | password: 'secretpassword', 9 | port: 3211, 10 | }, 11 | }); 12 | 13 | (async () => { 14 | await manager.start(); 15 | 16 | const handle = await manager.query(`SELECT * FROM posts`, { 17 | uniqueColumn: 'id', 18 | mode: 'changed', 19 | }); 20 | 21 | handle.on('start', () => { 22 | console.log("query has started"); 23 | }); 24 | 25 | handle.on('ready', () => { 26 | console.log("initial data have been provided"); 27 | }); 28 | 29 | handle.on('refresh', () => { 30 | console.log("all query changes have been provided"); 31 | }); 32 | 33 | handle.on('insert', (row) => { 34 | console.log("row inserted", row); 35 | }); 36 | 37 | handle.on('update', (row, columns) => { 38 | console.log("row updated", row, columns); 39 | }); 40 | 41 | handle.on('delete', (row) => { 42 | console.log("row deleted", row); 43 | }); 44 | 45 | handle.on('error', (error) => { 46 | console.error("query error", error); 47 | }); 48 | 49 | handle.on('stop', (error) => { 50 | console.log("query has stopped", error); 51 | }); 52 | 53 | handle.start(); 54 | })().catch((error) => { 55 | console.error("async error", error); 56 | }); 57 | 58 | process.on('SIGINT', () => { 59 | manager.stop().catch((error) => { 60 | console.error("async error", error); 61 | }).finally(() => { 62 | process.exit(); 63 | }); 64 | }); 65 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | // docker run -d --name postgres -e POSTGRES_PASSWORD=pass -p 5432:5432 postgres:11.1 2 | 3 | const {Pool} = require('pg'); 4 | const through2 = require('through2'); 5 | const {UNMISTAKABLE_CHARS} = require('@tozd/random-id'); 6 | 7 | const {Manager} = require('./index'); 8 | 9 | const CONNECTION_CONFIG = { 10 | user: 'postgres', 11 | database: 'postgres', 12 | password: 'pass', 13 | }; 14 | 15 | const jsonStream = through2.obj(function (chunk, encoding, callback) { 16 | this.push(JSON.stringify(chunk, null, 2) + '\n'); 17 | callback(); 18 | }); 19 | 20 | const pool = new Pool(CONNECTION_CONFIG); 21 | 22 | const manager = new Manager({ 23 | connectionConfig: CONNECTION_CONFIG, 24 | }); 25 | 26 | manager.on('start', () => { 27 | console.log(new Date(), 'manager start'); 28 | }); 29 | 30 | manager.on('error', (error, client) => { 31 | console.log(new Date(), 'manager error', error); 32 | }); 33 | 34 | manager.on('stop', (error) => { 35 | console.log(new Date(), 'manager stop', error); 36 | }); 37 | 38 | manager.on('connect', (client) => { 39 | client.on('notice', (notice) => { 40 | console.warn(new Date(), notice.message, Object.assign({}, notice)); 41 | }); 42 | }); 43 | 44 | async function sleep(ms) { 45 | return new Promise(resolve => setTimeout(resolve, ms)); 46 | } 47 | 48 | (async () => { 49 | await manager.start(); 50 | 51 | await pool.query(` 52 | CREATE OR REPLACE FUNCTION random_id() RETURNS TEXT LANGUAGE SQL AS $$ 53 | SELECT array_to_string( 54 | array( 55 | SELECT SUBSTRING('${UNMISTAKABLE_CHARS}' FROM floor(random()*55)::int+1 FOR 1) FROM generate_series(1, 17) 56 | ), 57 | '' 58 | ); 59 | $$; 60 | DROP TABLE IF EXISTS comments CASCADE; 61 | DROP TABLE IF EXISTS posts CASCADE; 62 | CREATE TABLE posts ( 63 | "_id" CHAR(17) PRIMARY KEY DEFAULT random_id(), 64 | "body" JSONB NOT NULL DEFAULT '{}'::JSONB 65 | ); 66 | CREATE TABLE comments ( 67 | "_id" CHAR(17) PRIMARY KEY DEFAULT random_id(), 68 | "postId" CHAR(17) NOT NULL REFERENCES posts("_id"), 69 | "body" JSONB NOT NULL DEFAULT '{}'::JSONB, 70 | "user" JSON NOT NULL DEFAULT '{}'::JSON 71 | ); 72 | `); 73 | 74 | console.log("Inserting initial posts with comments...") 75 | 76 | let result; 77 | for (let i = 0; i < 5; i++) { 78 | result = await pool.query(` 79 | INSERT INTO posts ("body") VALUES($1) RETURNING _id; 80 | `, [{'title': `Post title ${i}`}]); 81 | 82 | const postId = result.rows[0]._id; 83 | 84 | for (let j = 0; j < 10; j++) { 85 | await pool.query(` 86 | INSERT INTO comments ("postId", "body", "user") VALUES($1, $2, $3); 87 | `, [postId, {'title': `Comment title ${j}`}, {'name': 'Foobar'}]); 88 | } 89 | } 90 | 91 | const queries = [ 92 | // All comments with embedded post. 93 | `SELECT "_id", "body", "user", (SELECT to_jsonb(posts) FROM posts WHERE posts."_id"=comments."postId") AS "post" FROM comments`, 94 | // All posts with embedded comments. 95 | `SELECT "_id", "body", (SELECT to_jsonb(COALESCE(array_agg(comments), '{}')) FROM comments WHERE comments."postId"=posts."_id") AS "comments" FROM posts`, 96 | ]; 97 | 98 | let handle1 = await manager.query(queries[0], {uniqueColumn: '_id', mode: 'changed'}); 99 | 100 | handle1.on('start', () => { 101 | console.log(new Date(), 'query start', handle1.queryId); 102 | }); 103 | 104 | handle1.on('ready', () => { 105 | console.log(new Date(), 'query ready', handle1.queryId); 106 | }); 107 | 108 | handle1.on('refresh', () => { 109 | console.log(new Date(), 'query refresh', handle1.queryId); 110 | }); 111 | 112 | handle1.on('insert', (row) => { 113 | console.log(new Date(), 'insert', handle1.queryId, row); 114 | }); 115 | 116 | handle1.on('update', (row, columns) => { 117 | console.log(new Date(), 'update', handle1.queryId, row, columns); 118 | }); 119 | 120 | handle1.on('delete', (row) => { 121 | console.log(new Date(), 'delete', handle1.queryId, row); 122 | }); 123 | 124 | handle1.on('error', (error) => { 125 | console.log(new Date(), 'query error', handle1.queryId, error); 126 | }); 127 | 128 | handle1.on('stop', (error) => { 129 | console.log(new Date(), 'query stop', handle1.queryId, error); 130 | }); 131 | 132 | console.log("Starting queries..."); 133 | 134 | await handle1.start(); 135 | 136 | const handle2 = await manager.query(queries[1], {uniqueColumn: '_id', mode: 'changed'}); 137 | 138 | handle2.on('close', () => { 139 | console.log(new Date(), 'query pipe close', handle2.queryId); 140 | }); 141 | 142 | handle2.on('error', (error) => { 143 | console.log(new Date(), 'query pipe error', handle2.queryId, error); 144 | }); 145 | 146 | handle2.pipe(jsonStream).pipe(process.stdout); 147 | 148 | await sleep(1000); 149 | 150 | console.log("Inserting additional posts with comments..."); 151 | 152 | let commentIds = []; 153 | for (let i = 5; i < 7; i++) { 154 | result = await pool.query(` 155 | INSERT INTO posts ("body") VALUES($1) RETURNING _id; 156 | `, [{'title': `Post title ${i}`}]); 157 | 158 | const postId = result.rows[0]._id; 159 | 160 | for (let j = 0; j < 10; j++) { 161 | result = await pool.query(` 162 | INSERT INTO comments ("postId", "body", "user") VALUES($1, $2, $3) RETURNING _id; 163 | `, [postId, {'title': `Comment title ${j}`}, {'name': 'Foobar'}]); 164 | 165 | commentIds.push(result.rows[0]._id); 166 | } 167 | } 168 | 169 | await sleep(1000); 170 | 171 | console.log("Updating additional comments' body..."); 172 | 173 | for (let i = 0; i < commentIds.length; i++) { 174 | await pool.query(` 175 | UPDATE comments SET "body"=$1 WHERE "_id"=$2; 176 | `, [{'title': `Comment new title ${i}`}, commentIds[i]]); 177 | } 178 | 179 | await sleep(1000); 180 | 181 | console.log("Updating additional comments' user..."); 182 | 183 | for (let i = 0; i < commentIds.length; i++) { 184 | await pool.query(` 185 | UPDATE comments SET "user"=$1 WHERE "_id"=$2; 186 | `, [{'name': 'Foobar2'}, commentIds[i]]); 187 | } 188 | 189 | await sleep(1000); 190 | 191 | console.log("Deleting additional comments..."); 192 | 193 | await pool.query(` 194 | DELETE FROM comments WHERE "_id"=ANY($1); 195 | `, [commentIds]); 196 | 197 | await sleep(1000); 198 | 199 | await pool.end(); 200 | await manager.stop(); 201 | })(); 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # reactive-postgres 2 | 3 | This node.js package brings reactive (or live) queries to PostgreSQL. You can take an arbitrary `SELECT` query, 4 | using multiple joins, data transformations, and even custom functions, and besides the initial set of 5 | results also get real-time updates about any changes to those results. This can enable you to keep UI in sync 6 | with the database in a reactive manner. 7 | 8 | ## Installation 9 | 10 | This is a node.js package. You can install it using NPM: 11 | 12 | ```bash 13 | $ npm install reactive-postgres 14 | ``` 15 | 16 | This package uses PostgreSQL. See [documentation](https://www.postgresql.org/docs/devel/tutorial-start.html) 17 | for more information how to install and use it. 18 | 19 | Requires PostgreSQL 11 or newer. 20 | 21 | ## reactive-postgres for enterprise 22 | 23 | Available as part of the Tidelift Subscription. 24 | 25 | The maintainers of reactive-postgres and thousands of other packages are working with Tidelift to deliver commercial support and maintenance for the open source dependencies you use to build your applications. Save time, reduce risk, and improve code health, while paying the maintainers of the exact dependencies you use. [Learn more.](https://tidelift.com/subscription/pkg/npm-reactive-postgres?utm_source=npm-reactive-postgres&utm_medium=referral&utm_campaign=enterprise&utm_term=repo) 26 | 27 | ## Usage 28 | 29 | As an event emitter: 30 | 31 | ```js 32 | const {Manager} = require('reactive-postgres'); 33 | 34 | const manager = new Manager({ 35 | connectionConfig: { 36 | user: 'dbuser', 37 | host: 'database.server.com', 38 | database: 'mydb', 39 | password: 'secretpassword', 40 | port: 3211, 41 | }, 42 | }); 43 | 44 | (async () => { 45 | await manager.start(); 46 | 47 | const handle = await manager.query(`SELECT * FROM posts`, { 48 | uniqueColumn: 'id', 49 | mode: 'changed', 50 | }); 51 | 52 | handle.on('start', () => { 53 | console.log("query has started"); 54 | }); 55 | 56 | handle.on('ready', () => { 57 | console.log("initial data have been provided"); 58 | }); 59 | 60 | handle.on('refresh', () => { 61 | console.log("all query changes have been provided"); 62 | }); 63 | 64 | handle.on('insert', (row) => { 65 | console.log("row inserted", row); 66 | }); 67 | 68 | handle.on('update', (row, columns) => { 69 | console.log("row updated", row, columns); 70 | }); 71 | 72 | handle.on('delete', (row) => { 73 | console.log("row deleted", row); 74 | }); 75 | 76 | handle.on('error', (error) => { 77 | console.error("query error", error); 78 | }); 79 | 80 | handle.on('stop', (error) => { 81 | console.log("query has stopped", error); 82 | }); 83 | 84 | handle.start(); 85 | })().catch((error) => { 86 | console.error("async error", error); 87 | }); 88 | 89 | process.on('SIGINT', () => { 90 | manager.stop().catch((error) => { 91 | console.error("async error", error); 92 | }).finally(() => { 93 | process.exit(); 94 | }); 95 | }); 96 | ``` 97 | 98 | As a stream: 99 | 100 | ```js 101 | const {Manager} = require('reactive-postgres'); 102 | const through2 = require('through2'); 103 | 104 | const manager = new Manager({ 105 | connectionConfig: { 106 | user: 'dbuser', 107 | host: 'database.server.com', 108 | database: 'mydb', 109 | password: 'secretpassword', 110 | port: 3211, 111 | }, 112 | }); 113 | 114 | const jsonStream = through2.obj(function (chunk, encoding, callback) { 115 | this.push(JSON.stringify(chunk, null, 2) + '\n'); 116 | callback(); 117 | }); 118 | 119 | (async () => { 120 | await manager.start(); 121 | 122 | const handle = await manager.query(`SELECT * FROM posts`, { 123 | uniqueColumn: 'id', 124 | mode: 'changed', 125 | }); 126 | 127 | handle.on('error', (error) => { 128 | console.error("stream error", error); 129 | }); 130 | 131 | handle.on('close', () => { 132 | console.log("stream has closed"); 133 | }); 134 | 135 | handle.pipe(jsonStream).pipe(process.stdout); 136 | })().catch((error) => { 137 | console.error("async error", error); 138 | }); 139 | 140 | process.on('SIGINT', () => { 141 | manager.stop().catch((error) => { 142 | console.error("async error", error); 143 | }).finally(() => { 144 | process.exit(); 145 | }); 146 | }); 147 | ``` 148 | 149 | 150 | ## Design 151 | 152 | Reactive queries are implemented in the following manner: 153 | 154 | * For every reactive query, a `TEMPORARY TABLE` is created in the database 155 | which serves as cache for query results. 156 | * Moreover, the query is `PREPARE`d, so that it does not have to be parsed again 157 | and again. 158 | * Triggers are added to all query sources for the query, so that 159 | when any of sources change, this package is notified using 160 | `LISTEN`/`NOTIFY` that a source has changed, which can 161 | potentially influence the results of the query. 162 | * Package waits for source changed events, and throttles them based 163 | on `refreshThrottleWait` option. Once the delay expires, the package 164 | creates a new temporary table with new query results. It compares 165 | the old and new table and computes changes using another database query. 166 | * Changes are returned the package's client and exposed to the user. 167 | * The old temporary table is dropped and the new one is renamed to take 168 | its place. 169 | 170 | ## Performance 171 | 172 | * Memory use of node.js process is 173 | [very low, is constant and does not grow with the number of rows in query results](https://mitar.github.io/node-pg-reactivity-benchmark/viewer.html?results/reactive-postgres-id.json). 174 | Moreover, node.js process also does no heavy computation 175 | and mostly just passes data around. All this is achieved by caching a query using a 176 | temporary table in the database instead of the client, and using a database query 177 | to compare new and old query results. 178 | * Computing changes is done through one query, a very similar query to the one used 179 | internally by `REFRESH MATERIALIZED VIEW CONCURRENTLY` PostgreSQL command. 180 | * Based on the design, the time to compute changes and provide them to the client 181 | seems to be the lowest when compared with other similar packages. For more 182 | information about performance comparisons of this package and related packages, 183 | see [this benchmark tool](https://github.com/mitar/node-pg-reactivity-benchmark) and 184 | [results at the end](https://github.com/mitar/node-pg-reactivity-benchmark#results). 185 | * Because this package uses temporary tables, consider increasing 186 | [`temp_buffers`](https://www.postgresql.org/docs/devel/runtime-config-resource.html#GUC-TEMP-BUFFERS) 187 | PostgreSQL configuration so that there is more space for temporary tables in memory. 188 | Consider [creating a dedicated tablespace](https://www.postgresql.org/docs/9.4/sql-createtablespace.html) 189 | and configuring [`temp_tablespaces`](https://www.postgresql.org/docs/devel/runtime-config-client.html#GUC-TEMP-TABLESPACES). 190 | * You might consider increasing `refreshThrottleWait` for reactive queries for 191 | which you can tolerate lower refresh rate and higher update latency, to decrease 192 | load on the database. Making too many refreshes for complex queries can saturate 193 | the database which then leads to even higher delays. Paradoxically, having higher 194 | `refreshThrottleWait` could give you lower delay in comparison with a saturated state. 195 | * Multiple reactive queries share the same connection to the database. 196 | So correctly configuring `maxConnections` is important. More connections there are, 197 | higher load is on the database, but over more connections reactive queries can spread. 198 | But higher load on the database can lead to its saturation. 199 | * Currently, when any of sources change in any manner, whole query is rerun 200 | and results compared with cached results (after a throttling delay). 201 | To improve this, refresh could be done only when it is known that a source change 202 | is really influencing the results. Ideally, we could even compute changes to 203 | results directly based on changes to sources. See 204 | [#7](https://github.com/tozd/node-reactive-postgres/issues/7) for more information. 205 | 206 | ## Limitations 207 | 208 | * Queries require an unique column which serves to identify rows and 209 | changes to them. Which column this is is configured through 210 | `uniqueColumn` query option. By default is `id`. 211 | * Order of rows in query results are ignored when determining changes. 212 | Order still matters when selecting which rows are in query results 213 | through `ORDER BY X LIMIT Y` pattern. If you care about order of 214 | query results, order rows on the client. 215 | * Queries cannot contain placeholders or be prepared. You can use 216 | `client.escapeLiteral(...)` function to escape values when constructing 217 | a query. 218 | * To be able to determine if an UPDATE query really changed any source tables used in 219 | your reactive query, source tables should have only columns of types which have an 220 | equality operator defined (e.g., `json` column type does not). If this is not so, 221 | it is just assumed that every UPDATE query makes a change. See 222 | [#7](https://github.com/tozd/node-reactive-postgres/issues/16) for more information. 223 | 224 | ## API 225 | 226 | ### `Manager` 227 | 228 | Manager manages a pool of connections to the database and organizes reactive queries 229 | over them. You should initialize one instance of it and use it for your reactive queries 230 | 231 | ##### `constructor([options])` 232 | 233 | Available `options`: 234 | * `maxConnections`, default `10`: the maximum number of connections to the database 235 | for reactive queries, the final number of connections is `maxConnections` + 1, for the 236 | extra manager's connection 237 | * `connectionConfig`, default `{}`: PostgreSQL connection configuration, 238 | [more information](https://node-postgres.com/api/client#new-client-config-object-) 239 | * `handleClass`: default `ReactiveQueryHandle`: a class to use for reactive query 240 | handles returned by `query` method 241 | 242 | ##### `async start()` 243 | 244 | Initializes the manager, the database, and establishes its connection. 245 | It emits `'start'` event. 246 | 247 | ##### `async stop([error])` 248 | 249 | Stops all reactive queries and the manager itself. 250 | It emits `'stop'` event. Accepts optional `error` argument which is passed 251 | as payload in `'stop'` event. 252 | 253 | ##### `async query(query[, options={}])` 254 | 255 | Constructs a new reactive query and returns a handle. 256 | 257 | `query` can be any arbitrary [`SELECT`](https://www.postgresql.org/docs/devel/sql-select.html), 258 | [`TABLE`](https://www.postgresql.org/docs/10/sql-select.html#SQL-TABLE), or 259 | [`VALUES`](https://www.postgresql.org/docs/10/sql-values.html) command, adhering to 260 | [limitations](#limitations). 261 | 262 | Available `options`: 263 | * `uniqueColumn`, default `id`: the name of an unique column in query results, used 264 | to identify rows and changes to them 265 | * `refreshThrottleWait`, default 100: this option controls that refresh can happen at most once 266 | per every `refreshThrottleWait` milliseconds 267 | * this introduces a minimal delay between a source change and a refresh, you can 268 | control this delay based on requirements for a particular query 269 | * lower this value is, higher the load on the database will be, higher it is, lower the load 270 | will be 271 | * `mode`, default `changed`: in which mode to operate, it can be: 272 | * `columns`: for every query results change, provide only which row and columns changed 273 | * `changed`: for every query results change, provide new values for changed columns, too 274 | * `full`: for every query results change, provide full changed rows, 275 | both columns which have changed and those which have not 276 | * `types`, default `null`: [custom type parsers](https://node-postgres.com/features/queries#types) 277 | 278 | ##### `'start'` event `()` 279 | 280 | Event emitted when manager starts successfully. 281 | 282 | ##### `'connect'` event `(client)` 283 | 284 | Event emitted when a new [PostgreSQL client](https://node-postgres.com/api/client) is created 285 | and connected. `client` is provided as an argument. 286 | 287 | ##### `'disconnect'` event `(client)` 288 | 289 | Event emitted when a PostgreSQL client is disconnected. `client` is provided as an argument. 290 | 291 | ##### `'error'` event `(error[, client])` 292 | 293 | Event emitted when there is an error at the manager level. `error` is provided as an argument. 294 | If the error is associated with a PostgreSQL client, the `client` is provided as well. 295 | 296 | ##### `'stop'` event `([error])` 297 | 298 | Event emitted when the manager stops. If it stopped because of an error, 299 | the `error` is provided as an argument. 300 | 301 | ### `ReactiveQueryHandle` 302 | 303 | Reactive query handles can be used as an [`EventEmitter`](https://nodejs.org/api/events.html#events_class_eventemitter) 304 | or a [`Readable` stream](https://nodejs.org/api/stream.html#stream_readable_streams), 305 | but not both. 306 | 307 | Constructor is seen as private and you should not create instances of `ReactiveQueryHandle` 308 | yourself but always through `Manager`'s `query` method. The method also passes all options 309 | to `ReactiveQueryHandle`. 310 | 311 | #### `EventEmitter` API 312 | 313 | There are more methods, properties, and options available through the 314 | [`EventEmitter` base class](https://nodejs.org/api/events.html#events_class_eventemitter). 315 | 316 | ##### `async start()` 317 | 318 | Initializes the reactive query and starts observing the reactive query, emitting events for 319 | initial results data and later changes to results data. 320 | It emits `'start'` event. 321 | 322 | Having this method separate from constructing a reactive query allows attaching event handlers 323 | before starting, so that no events are missed. 324 | 325 | ##### `async stop([error])` 326 | 327 | Stops the reactive query. 328 | It emits `'stop'` event. Accepts optional `error` argument which is passed 329 | as payload in `'stop'` event. 330 | 331 | ##### `async refresh()` 332 | 333 | Forces the refresh of the reactive query, computation of changes, and emitting 334 | relevant events. This can override `refreshThrottleWait` option which otherwise 335 | controls the minimal delay between a source change and a refresh. 336 | 337 | ##### `async flush()` 338 | 339 | When operating in `changed` or `full` mode, changes are batched together before 340 | a query to fetch data is made. This method forces the query to be made using 341 | currently known changes instead of waiting for a batch as configured by the 342 | `batchSize` option. 343 | 344 | ##### `on(eventName, listener)` and `addListener(eventName, listener)` 345 | 346 | Adds the `listener` function to the end of the listeners array for the event named `eventName`. 347 | [More information](https://nodejs.org/api/events.html#events_emitter_on_eventname_listener). 348 | 349 | ##### `once(eventName, listener)` 350 | 351 | Adds a *one-time* `listener` function for the event named `eventName`. 352 | [More information](https://nodejs.org/api/events.html#events_emitter_once_eventname_listener). 353 | 354 | ##### `off(eventName, listener)` and `removeListener(eventName, listener)` 355 | 356 | Removes the specified `listener` from the listener array for the event named `eventName`. 357 | [More information](https://nodejs.org/api/events.html#events_emitter_removelistener_eventname_listener). 358 | 359 | ##### `'start'` event `()` 360 | 361 | Event emitted when reactive query starts successfully. 362 | 363 | ##### `'ready'` event `()` 364 | 365 | Event emitted when all events for initial results data have been emitted. 366 | Later events are about changes to results data. 367 | 368 | ##### `'refresh'` event `()` 369 | 370 | Event emitted when a refresh has finished and all events for changes to results data 371 | as part of one refresh have been emitted. 372 | 373 | ##### `'insert'` event `(row)` 374 | 375 | Event emitted when a new row has been added to the reactive query. 376 | `row` is provided as an argument. In `columns` mode, `row` contains only the value 377 | of the unique column of the row which has been inserted. In `changed` and `full` modes, 378 | `row` contains full row which has been inserted. 379 | 380 | ##### `'update'` event `(row, columns)` 381 | 382 | Event emitted when a row has been updated. 383 | `row` is provided as an argument. In `columns` mode, `row` contains only the value 384 | of the unique column of the row which has been updated. In `changed` mode, `row` 385 | contains also data for columns which have changed. In `full` mode, `row` 386 | contains the full updated row, both columns which have changed and those which have not. 387 | `columns` is a list of columns which have changed. 388 | 389 | ##### `'delete'` event `(row)` 390 | 391 | Event emitted when a row has been deleted. `row` is provided as an argument. 392 | In `columns` mode, `row` contains only the value 393 | of the unique column of the row which has been deleted. In `changed` and `full` modes, 394 | `row` contains full row which has been deleted. 395 | 396 | ##### `'error'` event `(error)` 397 | 398 | Event emitted when there is an error at the reactive query level. 399 | `error` is provided as an argument. 400 | 401 | ##### `'stop'` event `([error])` 402 | 403 | Event emitted when the reactive query stops. If it stopped because of an error, 404 | the `error` is provided as an argument. 405 | 406 | #### `Readable` stream API 407 | 408 | There are more methods, properties, and options available through the 409 | [`Readable` stream base class](https://nodejs.org/api/stream.html#stream_readable_streams). 410 | 411 | When operating as a stream, the reactive handle is producing objects 412 | describing a change to the reactive query. They are of the following 413 | structure: 414 | 415 | ```js 416 | { 417 | op: , 418 | ... ... 419 | } 420 | ``` 421 | 422 | `event name` matches names of events emitted when operating as an event emitter. 423 | Arguments of those events are converted to object's payload. 424 | 425 | Stream supports backpressure and if consumer of the stream reads slower than 426 | what `refreshThrottleWait` dictates should be the minimal delay between refreshes, 427 | further refreshing is paused until stream is drained. 428 | 429 | ##### `read([size])` 430 | 431 | Reads the next object describing a change to the reactive query, if available. 432 | [More information](https://nodejs.org/api/stream.html#stream_readable_read_size). 433 | 434 | ##### `pipe(destination[, options])` 435 | 436 | The method attaches the stream to the `destination`, using `options`. 437 | [More information](https://nodejs.org/api/stream.html#stream_readable_pipe_destination_options). 438 | 439 | ##### `destroy([error])` 440 | 441 | Destroy the stream and stop the reactive query. It emits `'error'` (if `error` argument 442 | is provided, which is then passed as payload in `'error'` event) and `'close'` events. 443 | [More information](https://nodejs.org/api/stream.html#stream_readable_destroy_error). 444 | 445 | ##### 'data' event `(chunk)` 446 | 447 | Event emitted whenever the stream is relinquishing ownership of a chunk 448 | of data to a consumer. 449 | `chunk` is provided as an argument. 450 | [More information](https://nodejs.org/api/stream.html#stream_event_data). 451 | 452 | ##### 'readable' event `()` 453 | 454 | Event emitted when there is data available to be read from the stream. 455 | [More information](https://nodejs.org/api/stream.html#stream_event_readable). 456 | 457 | ##### 'error' event `(error)` 458 | 459 | Event emitted when there is an error at the reactive query level. 460 | `error` is provided as an argument. 461 | [More information](https://nodejs.org/api/stream.html#stream_event_error_1). 462 | 463 | ##### 'close' event `()` 464 | 465 | Event emitted when the stream has been destroyed. 466 | [More information](https://nodejs.org/api/stream.html#stream_event_close_1). 467 | 468 | ## Related projects 469 | 470 | * [pg-live-select](https://github.com/numtel/pg-live-select) – when the query's sources change, the client reruns the 471 | query to obtain updated rows, which are identified through hashes of their full content and client then computes changes, 472 | on the other hand, this package maintains cached results of the query in a temporary table in the database 473 | and just compares new results with cached results and return updates to the client 474 | * [pg-live-query](https://github.com/nothingisdead/pg-live-query) – adds revision columns to sources and additional 475 | temporary table to store latest revisions for results, moreover, it rewrites queries to expose those additional 476 | revision columns, all this then allows simpler determination of what has changed when the query is refreshed, 477 | inside the database through a query, but in benchmarks it seems all this additional complexity does not really 478 | make things faster in comparison with just comparing new results with cached results directly, which is what this 479 | package does 480 | * [pg-query-observer](https://github.com/Richie765/pg-query-observer) – it seems like a bit cleaned and updated version 481 | of `pg-live-select`, but buggy and does not work with multiple parallel queries 482 | * [pg-reactivity-benchmark](https://github.com/mitar/node-pg-reactivity-benchmark) – a benchmark for this and above 483 | mentioned packages 484 | * [supabase realtime](https://github.com/supabase/realtime) – it broadcasts changes in PostgreSQL over WebSockets 485 | by observing write-ahead log, but it does not support arbitrary reactive queries, just changes to tables themselves, 486 | so it is more of a database replication over WebSockets 487 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const EventEmitter = require('events'); 3 | const {Readable} = require('stream'); 4 | 5 | const AwaitLock = require('await-lock').default; 6 | const {Client} = require('pg'); 7 | const {randomId} = require('@tozd/random-id'); 8 | 9 | const DEFAULT_QUERY_OPTIONS = { 10 | // TODO: Allow multi-column unique index as well. 11 | uniqueColumn: 'id', 12 | refreshThrottleWait: 100, // ms 13 | mode: 'changed', 14 | types: null, 15 | }; 16 | 17 | const OP_MAP = new Map([ 18 | [1, 'insert'], 19 | [2, 'update'], 20 | [3, 'delete'], 21 | ]); 22 | 23 | // TODO: Should we expose an event that the source has changed? 24 | // We might not want this because it is an internal detail. Once we move to 25 | // something like Incremental View Maintenance there will be no such event possible. 26 | // TODO: Should we allow disabling automatic refresh? 27 | // This could allow one to then provide custom logic for refresh. 28 | // We could use a negative value of "refreshThrottleWait" for this. 29 | // But how would the user know when to refresh without an event that source has changed? 30 | class ReactiveQueryHandle extends Readable { 31 | constructor(manager, client, queryId, query, options) { 32 | super({ 33 | // We disable internal buffering in "Readable" because we buffer ourselves. 34 | // We want to detect backpressure as soon as possible so that we do not refresh unnecessary. 35 | highWaterMark: 0, 36 | objectMode: true, 37 | }); 38 | 39 | this.manager = manager; 40 | this.client = client; 41 | this.queryId = queryId; 42 | this.query = query; 43 | this.options = options; 44 | 45 | this._isStream = false; 46 | this._started = false; 47 | this._stopped = false; 48 | this._sources = null; 49 | this._throttleTimestamp = 0; 50 | this._throttleTimeout = null; 51 | this._streamQueue = []; 52 | this._streamPaused = false; 53 | this._sourceChangedPending = false; 54 | this._refreshInProgress = false; 55 | } 56 | 57 | async start() { 58 | if (this._started) { 59 | throw new Error("Query has already been started."); 60 | } 61 | if (this._stopped) { 62 | throw new Error("Query has already been stopped."); 63 | } 64 | 65 | this._started = true; 66 | 67 | let queryExplanation; 68 | await this.client.lock.acquireAsync(); 69 | try { 70 | const results = await this.client.query({ 71 | text: ` 72 | PREPARE "${this.queryId}_query" AS (${this.query}); 73 | EXPLAIN (VERBOSE, FORMAT JSON) EXECUTE "${this.queryId}_query"; 74 | `, 75 | rowMode: 'array', 76 | }); 77 | 78 | queryExplanation = results[1].rows; 79 | } 80 | catch (error) { 81 | this._stopped = true; 82 | 83 | if (!this._isStream) { 84 | this.emit('error', error); 85 | this.emit('stop', error); 86 | } 87 | 88 | throw error; 89 | } 90 | finally { 91 | await this.client.lock.release(); 92 | } 93 | 94 | this._sources = [...this._extractSources(queryExplanation)].sort(); 95 | 96 | // We are just starting, how could it be true? 97 | assert(!this._refreshInProgress); 98 | // We set the flag so that any source changed event gets postponed. 99 | this._refreshInProgress = true; 100 | 101 | try { 102 | // We create triggers for each source in its own transaction instead of all of them 103 | // at once so that we do not have to lock all sources at the same time and potentially 104 | // introduce a deadlock if any of those sources is also used in some other materialized 105 | // view at the same time and it is just being refreshed. This means that source changed 106 | // events could already start arriving before we have a materialized view created, 107 | // but we have them postponed by setting "_refreshInProgress" to "true". 108 | for (const source of this._sources) { 109 | await this.manager.useSource(this.queryId, source); 110 | } 111 | } 112 | catch (error) { 113 | this._stopped = true; 114 | this._refreshInProgress = false; 115 | 116 | if (!this._isStream) { 117 | this.emit('error', error); 118 | this.emit('stop', error); 119 | } 120 | 121 | throw error; 122 | } 123 | 124 | let initialChanges; 125 | await this.client.lock.acquireAsync(); 126 | try { 127 | // We create a temporary table into which we cache current results of the query. 128 | const results = await this.client.query({ 129 | text: ` 130 | START TRANSACTION; 131 | CREATE TEMPORARY TABLE "${this.queryId}_cache" AS EXECUTE "${this.queryId}_query"; 132 | CREATE UNIQUE INDEX "${this.queryId}_cache_id" ON "${this.queryId}_cache" ("${this.options.uniqueColumn}"); 133 | SELECT 1 AS __op__, ARRAY[]::TEXT[] AS __columns__, ${this.options.mode === 'columns' ? `"${this.options.uniqueColumn}"` : '*'} FROM "${this.queryId}_cache"; 134 | COMMIT; 135 | `, 136 | types: this.types, 137 | }); 138 | 139 | initialChanges = results[3].rows; 140 | } 141 | catch (error) { 142 | this._stopped = true; 143 | this._refreshInProgress = false; 144 | 145 | await this.client.query(` 146 | ROLLBACK; 147 | `); 148 | 149 | if (!this._isStream) { 150 | this.emit('error', error); 151 | this.emit('stop', error); 152 | } 153 | 154 | throw error; 155 | } 156 | finally { 157 | await this.client.lock.release(); 158 | } 159 | 160 | if (!this._isStream) { 161 | this.emit('start'); 162 | } 163 | 164 | this._processData(initialChanges); 165 | if (this._isStream) { 166 | this._streamPush({op: 'ready'}); 167 | } 168 | else { 169 | this.emit('ready'); 170 | } 171 | 172 | this._refreshInProgress = false; 173 | 174 | // If we successfully started, let us retry processing a source 175 | // changed event, if it happened while we were starting. If there was 176 | // an error things are probably going bad anyway so one skipped retry 177 | // should not be too problematic in such case. 178 | this._retrySourceChanged(); 179 | } 180 | 181 | async stop(error) { 182 | if (this._isStream) { 183 | await new Promise((resolve, reject) => { 184 | this.once('close', resolve); 185 | this.once('error', reject); 186 | this.destroy(error); 187 | }); 188 | } 189 | else { 190 | await this._stop(error); 191 | } 192 | } 193 | 194 | // TODO: Should we try to cleanup as much as possible even when there are errors? 195 | // Instead of stopping cleanup at the first error? 196 | async _stop(error) { 197 | if (this._stopped) { 198 | return; 199 | } 200 | 201 | this._stopped = true; 202 | 203 | if (!this._started) { 204 | return; 205 | } 206 | 207 | if (this._throttleTimeout) { 208 | clearTimeout(this._throttleTimeout); 209 | this._throttleTimeout = null; 210 | } 211 | 212 | try { 213 | // We drop triggers for each source not in a transaction, and especially not all of them 214 | // at once so that we do not have to lock all sources at the same time and potentially 215 | // introduce a deadlock if any of those sources is also used in some other materialized 216 | // view at the same time and it is just being refreshed. 217 | for (const source of this._sources) { 218 | await this.manager.releaseSource(this.queryId, source); 219 | } 220 | } 221 | catch (error) { 222 | if (!this._isStream) { 223 | this.emit('error', error); 224 | this.emit('stop', error); 225 | } 226 | 227 | throw error; 228 | } 229 | 230 | await this.client.lock.acquireAsync(); 231 | try { 232 | await this.client.query(` 233 | DEALLOCATE "${this.queryId}_query"; 234 | DROP TABLE "${this.queryId}_cache" CASCADE; 235 | `); 236 | } 237 | catch (error) { 238 | if (!this._isStream) { 239 | this.emit('error', error); 240 | this.emit('stop', error); 241 | } 242 | 243 | throw error; 244 | } 245 | finally { 246 | await this.client.lock.release(); 247 | } 248 | 249 | if (!this._isStream) { 250 | this.emit('stop', error); 251 | } 252 | } 253 | 254 | _processData(changes) { 255 | for (const row of changes) { 256 | const op = OP_MAP.get(row.__op__); 257 | assert(op, `Unexpected query changed op '${row.__op__}'.`); 258 | delete row.__op__; 259 | const columns = row.__columns__; 260 | delete row.__columns__; 261 | if (op === 'update') { 262 | // TODO: Select and fetch only changed columns in "changed" mode. 263 | // Currently we select all columns and remove unchanged columns on the client. 264 | if (this.options.mode === 'changed') { 265 | for (const key of Object.keys(row)) { 266 | if (key !== this.options.uniqueColumn && !columns.includes(key)) { 267 | delete row[key]; 268 | } 269 | } 270 | } 271 | if (this._isStream) { 272 | this._streamPush({ 273 | op, 274 | columns, 275 | row, 276 | }); 277 | } 278 | else { 279 | this.emit(op, row, columns); 280 | } 281 | } 282 | else { 283 | if (this._isStream) { 284 | this._streamPush({ 285 | op, 286 | row, 287 | }); 288 | } 289 | else { 290 | this.emit(op, row); 291 | } 292 | } 293 | } 294 | 295 | if (this._isStream) { 296 | this._streamPush({op: 'refresh'}); 297 | } 298 | else { 299 | this.emit('refresh'); 300 | } 301 | } 302 | 303 | async refresh() { 304 | if (this._stopped) { 305 | // This method is not returning anything, so we just ignore the call. 306 | return; 307 | } 308 | if (!this._started) { 309 | throw new Error("Query has not been started."); 310 | } 311 | 312 | // Exit early and not try to lock. 313 | if (this._refreshInProgress) { 314 | return; 315 | } 316 | this._refreshInProgress = true; 317 | 318 | let changes; 319 | await this.client.lock.acquireAsync(); 320 | try { 321 | // Create a new temporary table with new results of the query. 322 | // Computes a diff, swap the tables, and drop the old one. 323 | // TODO: Select and fetch only changed columns in "changed" mode. 324 | // Currently we select all columns and remove unchanged columns on the client. 325 | const results = await this.client.query({ 326 | text: ` 327 | START TRANSACTION; 328 | CREATE TEMPORARY TABLE "${this.queryId}_new" AS EXECUTE "${this.queryId}_query"; 329 | CREATE UNIQUE INDEX "${this.queryId}_new_id" ON "${this.queryId}_new" ("${this.options.uniqueColumn}"); 330 | SELECT 331 | CASE WHEN "${this.queryId}_cache" IS NULL THEN 1 332 | WHEN "${this.queryId}_new" IS NULL THEN 3 333 | ELSE 2 334 | END AS __op__, 335 | CASE WHEN "${this.queryId}_cache" IS NULL THEN ARRAY[]::TEXT[] 336 | WHEN "${this.queryId}_new" IS NULL THEN ARRAY[]::TEXT[] 337 | ELSE (SELECT COALESCE(array_agg(row1.key), ARRAY[]::TEXT[]) FROM each(hstore("${this.queryId}_new")) AS row1 INNER JOIN each(hstore("${this.queryId}_cache")) AS row2 ON (row1.key=row2.key) WHERE row1.value IS DISTINCT FROM row2.value) 338 | END AS __columns__, 339 | (COALESCE("${this.queryId}_new", ROW("${this.queryId}_cache".*)::"${this.queryId}_new")).${this.options.mode === 'columns' ? `"${this.options.uniqueColumn}"` : '*'} 340 | FROM "${this.queryId}_cache" FULL OUTER JOIN "${this.queryId}_new" ON ("${this.queryId}_cache"."${this.options.uniqueColumn}"="${this.queryId}_new"."${this.options.uniqueColumn}") 341 | WHERE "${this.queryId}_cache" IS NULL OR "${this.queryId}_new" IS NULL OR "${this.queryId}_cache" OPERATOR(pg_catalog.*<>) "${this.queryId}_new"; 342 | DROP TABLE "${this.queryId}_cache"; 343 | ALTER TABLE "${this.queryId}_new" RENAME TO "${this.queryId}_cache"; 344 | ALTER INDEX "${this.queryId}_new_id" RENAME TO "${this.queryId}_cache_id"; 345 | COMMIT; 346 | `, 347 | types: this.types, 348 | }); 349 | 350 | changes = results[3].rows; 351 | } 352 | catch (error) { 353 | this._refreshInProgress = false; 354 | 355 | await this.client.query(` 356 | ROLLBACK; 357 | `); 358 | 359 | throw error; 360 | } 361 | finally { 362 | await this.client.lock.release(); 363 | } 364 | 365 | this._processData(changes); 366 | 367 | this._refreshInProgress = false; 368 | 369 | // If we successfully refreshed, let us retry processing a source 370 | // changed event, if it happened while we were refreshing. If there was 371 | // an error things are probably going bad anyway so one skipped retry 372 | // should not be too problematic in such case. 373 | this._retrySourceChanged(); 374 | } 375 | 376 | _onSourceChanged() { 377 | if (!this._started || this._stopped) { 378 | return; 379 | } 380 | 381 | const timestamp = Date.now(); 382 | if (!this._throttleTimestamp) { 383 | this._throttleTimestamp = timestamp; 384 | } 385 | 386 | // If stream is paused, if queue is not empty, or we are in refresh, we set 387 | // a flag to postpone processing this source changed event. We retry once 388 | // stream is not paused anymore or once we emptied the queue. 389 | if (this._streamPaused || this._streamQueue.length || this._refreshInProgress) { 390 | this._sourceChangedPending = true; 391 | return; 392 | } 393 | 394 | const remaining = this.options.refreshThrottleWait - (timestamp - this._throttleTimestamp); 395 | if (remaining <= 0 || remaining > this.options.refreshThrottleWait) { 396 | if (this._throttleTimeout) { 397 | clearTimeout(this._throttleTimeout); 398 | this._throttleTimeout = null; 399 | } 400 | this._throttleTimestamp = timestamp; 401 | this.refresh().catch((error) => { 402 | this.emit('error', error); 403 | }); 404 | } 405 | else if (!this._throttleTimeout) { 406 | this._throttleTimeout = setTimeout(() => { 407 | this._throttleTimestamp = 0; 408 | this._throttleTimeout = null; 409 | this.refresh().catch((error) => { 410 | this.emit('error', error); 411 | }); 412 | }, remaining); 413 | } 414 | } 415 | 416 | _extractSources(queryExplanation) { 417 | let sources = new Set(); 418 | 419 | if (Array.isArray(queryExplanation)) { 420 | for (const element of queryExplanation) { 421 | sources = new Set([...sources, ...this._extractSources(element)]); 422 | } 423 | } 424 | else if (queryExplanation instanceof Object) { 425 | for (const [key, value] of Object.entries(queryExplanation)) { 426 | if (key === 'Relation Name') { 427 | sources.add(qualifySource(queryExplanation['Schema'], value)); 428 | } 429 | else { 430 | sources = new Set([...sources, ...this._extractSources(value)]); 431 | } 432 | } 433 | } 434 | 435 | return sources; 436 | } 437 | 438 | _read(size) { 439 | (async () => { 440 | // We can resume the stream now. This does not necessary mean we also start processing 441 | // source changed events again, because the queue might not yet be empty. 442 | this._resumeStream(); 443 | if (!this._started) { 444 | this._isStream = true; 445 | await this.start(); 446 | } 447 | // We try to get the queue to be empty. If this happens, we will retry processing 448 | // a source changed event (because stream is also not paused anymore), 449 | // if it happened in the past. 450 | this._streamFlush(); 451 | })().catch((error) => { 452 | this.emit('error', error); 453 | }); 454 | } 455 | 456 | _destroy(error, callback) { 457 | (async () => { 458 | await this._stop(); 459 | })().catch((error) => { 460 | callback(error); 461 | }).then(() => { 462 | callback(error); 463 | }); 464 | } 465 | 466 | _streamPush(data) { 467 | this._streamQueue.push(data); 468 | this._streamFlush(); 469 | } 470 | 471 | _streamFlush() { 472 | while (!this._streamPaused && this._streamQueue.length) { 473 | const chunk = this._streamQueue.shift(); 474 | if (!this.push(chunk)) { 475 | this._pauseStream(); 476 | } 477 | } 478 | 479 | if (!this._streamPaused) { 480 | // Stream is not paused which means the queue is empty. We retry processing a 481 | // source changed event, if it happened in the past but we postponed it. 482 | this._retrySourceChanged(); 483 | } 484 | } 485 | 486 | _pauseStream() { 487 | this._streamPaused = true; 488 | if (this._throttleTimeout) { 489 | clearTimeout(this._throttleTimeout); 490 | this._throttleTimeout = null; 491 | } 492 | } 493 | 494 | _resumeStream() { 495 | this._streamPaused = false; 496 | // Stream is not paused anymore, retry processing a source 497 | // changed event, if it happened while we were paused. 498 | this._retrySourceChanged(); 499 | } 500 | 501 | _retrySourceChanged() { 502 | if (this._sourceChangedPending) { 503 | this._sourceChangedPending = false; 504 | this._onSourceChanged(); 505 | } 506 | } 507 | } 508 | 509 | const DEFAULT_MANAGER_OPTIONS = { 510 | maxConnections: 10, 511 | connectionConfig: {}, 512 | handleClass: ReactiveQueryHandle, 513 | }; 514 | 515 | const NOTIFICATION_REGEX = /^(.+)_(source_changed)$/; 516 | 517 | // TODO: Disconnect idle clients after some time. 518 | // Idle meaning that they do not have any reactive queries using them. 519 | class Manager extends EventEmitter { 520 | constructor(options={}) { 521 | super(); 522 | 523 | this.options = Object.assign({}, DEFAULT_MANAGER_OPTIONS, options); 524 | 525 | if (!(this.options.maxConnections > 0)) { 526 | throw new Error("\"maxConnections\" option has to be larger than 0.") 527 | } 528 | 529 | this.managerId = null; 530 | 531 | // Manager's client used for notifications. Queries use their 532 | // own set of clients from a client pool, "this._clients". 533 | this.client = null; 534 | 535 | this._started = false; 536 | this._stopped = false; 537 | // Map between a "queryId" and a reactive query handle. 538 | this._handlesForQuery = new Map(); 539 | // Map between a client and a number of reactive queries using it. 540 | this._clients = new Map(); 541 | this._pendingClients = []; 542 | // Map between a source name and a list of ids of reactive queries using it. 543 | this._sources = new Map(); 544 | } 545 | 546 | async start() { 547 | if (this._started) { 548 | throw new Error("Manager has already been started."); 549 | } 550 | if (this._stopped) { 551 | throw new Error("Manager has already been stopped."); 552 | } 553 | 554 | this._started = true; 555 | 556 | this.managerId = await randomId(); 557 | 558 | try { 559 | this.client = new Client(this.options.connectionConfig); 560 | this.client.lock = new AwaitLock(); 561 | 562 | this.client.on('error', (error) => { 563 | this.emit('error', error, this.client); 564 | }); 565 | this.client.on('end', () => { 566 | this.emit('disconnect', this.client); 567 | }); 568 | this.client.on('notification', (message) => { 569 | this._onNotification(message); 570 | }); 571 | 572 | await this.client.connect(); 573 | 574 | // We define the function as temporary for every client so that triggers using it are dropped when the client disconnects 575 | // (session ends). We have a special case for UPDATE operation because it checks if UPDATE really changed anything 576 | // but that requires that equality operator is defined for all columns. If that is not so, we just assume data has changed 577 | // if UPDATE affected any rows. See: https://github.com/tozd/node-reactive-postgres/issues/16 578 | await this.client.query(` 579 | CREATE OR REPLACE FUNCTION pg_temp.notify_source_changed() RETURNS TRIGGER LANGUAGE plpgsql AS $$ 580 | DECLARE 581 | manager_id TEXT := TG_ARGV[0]; 582 | BEGIN 583 | IF (TG_OP = 'INSERT') THEN 584 | PERFORM * FROM new_rows LIMIT 1; 585 | IF FOUND THEN 586 | EXECUTE 'NOTIFY "' || manager_id || '_source_changed", ''{"name": "' || TG_TABLE_NAME || '", "schema": "' || TG_TABLE_SCHEMA || '"}'''; 587 | END IF; 588 | ELSIF (TG_OP = 'UPDATE') THEN 589 | BEGIN 590 | PERFORM * FROM ((TABLE old_rows EXCEPT TABLE new_rows) UNION ALL (TABLE new_rows EXCEPT TABLE old_rows)) AS differences LIMIT 1; 591 | IF FOUND THEN 592 | EXECUTE 'NOTIFY "' || manager_id || '_source_changed", ''{"name": "' || TG_TABLE_NAME || '", "schema": "' || TG_TABLE_SCHEMA || '"}'''; 593 | END IF; 594 | EXCEPTION WHEN undefined_function THEN 595 | PERFORM * FROM new_rows LIMIT 1; 596 | IF FOUND THEN 597 | EXECUTE 'NOTIFY "' || manager_id || '_source_changed", ''{"name": "' || TG_TABLE_NAME || '", "schema": "' || TG_TABLE_SCHEMA || '"}'''; 598 | END IF; 599 | END; 600 | ELSIF (TG_OP = 'DELETE') THEN 601 | PERFORM * FROM old_rows LIMIT 1; 602 | IF FOUND THEN 603 | EXECUTE 'NOTIFY "' || manager_id || '_source_changed", ''{"name": "' || TG_TABLE_NAME || '", "schema": "' || TG_TABLE_SCHEMA || '"}'''; 604 | END IF; 605 | ELSIF (TG_OP = 'TRUNCATE') THEN 606 | EXECUTE 'NOTIFY "' || manager_id || '_source_changed", ''{"name": "' || TG_TABLE_NAME || '", "schema": "' || TG_TABLE_SCHEMA || '"}'''; 607 | END IF; 608 | RETURN NULL; 609 | END 610 | $$; 611 | `); 612 | 613 | // We use hstore to compute which columns changed. 614 | // TODO: It is possible that the user does not have permissions to create extensions. 615 | await this.client.query(` 616 | CREATE EXTENSION IF NOT EXISTS hstore; 617 | `); 618 | 619 | this.emit('connect', this.client); 620 | } 621 | catch (error) { 622 | this._stopped = true; 623 | 624 | this.emit('error', error); 625 | this.emit('stop', error); 626 | 627 | throw error; 628 | } 629 | 630 | this.emit('start'); 631 | } 632 | 633 | // TODO: Should we try to cleanup as much as possible even when there are errors? 634 | // Instead of stopping cleanup at the first error? 635 | async stop(error) { 636 | if (this._stopped) { 637 | return; 638 | } 639 | 640 | this._stopped = true; 641 | 642 | if (!this._started) { 643 | return; 644 | } 645 | 646 | try { 647 | while (this._handlesForQuery.size) { 648 | const promises = []; 649 | for (const [queryId, handle] of this._handlesForQuery.entries()) { 650 | // "stop" triggers "end" callback which removes the handle. 651 | promises.push(handle.stop()); 652 | } 653 | await Promise.all(promises); 654 | } 655 | 656 | // All sources should be released when we called "stop" on all handles. 657 | assert(this._sources.size === 0, "\"sources\" should be empty."); 658 | 659 | // They should all be removed now through "end" callbacks when we 660 | // called "stop" on all handles. 661 | assert(this._handlesForQuery.size === 0, "\"handlesForQuery\" should be empty."); 662 | 663 | // Disconnect all clients. 664 | while (this._clients.size) { 665 | for (const [client, utilization] of this._clients.entries()) { 666 | assert(utilization === 0, "Utilization of a client should be zero."); 667 | 668 | await client.end(); 669 | this._clients.delete(client); 670 | } 671 | } 672 | 673 | await this.client.end(); 674 | } 675 | catch (error) { 676 | this.emit('error', error); 677 | this.emit('stop', error); 678 | 679 | throw error; 680 | } 681 | 682 | this.emit('stop', error); 683 | } 684 | 685 | async _getClient() { 686 | if (this._stopped) { 687 | // This method is returning a client, so we throw. 688 | throw new Error("Manager has been stopped."); 689 | } 690 | if (!this._started) { 691 | throw new Error("Manager has not been started."); 692 | } 693 | 694 | // If we do not yet have all connections open, make a new one now. 695 | if (this._clients.size + this._pendingClients.length < this.options.maxConnections) { 696 | const clientPromise = this._createClient(); 697 | clientPromise.then((client) => { 698 | this._clients.set(client, 0); 699 | const index = this._pendingClients.indexOf(clientPromise); 700 | if (index >= 0) { 701 | this._pendingClients.splice(index, 1); 702 | } 703 | }).catch((error) => { 704 | this.emit('error', error); 705 | }); 706 | this._pendingClients.push(clientPromise); 707 | } 708 | 709 | await Promise.all(this._pendingClients); 710 | 711 | assert(this._clients.size > 0, "\"maxConnections\" has to be larger than 0."); 712 | 713 | // Find clients with the least number of active queries. We want 714 | // to distribute all queries between all clients equally. 715 | let availableClients = []; 716 | let lowestUtilization = Number.POSITIVE_INFINITY; 717 | for (const [client, utilization] of this._clients.entries()) { 718 | if (utilization < lowestUtilization) { 719 | lowestUtilization = utilization; 720 | availableClients = [client]; 721 | } 722 | else if (utilization === lowestUtilization) { 723 | availableClients.push(client); 724 | } 725 | } 726 | 727 | // We pick a random client from available clients. 728 | const client = availableClients[Math.floor(Math.random() * availableClients.length)]; 729 | 730 | // We mark it as used. 731 | this._useClient(client); 732 | 733 | // And return it. 734 | return client; 735 | } 736 | 737 | _setHandleForQuery(handle, queryId) { 738 | this._handlesForQuery.set(queryId, handle); 739 | } 740 | 741 | _getHandleForQuery(queryId) { 742 | return this._handlesForQuery.get(queryId); 743 | } 744 | 745 | _deleteHandleForQuery(queryId) { 746 | this._handlesForQuery.delete(queryId); 747 | } 748 | 749 | _useClient(client) { 750 | this._clients.set(client, this._clients.get(client) + 1); 751 | } 752 | 753 | _releaseClient(client) { 754 | this._clients.set(client, this._clients.get(client) - 1); 755 | } 756 | 757 | async useSource(queryId, source) { 758 | if (!this._sources.has(source)) { 759 | this._sources.set(source, []); 760 | } 761 | this._sources.get(source).push(queryId); 762 | 763 | // The first time this source is being used. 764 | if (this._sources.get(source).length === 1) { 765 | await this.client.lock.acquireAsync(); 766 | try { 767 | await this.client.query(` 768 | LISTEN "${this.managerId}_source_changed"; 769 | `); 770 | 771 | try { 772 | await this.client.query(` 773 | START TRANSACTION; 774 | CREATE TRIGGER "${this.managerId}_source_changed_${escapeSource(source)}_insert" AFTER INSERT ON ${source} REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION pg_temp.notify_source_changed('${this.managerId}'); 775 | CREATE TRIGGER "${this.managerId}_source_changed_${escapeSource(source)}_update" AFTER UPDATE ON ${source} REFERENCING NEW TABLE AS new_rows OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION pg_temp.notify_source_changed('${this.managerId}'); 776 | CREATE TRIGGER "${this.managerId}_source_changed_${escapeSource(source)}_delete" AFTER DELETE ON ${source} REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION pg_temp.notify_source_changed('${this.managerId}'); 777 | COMMIT; 778 | `); 779 | } 780 | catch (error) { 781 | await this.client.query(` 782 | ROLLBACK; 783 | `); 784 | 785 | throw error; 786 | } 787 | 788 | try { 789 | await this.client.query(` 790 | CREATE TRIGGER "${this.managerId}_source_changed_${escapeSource(source)}_truncate" AFTER TRUNCATE ON ${source} FOR EACH STATEMENT EXECUTE FUNCTION pg_temp.notify_source_changed('${this.managerId}'); 791 | `); 792 | } 793 | // Ignoring errors. The source might not support TRUNCATE trigger. 794 | // For example, tables do, but views do not. 795 | catch (error) {} 796 | } 797 | catch (error) { 798 | this.emit('error', error); 799 | 800 | throw error; 801 | } 802 | finally { 803 | await this.client.lock.release(); 804 | } 805 | } 806 | } 807 | 808 | async releaseSource(queryId, source) { 809 | const handles = this._sources.get(source) || []; 810 | const index = handles.indexOf(queryId); 811 | if (index >= 0) { 812 | handles.splice(index, 1); 813 | } 814 | else { 815 | console.warn(`Releasing a source '${source}' which is not being used by queryId '${queryId}'.`); 816 | return; 817 | } 818 | 819 | // Source is not used by anyone anymore. 820 | if (handles.length === 0) { 821 | this._sources.delete(source); 822 | 823 | await this.client.lock.acquireAsync(); 824 | try { 825 | await this.client.query(` 826 | UNLISTEN "${this.managerId}_source_changed"; 827 | DROP TRIGGER IF EXISTS "${this.managerId}_source_changed_${escapeSource(source)}_insert" ON ${source}; 828 | DROP TRIGGER IF EXISTS "${this.managerId}_source_changed_${escapeSource(source)}_update" ON ${source}; 829 | DROP TRIGGER IF EXISTS "${this.managerId}_source_changed_${escapeSource(source)}_delete" ON ${source}; 830 | DROP TRIGGER IF EXISTS "${this.managerId}_source_changed_${escapeSource(source)}_truncate" ON ${source}; 831 | `); 832 | } 833 | catch (error) { 834 | this.emit('error', error); 835 | 836 | throw error; 837 | } 838 | finally { 839 | await this.client.lock.release(); 840 | } 841 | } 842 | } 843 | 844 | async _createClient() { 845 | const client = new Client(this.options.connectionConfig); 846 | client.lock = new AwaitLock(); 847 | 848 | client.on('error', (error) => { 849 | this.emit('error', error, client); 850 | }); 851 | client.on('end', () => { 852 | this.emit('disconnect', client); 853 | }); 854 | 855 | await client.connect(); 856 | 857 | this.emit('connect', client); 858 | 859 | return client; 860 | } 861 | 862 | async query(query, options={}) { 863 | if (this._stopped) { 864 | // This method is returning a handle, so we throw. 865 | throw new Error("Manager has been stopped."); 866 | } 867 | if (!this._started) { 868 | throw new Error("Manager has not been started."); 869 | } 870 | 871 | options = Object.assign({}, DEFAULT_QUERY_OPTIONS, options); 872 | 873 | // We generate ID outside of the constructor so that it can be async. 874 | const queryId = await randomId(); 875 | 876 | const client = await this._getClient(); 877 | try { 878 | const handle = new this.options.handleClass(this, client, queryId, query, options); 879 | this._setHandleForQuery(handle, queryId); 880 | handle.once('stop', (error) => { 881 | this._deleteHandleForQuery(queryId); 882 | this._releaseClient(client); 883 | }); 884 | handle.once('close', () => { 885 | this._deleteHandleForQuery(queryId); 886 | this._releaseClient(client); 887 | }); 888 | return handle; 889 | } 890 | catch (error) { 891 | // There was an error, client will not really be used. 892 | this._releaseClient(client); 893 | 894 | throw error; 895 | } 896 | } 897 | 898 | _onNotification(message) { 899 | if (!this._started || this._stopped) { 900 | return; 901 | } 902 | 903 | const channel = message.channel; 904 | 905 | const match = NOTIFICATION_REGEX.exec(channel); 906 | if (!match) { 907 | return; 908 | } 909 | 910 | const payload = JSON.parse(message.payload); 911 | 912 | const managerId = match[1]; 913 | const notificationType = match[2]; 914 | 915 | assert(managerId === this.managerId, "Notification id should match manager's id."); 916 | 917 | if (notificationType === 'source_changed') { 918 | // We ignore notifications for unknown sources. 919 | const source = qualifySource(payload.schema, payload.name); 920 | for (const queryId of (this._sources.get(source) || [])) { 921 | const handle = this._getHandleForQuery(queryId); 922 | if (handle) { 923 | handle._onSourceChanged(); 924 | } 925 | } 926 | } 927 | else { 928 | console.warn(`Unknown notification type '${notificationType}'.`); 929 | } 930 | } 931 | } 932 | 933 | function qualifySource(schema, name) { 934 | return `"${schema}"."${name}"`; 935 | } 936 | 937 | function escapeSource(source) { 938 | return source.replace(/[^a-z0-9_]/gi, '_'); 939 | } 940 | 941 | module.exports = { 942 | Manager, 943 | ReactiveQueryHandle, 944 | }; 945 | --------------------------------------------------------------------------------