├── .gitignore ├── .npmignore ├── .travis.yml ├── LICENSE ├── README.md ├── cjs ├── index.js └── package.json ├── esm └── index.js ├── package.json └── test ├── exit.js ├── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .nyc_output 3 | node_modules/ 4 | package-lock.json 5 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .nyc_output 3 | .travis.yml 4 | node_modules/ 5 | rollup/ 6 | test/ 7 | package-lock.json 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - stable 4 | git: 5 | depth: 1 6 | branches: 7 | only: 8 | - master 9 | after_success: 10 | - "npm run coveralls" 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ISC License 2 | 3 | Copyright (c) 2020, Andrea Giammarchi, @WebReflection 4 | 5 | Permission to use, copy, modify, and/or distribute this software for any 6 | purpose with or without fee is hereby granted, provided that the above 7 | copyright notice and this permission notice appear in all copies. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 14 | OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | PERFORMANCE OF THIS SOFTWARE. 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # id-promise 2 | 3 | [![Build Status](https://travis-ci.com/WebReflection/id-promise.svg?branch=master)](https://travis-ci.com/WebReflection/id-promise) [![Coverage Status](https://coveralls.io/repos/github/WebReflection/id-promise/badge.svg?branch=master)](https://coveralls.io/github/WebReflection/id-promise?branch=master) 4 | 5 | The goal of this module is to hold on the very first promise that asks for a specific task, resolved once for all concurrent promises that meanwhile asked for that very same task during its resolution time. 6 | 7 | ## In a nutshell 8 | 9 | In the following example, after the first `samePromise()`, all other `samePromise()` invokes will simply hold until the first invoke has been resolved, granting that for 300 ms, dictated in the following example by `setTimeout`, no extra timer will be set, and `++samePromiseCalls` won't be incremented more than once. 10 | 11 | ```js 12 | import idPromise from 'id-promise'; 13 | // const idPromise = require('id-promise'); 14 | 15 | let samePromiseCalls = 0; 16 | const samePromise = () => idPromise( 17 | 'some-unique-id:samePromise', 18 | (resolve, reject) => { 19 | setTimeout(resolve, 300, ++samePromiseCalls); 20 | } 21 | ); 22 | 23 | // ask for the same task as many times as you want 24 | samePromise().then(console.log); 25 | samePromise().then(console.log); 26 | samePromise().then(console.log); 27 | samePromise().then(console.log); 28 | ``` 29 | 30 | ### Cluster Friendly 31 | 32 | If the callback is executed within a _forked_ worker, it will put on hold the same _id_ for all workers that meanwhile might ask for the same operation. 33 | 34 | This is specially useful when a single _fork_ would need to perform a potentially very expensive operation, either DB or file system related, but it shouldn't perform such operation more than once, as both DB and file system are shared across all workers. 35 | 36 | ```js 37 | import idPromise from 'id-promise'; 38 | // const idPromise = require('id-promise'); 39 | 40 | const optimizePath = path => idPromise( 41 | // ⚠ use strong identifiers, not just path! 42 | `my-project:optimizePath:${path}`, 43 | (resolve, reject) => { 44 | performSomethingVeryExpensive(path) 45 | .then(resolve, reject); 46 | } 47 | ); 48 | 49 | // invoke it as many times as you need 50 | optimizePath(__filename).then(...); 51 | optimizePath(__filename).then(...); 52 | optimizePath(__filename).then(...); 53 | ``` 54 | 55 | ### How does it work 56 | 57 | In _master_, each _unique id_ is simply stored once, and removed from the _Map_ based cache once resolved, or rejected. Each call to the same _unique id_ will return the very same promise that is in charge of resolving or rejecting. 58 | 59 | In _workers_, each _uunique id_ would pass through _master_ to understand if other workers asked for it already, or it should be executed as task. 60 | The task is then eventually executed within the single _worker_ and, once resolved, propagated to every other possible worker that meanwhile asked for the same task. 61 | 62 | 63 | ## Caveats 64 | 65 | This module has been created to solve some very specific use case and it's important to understand where it easily fails. 66 | 67 | There are 3 kinds of caveats to consider with this module: 68 | 69 | * **name clashes**, so that weak unique identifiers will easily cause troubles. Try to use your project/module namespace as prefix, plus the functionality, plus any other static information that summed to the previous details would make the operation really unique (i.e. a fully resolved file path) 70 | * **serialization**, so that you cannot resolve values that cannot be serialized and passed around workers, and you should rather stick with _JSON_ compatible values only. 71 | * **different parameters**, so that if a promise is cached but the next call internally refers to different values, the result might be unexpected 72 | 73 | While the first caveat is quite easy to understand, the last one is subtle: 74 | 75 | ```js 76 | import idPromise from 'id-promise'; 77 | // const idPromise = require('id-promise'); 78 | 79 | const writeOnce = (where, what) => idPromise( 80 | `my-project:writeOnce:${where}`, 81 | (resolve, reject) => { 82 | fs.writeFile(where, what, err => { 83 | if (err) reject(err); 84 | else resolve(what); 85 | }); 86 | } 87 | ); 88 | 89 | // concurrent writes 90 | writeOnce('/tmp/test.txt', 'a').then(console.log); 91 | writeOnce('/tmp/test.txt', 'b').then(console.log); 92 | writeOnce('/tmp/test.txt', 'c').then(console.log); 93 | ``` 94 | 95 | Above concurrent `writeOnce(where, what)` invokes uses the same _id_ with different values to write. Accordingly with how fast the writing operation would be, the outcome might be unpredictable, but in the worst case scenario, where it was something very expensive, all 3 invokes will resolve with the string `"a"`. 96 | 97 | The rule of thumbs here is that _First Come, First Serve_, so specifically for writing files this module might be not the solution. 98 | 99 | 100 | ## Use cases 101 | 102 | * expensive operations that don't need to be performed frequently, including recursive asynchronous folders crawling or scanning 103 | * expensive file operations such as compression, archive zipping, archive extraction, and so on and so forth, where the source path is unique and operation would grant always the same outcome 104 | * any expensive operation that accepts *always* a unique entry point that should grant always the same outcome 105 | -------------------------------------------------------------------------------- /cjs/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /*! 3 | * ISC License 4 | * 5 | * Copyright (c) 2020, Andrea Giammarchi, @WebReflection 6 | * 7 | * Permission to use, copy, modify, and/or distribute this software for any 8 | * purpose with or without fee is hereby granted, provided that the above 9 | * copyright notice and this permission notice appear in all copies. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 12 | * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 13 | * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 14 | * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 15 | * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 16 | * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 17 | * PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | const cluster = (m => m.__esModule ? /* istanbul ignore next */ m.default : /* istanbul ignore next */ m)(require('cluster')); 20 | const {pid, ppid} = require('process'); 21 | 22 | const {isMaster} = cluster; 23 | const reject = Promise.reject.bind(Promise); 24 | 25 | const CHANNEL = `\x01I'd promise ${isMaster ? pid : ppid}\x01`; 26 | const EXECUTE = 'execute'; 27 | const REJECT = 'reject'; 28 | const RESOLVE = 'resolve'; 29 | const VERIFY = new RegExp(`^${CHANNEL}:`); 30 | 31 | const cache = new Map; 32 | 33 | const getError = error => typeof error == 'object' ? 34 | {message: error.message, stack: error.stack} : 35 | /* istanbul ignore next */ 36 | String(error) 37 | ; 38 | 39 | const resolvable = () => { 40 | let $, _; 41 | const promise = new Promise((resolve, reject) => { 42 | $ = resolve; 43 | _ = reject; 44 | }); 45 | promise.$ = $; 46 | promise._ = _; 47 | return promise; 48 | }; 49 | 50 | if (isMaster) { 51 | const clusters = new Map; 52 | const send = (workers, worker, uid, message) => { 53 | clusters.delete(uid); 54 | for (const id in workers) { 55 | if (id != worker) 56 | workers[id].send(message); 57 | } 58 | }; 59 | const onMessage = message => { 60 | /* istanbul ignore else */ 61 | if (typeof message === 'object') { 62 | const {id: uid, worker, action, error, result} = message; 63 | /* istanbul ignore else */ 64 | if (typeof uid === 'string' && VERIFY.test(uid)) { 65 | const {workers} = cluster; 66 | if (action === EXECUTE) { 67 | if (!clusters.has(uid)) { 68 | clusters.set(uid, worker); 69 | workers[worker].send({id: uid, action}); 70 | } 71 | } 72 | else { 73 | const resolved = action === RESOLVE; 74 | const key = resolved ? 'result' : 'error'; 75 | const value = resolved ? result : error; 76 | send(workers, worker, uid, {id: uid, [key]: value, action}); 77 | } 78 | } 79 | } 80 | }; 81 | cluster 82 | .on('fork', worker => { 83 | worker.on('message', onMessage); 84 | }) 85 | .on('exit', (worker, code) => { 86 | /* istanbul ignore next */ 87 | clusters.forEach((id, uid) => { 88 | if (id == worker.id) { 89 | const error = `id ${uid.slice(CHANNEL.length + 1)} failed with code ${code}`; 90 | send(cluster.workers, id, uid, {id: uid, error, action: REJECT}); 91 | } 92 | }); 93 | }); 94 | } 95 | 96 | const set = (id, callback, promise) => { 97 | const {$, _} = promise; 98 | if (isMaster) { 99 | cache.set(id, promise = promise.then( 100 | result => { 101 | cache.delete(id); 102 | return result; 103 | }, 104 | error => { 105 | cache.delete(id); 106 | return reject(error); 107 | } 108 | )); 109 | callback($, _); 110 | } 111 | else { 112 | let main = false; 113 | const worker = cluster.worker.id; 114 | cache.set(id, promise = promise.then( 115 | result => { 116 | cache.delete(id); 117 | if (main) 118 | process.send({id, worker, result, action: RESOLVE}); 119 | return result; 120 | }, 121 | error => { 122 | cache.delete(id); 123 | if (main) 124 | process.send({id, worker, error: getError(error), action: REJECT}); 125 | return reject(error); 126 | } 127 | )); 128 | process.on('message', function listener(message) { 129 | /* istanbul ignore else */ 130 | if (typeof message === 'object') { 131 | const {id: uid, action, error, result} = message; 132 | /* istanbul ignore else */ 133 | if (uid === id) { 134 | process.removeListener('message', listener); 135 | switch (action) { 136 | case EXECUTE: 137 | main = true; 138 | callback($, _); 139 | break; 140 | case RESOLVE: 141 | $(result); 142 | break; 143 | case REJECT: 144 | _(error); 145 | break; 146 | } 147 | } 148 | } 149 | }); 150 | process.send({id, worker, action: EXECUTE}); 151 | } 152 | return promise; 153 | }; 154 | 155 | module.exports = (id, callback) => { 156 | id = `${CHANNEL}:${id}`; 157 | return cache.get(id) || set(id, callback, resolvable()); 158 | }; 159 | -------------------------------------------------------------------------------- /cjs/package.json: -------------------------------------------------------------------------------- 1 | {"type":"commonjs"} -------------------------------------------------------------------------------- /esm/index.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * ISC License 3 | * 4 | * Copyright (c) 2020, Andrea Giammarchi, @WebReflection 5 | * 6 | * Permission to use, copy, modify, and/or distribute this software for any 7 | * purpose with or without fee is hereby granted, provided that the above 8 | * copyright notice and this permission notice appear in all copies. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 11 | * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 12 | * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 13 | * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 14 | * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 15 | * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 | * PERFORMANCE OF THIS SOFTWARE. 17 | */ 18 | import cluster from 'cluster'; 19 | import {pid, ppid} from 'process'; 20 | 21 | const {isMaster} = cluster; 22 | const reject = Promise.reject.bind(Promise); 23 | 24 | const CHANNEL = `\x01I'd promise ${isMaster ? pid : ppid}\x01`; 25 | const EXECUTE = 'execute'; 26 | const REJECT = 'reject'; 27 | const RESOLVE = 'resolve'; 28 | const VERIFY = new RegExp(`^${CHANNEL}:`); 29 | 30 | const cache = new Map; 31 | 32 | const getError = error => typeof error == 'object' ? 33 | {message: error.message, stack: error.stack} : 34 | /* istanbul ignore next */ 35 | String(error) 36 | ; 37 | 38 | const resolvable = () => { 39 | let $, _; 40 | const promise = new Promise((resolve, reject) => { 41 | $ = resolve; 42 | _ = reject; 43 | }); 44 | promise.$ = $; 45 | promise._ = _; 46 | return promise; 47 | }; 48 | 49 | if (isMaster) { 50 | const clusters = new Map; 51 | const send = (workers, worker, uid, message) => { 52 | clusters.delete(uid); 53 | for (const id in workers) { 54 | if (id != worker) 55 | workers[id].send(message); 56 | } 57 | }; 58 | const onMessage = message => { 59 | /* istanbul ignore else */ 60 | if (typeof message === 'object') { 61 | const {id: uid, worker, action, error, result} = message; 62 | /* istanbul ignore else */ 63 | if (typeof uid === 'string' && VERIFY.test(uid)) { 64 | const {workers} = cluster; 65 | if (action === EXECUTE) { 66 | if (!clusters.has(uid)) { 67 | clusters.set(uid, worker); 68 | workers[worker].send({id: uid, action}); 69 | } 70 | } 71 | else { 72 | const resolved = action === RESOLVE; 73 | const key = resolved ? 'result' : 'error'; 74 | const value = resolved ? result : error; 75 | send(workers, worker, uid, {id: uid, [key]: value, action}); 76 | } 77 | } 78 | } 79 | }; 80 | cluster 81 | .on('fork', worker => { 82 | worker.on('message', onMessage); 83 | }) 84 | .on('exit', (worker, code) => { 85 | /* istanbul ignore next */ 86 | clusters.forEach((id, uid) => { 87 | if (id == worker.id) { 88 | const error = `id ${uid.slice(CHANNEL.length + 1)} failed with code ${code}`; 89 | send(cluster.workers, id, uid, {id: uid, error, action: REJECT}); 90 | } 91 | }); 92 | }); 93 | } 94 | 95 | const set = (id, callback, promise) => { 96 | const {$, _} = promise; 97 | if (isMaster) { 98 | cache.set(id, promise = promise.then( 99 | result => { 100 | cache.delete(id); 101 | return result; 102 | }, 103 | error => { 104 | cache.delete(id); 105 | return reject(error); 106 | } 107 | )); 108 | callback($, _); 109 | } 110 | else { 111 | let main = false; 112 | const worker = cluster.worker.id; 113 | cache.set(id, promise = promise.then( 114 | result => { 115 | cache.delete(id); 116 | if (main) 117 | process.send({id, worker, result, action: RESOLVE}); 118 | return result; 119 | }, 120 | error => { 121 | cache.delete(id); 122 | if (main) 123 | process.send({id, worker, error: getError(error), action: REJECT}); 124 | return reject(error); 125 | } 126 | )); 127 | process.on('message', function listener(message) { 128 | /* istanbul ignore else */ 129 | if (typeof message === 'object') { 130 | const {id: uid, action, error, result} = message; 131 | /* istanbul ignore else */ 132 | if (uid === id) { 133 | process.removeListener('message', listener); 134 | switch (action) { 135 | case EXECUTE: 136 | main = true; 137 | callback($, _); 138 | break; 139 | case RESOLVE: 140 | $(result); 141 | break; 142 | case REJECT: 143 | _(error); 144 | break; 145 | } 146 | } 147 | } 148 | }); 149 | process.send({id, worker, action: EXECUTE}); 150 | } 151 | return promise; 152 | }; 153 | 154 | export default (id, callback) => { 155 | id = `${CHANNEL}:${id}`; 156 | return cache.get(id) || set(id, callback, resolvable()); 157 | }; 158 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "id-promise", 3 | "version": "0.3.0", 4 | "description": "A cluster friendly, identity based, Promise resolver", 5 | "main": "./cjs/index.js", 6 | "scripts": { 7 | "build": "npm run cjs && npm run test", 8 | "cjs": "ascjs --no-default esm cjs", 9 | "coveralls": "nyc report --reporter=text-lcov | coveralls", 10 | "test": "nyc node test/index.js" 11 | }, 12 | "keywords": [ 13 | "Promise", 14 | "unique", 15 | "lazy", 16 | "cached", 17 | "cluster" 18 | ], 19 | "author": "Andrea Giammarchi", 20 | "license": "ISC", 21 | "devDependencies": { 22 | "ascjs": "^4.0.1", 23 | "coveralls": "^3.1.0", 24 | "nyc": "^15.1.0" 25 | }, 26 | "module": "./esm/index.js", 27 | "type": "module", 28 | "exports": { 29 | "import": "./esm/index.js", 30 | "default": "./cjs/index.js" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /test/exit.js: -------------------------------------------------------------------------------- 1 | const cluster = require('cluster'); 2 | 3 | const idPromise = require('../cjs'); 4 | 5 | const {fork, isMaster} = cluster; 6 | 7 | if (isMaster) { 8 | fork(); 9 | fork(); 10 | setTimeout(process.exit.bind(process), 2000, 0); 11 | } 12 | else { 13 | idPromise('exit:ok', res => { 14 | setTimeout(res, 500, 'id exit:ok was executed'); 15 | }).then(console.log); 16 | idPromise('exit:error', () => { 17 | setTimeout(() => { process.exit(1); }, 1000); 18 | }).catch(console.error); 19 | idPromise('exit:late', res => { 20 | setTimeout(res, 1500, 'OK'); 21 | }).catch(console.error); 22 | } 23 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | const cluster = require('cluster'); 2 | 3 | const idPromise = require('../cjs'); 4 | 5 | const {fork, isMaster} = cluster; 6 | 7 | let samePromiseCalls = 0; 8 | const samePromise = () => idPromise('id-promise:timer', res => { 9 | setTimeout(res, 300, ++samePromiseCalls); 10 | }); 11 | 12 | let errorsCalls = 0; 13 | const rejectingPromise = () => idPromise('id-promise:throw', (_, rej) => { 14 | setTimeout(() => { 15 | rej(new Error(++errorsCalls)); 16 | }, 600); 17 | }); 18 | 19 | if (isMaster) { 20 | Promise.all([ 21 | samePromise(), 22 | samePromise(), 23 | samePromise() 24 | ]) 25 | .then(([a, b, c]) => { 26 | console.log('master', a); 27 | console.assert(a == 1 && b === a && c === a, 'master works'); 28 | }); 29 | Promise.all([ 30 | rejectingPromise(), 31 | rejectingPromise() 32 | ]).catch(error => { 33 | console.log('master error', error.message); 34 | console.assert(error.message == 1, 'master can reject'); 35 | }); 36 | fork(); 37 | fork(); 38 | fork(); 39 | } 40 | else { 41 | samePromise().then( 42 | result => { 43 | console.log('worker', result); 44 | console.assert(result === 1, 'worker works'); 45 | } 46 | ); 47 | rejectingPromise().catch(error => { 48 | console.log('worker error', error.message); 49 | console.assert(error.message == 1, 'worker can reject'); 50 | setTimeout(() => { 51 | cluster.worker.kill(); 52 | }, 900); 53 | }); 54 | } 55 | -------------------------------------------------------------------------------- /test/package.json: -------------------------------------------------------------------------------- 1 | {"type":"commonjs"} --------------------------------------------------------------------------------