├── .npmignore ├── README.md ├── index.js ├── jest.config.js ├── .gitignore ├── eslint.config.js ├── LICENSE.md ├── package.json ├── lib └── timebatch.js └── __tests__ └── timebatch.test.js /.npmignore: -------------------------------------------------------------------------------- 1 | __tests__ 2 | .github 3 | coverage 4 | tmp 5 | *.tgz -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Batcher 2 | 3 | > Classes to assist batch grouping and reconciliation 4 | 5 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Batcher 3 | * 4 | * Copyright (c) 2025 Alex Grant (@localnerve), LocalNerve LLC 5 | * Private use for LocalNerve, LLC only. Unlicensed for any other use. 6 | */ 7 | 8 | export { TimeBatch } from './lib/timebatch.js'; -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | collectCoverage: true, 3 | coverageDirectory: 'coverage', 4 | verbose: true, 5 | testEnvironment: 'node', 6 | testPathIgnorePatterns: [ 7 | '/node_modules/', 8 | '/tmp/' 9 | ] 10 | }; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | *.tgz 11 | 12 | tmp 13 | dist 14 | 15 | # Dependency directory 16 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 17 | node_modules 18 | 19 | coverage -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | import js from '@eslint/js'; 2 | import globals from 'globals'; 3 | import jest from 'eslint-plugin-jest'; 4 | 5 | export default [{ 6 | name: 'global', 7 | ignores: [ 8 | 'coverage/**', 9 | 'tmp/**', 10 | 'node_modules/**' 11 | ] 12 | }, { 13 | name: 'lib', 14 | ignores: [ 15 | '__tests__/**' 16 | ], 17 | files: [ 18 | '*.js', 19 | 'lib/**' 20 | ], 21 | rules: js.configs.recommended.rules, 22 | languageOptions: { 23 | globals: { 24 | ...globals.node 25 | } 26 | } 27 | }, { 28 | name: 'tests', 29 | files: [ 30 | '__tests__/**' 31 | ], 32 | ...jest.configs['flat/recommended'], 33 | rules: { 34 | ...jest.configs['flat/recommended'].rules, 35 | 'jest/no-conditional-expect': 'off', 36 | } 37 | }]; -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2023-2025, LocalNerve 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@localnerve/batcher", 3 | "version": "1.0.0", 4 | "description": "Batching Classes to collect info until an event, reconcile, and callback", 5 | "main": "index.js", 6 | "exports": { 7 | ".": { 8 | "default": "./index.js" 9 | } 10 | }, 11 | "type": "module", 12 | "scripts": { 13 | "lint": "eslint .", 14 | "test": "NODE_OPTIONS=\"--experimental-vm-modules\" jest .", 15 | "test:debug": "NODE_OPTIONS=\"--experimental-vm-modules --inspect-brk\" jest . --runInBand" 16 | }, 17 | "repository": { 18 | "type": "git", 19 | "url": "git+https://github.com/localnerve/batcher.git" 20 | }, 21 | "keywords": [ 22 | "Batcher", 23 | "Batch", 24 | "Mutation", 25 | "Reconcilation" 26 | ], 27 | "author": "Alex Grant (https://www.localnerve.com)", 28 | "license": "MIT", 29 | "bugs": { 30 | "url": "https://github.com/localnerve/batcher/issues" 31 | }, 32 | "homepage": "https://github.com/localnerve/batcher#readme", 33 | "devDependencies": { 34 | "@eslint/js": "^9.26.0", 35 | "@jest/globals": "^29.7.0", 36 | "eslint": "^9.26.0", 37 | "eslint-plugin-jest": "^28.11.0", 38 | "globals": "^16.1.0", 39 | "jest": "^29.7.0" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /lib/timebatch.js: -------------------------------------------------------------------------------- 1 | /** 2 | * TimeBatch 3 | * 4 | * Collect info, after an idle timeout reconcile the info and execute a callback. 5 | * *Useful for reconciling batched data mutations (puts, deletes)* 6 | * 7 | * Collects information until an idle timer expires, 8 | * (idle timer gets reset on each add). On expiration, executes a callback for 9 | * each distinct item in the batch after reconciliation. 10 | * 11 | * Distinct operational items are identified by type and group. 12 | * type:group 13 | * Each type and group can have a collection of names (strings) 14 | * representing anything. 15 | * 16 | * Reconciliation: 17 | * At the end of the batching window as determined by the idle timer, 18 | * for each operation and type:group, the supplied callback is invoked 19 | * with all its relevant collections in one array. 20 | * 21 | * Copyright (c) 2025 Alex Grant (@localnerve), LocalNerve LLC 22 | * Private use for LocalNerve, LLC only. Unlicensed for any other use. 23 | */ 24 | 25 | export class TimeBatch { 26 | #_batchingTime = 0; 27 | #_timerCallback = null; 28 | #_ops = null; 29 | #_batchQueue = []; 30 | #_batchTimer = 0; 31 | 32 | async #reconcile () { 33 | if (this.#_batchQueue.length === 0) { 34 | clearTimeout(this.#_batchTimer); 35 | return; 36 | } 37 | 38 | const output = Array.from(this.#_ops).reduce((acc, curr) => { 39 | acc[curr] = []; 40 | return acc; 41 | }, {}); 42 | 43 | // Sort the batchQueue with the latest ops (skipping any duplicates) at the end. 44 | // Latest always wins, regardless of the op. 45 | this.#_batchQueue.sort((a, b) => { 46 | const atext = `${a.type}${a.group}${a.collection}`; 47 | const btext = `${b.type}${b.group}${b.collection}`; 48 | let result = atext.localeCompare(btext); 49 | if (result === 0) { 50 | result = a.timestamp - b.timestamp; 51 | } 52 | return result; 53 | }); 54 | 55 | // Get the latest unique item from the sorted batchQueue, store by op in output. 56 | // If there's already a matching type+group for this op, add the collection to its collections. 57 | let lastKey; 58 | for (let i = this.#_batchQueue.length - 1; i >= 0; i--) { 59 | const item = this.#_batchQueue[i]; 60 | const key = `${item.type}${item.group}${item.collection}`; 61 | 62 | if (key !== lastKey) { // a new unique op entry 63 | const duplicate = output[item.op].find(i => ( 64 | i.type === item.type && i.group === item.group 65 | )); 66 | if (duplicate) { 67 | duplicate.collections.push(item.collection); 68 | } else { 69 | output[item.op].push({ 70 | type: item.type, 71 | group: item.group, 72 | collections: [item.collection], 73 | op: item.op 74 | }); 75 | } 76 | } 77 | 78 | lastKey = key; 79 | } 80 | 81 | // The output arrays now have the latest, unique mutations 82 | // Each op entry has a unique type+group, with an array of collections. 83 | // There are no type+group+collection repeats in both op arrays. 84 | // (assert that truth here in DEVELOPMENT) 85 | 86 | // issue the callback for each item in each op queue 87 | for (const op of Object.keys(output)) { 88 | for (const item of output[op]) { 89 | await this.#_timerCallback(item); 90 | } 91 | } 92 | 93 | clearTimeout(this.#_batchTimer); 94 | this.#_batchQueue.length = 0; 95 | } 96 | 97 | constructor (batchingTime, ops, timerCallback) { 98 | this.#_ops = new Set(ops); 99 | this.#_batchingTime = batchingTime; 100 | this.#_timerCallback = timerCallback; 101 | } 102 | 103 | add ({ op, type, group, collection }) { 104 | clearTimeout(this.#_batchTimer); // restart the count down 105 | 106 | this.#_batchQueue.push({ 107 | op, 108 | type, 109 | group, 110 | collection, 111 | timestamp: Date.now() 112 | }); 113 | 114 | this.#_batchTimer = setTimeout(this.#reconcile.bind(this), this.#_batchingTime); 115 | } 116 | 117 | resetTimer () { 118 | clearTimeout(this.#_batchTimer); 119 | this.#_batchTimer = setTimeout(this.#reconcile.bind(this), this.#_batchingTime); 120 | } 121 | 122 | stop () { 123 | clearTimeout(this.#_batchTimer); 124 | this.#_batchQueue.length = 0; 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /__tests__/timebatch.test.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Test the batcher classes. 3 | * 4 | * Copyright (c) 2025 Alex Grant (@localnerve), LocalNerve LLC 5 | * Private use for LocalNerve, LLC only. Unlicensed for any other use. 6 | */ 7 | import { jest } from '@jest/globals'; 8 | import { TimeBatch } from '../lib/timebatch.js'; 9 | 10 | describe('TestBatch', () => { 11 | let timeBatch; 12 | 13 | test('one type+group+coll should get latest and called once', async () => { 14 | const callback = jest.fn(({ type, group, op, collections }) => { 15 | expect(type).toEqual('type1'); 16 | expect(group).toEqual('group1'); 17 | expect(op).toEqual('delete'); 18 | expect(collections).toContain('coll1'); 19 | }); 20 | 21 | timeBatch = new TimeBatch(100, ['put', 'delete'], callback); 22 | 23 | timeBatch.add({ op: 'put', type: 'type1', group: 'group1', collection: 'coll1' }); 24 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll1' }); 25 | 26 | return new Promise((resolve, reject) => setTimeout(() => { 27 | try { 28 | expect(callback).toHaveBeenCalledTimes(1); 29 | resolve(); 30 | } catch (e) { 31 | reject(e); 32 | } 33 | }, 200)); 34 | }); 35 | 36 | test('type+group+col should get latest, called once for each op per collection', async () => { 37 | const callback = jest.fn(({ type, group, op, collections }) => { 38 | expect(type).toEqual('type1'); 39 | expect(group).toEqual('group1'); 40 | expect(collections).toHaveLength(1); 41 | if (op === 'put') 42 | expect(collections).toContain('coll1'); 43 | if (op === 'delete') 44 | expect(collections).toContain('coll2'); 45 | }); 46 | 47 | timeBatch = new TimeBatch(100, ['put', 'delete'], callback); 48 | 49 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll1' }); 50 | timeBatch.add({ op: 'put', type: 'type1', group: 'group1', collection: 'coll1' }); 51 | timeBatch.add({ op: 'put', type: 'type1', group: 'group1', collection: 'coll2' }); 52 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll2' }); 53 | 54 | return new Promise((resolve, reject) => setTimeout(() => { 55 | try { 56 | expect(callback).toHaveBeenCalledTimes(2); 57 | resolve(); 58 | } catch (e) { 59 | reject(e); 60 | } 61 | }, 200)); 62 | }); 63 | 64 | test('type+group+col should get latest, should collect collections for each op', async () => { 65 | const callback = jest.fn(({ type, group, op, collections }) => { 66 | expect(type).toEqual('type1'); 67 | expect(group).toEqual('group1'); 68 | expect(collections).toHaveLength(2); 69 | if (op === 'put') 70 | expect(collections).toContain('coll0', 'coll2'); 71 | if (op === 'delete') 72 | expect(collections).toContain('coll1', 'coll3'); 73 | }); 74 | 75 | timeBatch = new TimeBatch(100, ['put', 'delete'], callback); 76 | 77 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll0' }); 78 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll1' }); 79 | timeBatch.add({ op: 'put', type: 'type1', group: 'group1', collection: 'coll0' }); 80 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll2' }); 81 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll3' }); 82 | timeBatch.add({ op: 'put', type: 'type1', group: 'group1', collection: 'coll2' }); 83 | 84 | return new Promise((resolve, reject) => setTimeout(() => { 85 | try { 86 | expect(callback).toHaveBeenCalledTimes(2); 87 | resolve(); 88 | } catch (e) { 89 | reject(e); 90 | } 91 | }, 200)); 92 | }); 93 | 94 | test('type+group+col should skip duplicates', async () => { 95 | const callback = jest.fn(({ type, group, op, collections }) => { 96 | expect(type).toEqual('type1'); 97 | expect(group).toEqual('group1'); 98 | expect(collections).toHaveLength(1); 99 | expect(op).toEqual('delete'); 100 | expect(collections).toEqual(['coll0']); 101 | }); 102 | 103 | timeBatch = new TimeBatch(100, ['put', 'delete'], callback); 104 | 105 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll0' }); 106 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll0' }); 107 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll0' }); 108 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll0' }); 109 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll0' }); 110 | timeBatch.add({ op: 'delete', type: 'type1', group: 'group1', collection: 'coll0' }); 111 | 112 | return new Promise((resolve, reject) => setTimeout(() => { 113 | try { 114 | expect(callback).toHaveBeenCalledTimes(1); 115 | resolve(); 116 | } catch (e) { 117 | reject(e); 118 | } 119 | }, 200)); 120 | }); 121 | }); --------------------------------------------------------------------------------