├── .eslintrc ├── .gitignore ├── logos ├── logo-box-builtby.png └── logo-box-madefor.png ├── .github └── workflows │ └── test.yml ├── package.json ├── index.js ├── test └── tests.js └── README.md /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "apostrophe" 3 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | package-lock.json 2 | npm-debug.log 3 | *.DS_Store 4 | node_modules 5 | -------------------------------------------------------------------------------- /logos/logo-box-builtby.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apostrophecms/broadband/master/logos/logo-box-builtby.png -------------------------------------------------------------------------------- /logos/logo-box-madefor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apostrophecms/broadband/master/logos/logo-box-madefor.png -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | pull_request: 7 | branches: ["*"] 8 | 9 | workflow_dispatch: 10 | 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | node-version: [18, 20] 17 | mongodb-version: [6.0, 7.0] 18 | 19 | steps: 20 | - name: Git checkout 21 | uses: actions/checkout@v4 22 | 23 | - name: Use Node.js ${{ matrix.node-version }} 24 | uses: actions/setup-node@v4 25 | with: 26 | node-version: ${{ matrix.node-version }} 27 | 28 | - name: Start MongoDB 29 | uses: supercharge/mongodb-github-action@1.11.0 30 | with: 31 | mongodb-version: ${{ matrix.mongodb-version }} 32 | 33 | - run: npm install 34 | 35 | - run: npm test 36 | env: 37 | CI: true 38 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "broadband", 3 | "version": "1.1.0", 4 | "description": "Given a MongoDB query cursor, process the results in parallel, up to the specified limit.", 5 | "main": "index.js", 6 | "scripts": { 7 | "lint": "npm run eslint", 8 | "eslint": "eslint .", 9 | "test": "npm run lint && npm run mocha", 10 | "mocha": "mocha" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "https://github.com/apostrophecms/broadband" 15 | }, 16 | "keywords": [ 17 | "async", 18 | "parallel", 19 | "concurrency", 20 | "limit", 21 | "series", 22 | "mongodb", 23 | "mongo" 24 | ], 25 | "author": "Apostrophe Technologies, Inc.", 26 | "license": "MIT", 27 | "bugs": { 28 | "url": "https://github.com/apostrophecms/broadband/issues" 29 | }, 30 | "homepage": "https://github.com/apostrophecms/broadband", 31 | "dependencies": { 32 | "lodash": "^4.0.0" 33 | }, 34 | "devDependencies": { 35 | "eslint-config-apostrophe": "^5.0.0", 36 | "mocha": "^5.0.0" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | 3 | module.exports = function(cursor, limit, each, callback) { 4 | const taskQueue = []; 5 | let error; 6 | let eof; 7 | let id = 0; 8 | const nextObjectQueue = []; 9 | let nextObjectActive = false; 10 | 11 | function fill() { 12 | // Because the end condition tests in our tasks are 13 | // not guaranteed to wait for nextTick, we must check for 14 | // the same end conditions here, otherwise broadband 15 | // may terminate twice. 16 | while ((taskQueue.length < limit) && (!(error || eof))) { 17 | const fn = makeTask(id++); 18 | taskQueue.push(fn); 19 | fn(); 20 | } 21 | 22 | function makeTask(id) { 23 | const fn = function() { 24 | if (error || eof) { 25 | return setImmediate(_.partial(finished, id)); 26 | } 27 | // cursor.nextObject must never be called concurrently, 28 | // so call our wrapper that serializes it 29 | return nextObject(function(err, doc) { 30 | error = error || err; 31 | if (error) { 32 | return finished(id); 33 | } 34 | if (!doc) { 35 | eof = true; 36 | return finished(id); 37 | } 38 | return each(doc, function(err) { 39 | error = error || err; 40 | return finished(id); 41 | }); 42 | }); 43 | }; 44 | fn.id = id; 45 | return fn; 46 | } 47 | 48 | function finished(id) { 49 | _.remove(taskQueue, function(fn) { 50 | return fn.id === id; 51 | }); 52 | if ((error || eof) && (!taskQueue.length)) { 53 | // End of a long road 54 | return callback(error); 55 | } 56 | if (!(error || eof)) { 57 | fill(); 58 | } 59 | } 60 | } 61 | 62 | fill(); 63 | 64 | // Serialize calls to nextObject 65 | function nextObject(callback) { 66 | if (callback) { 67 | nextObjectQueue.push(callback); 68 | } 69 | if (!nextObjectQueue.length) { 70 | return; 71 | } 72 | if (!nextObjectActive) { 73 | nextObjectActive = true; 74 | const fn = nextObjectQueue.shift(); 75 | if (error || eof) { 76 | return setImmediate(function() { 77 | fn(error, null); 78 | nextObjectActive = false; 79 | nextObject(); 80 | }); 81 | } 82 | return cursor[cursor.nextObject ? 'nextObject' : 'next'](function(err, doc) { 83 | fn(err, doc); 84 | nextObjectActive = false; 85 | nextObject(); 86 | }); 87 | } 88 | } 89 | }; 90 | -------------------------------------------------------------------------------- /test/tests.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const broadband = require('../index.js'); 3 | 4 | // Mock mongodb 5 | 6 | const data = []; 7 | let i; 8 | 9 | for (i = 0; (i < 100); i++) { 10 | data[i] = { _id: i }; 11 | } 12 | 13 | function Cursor() { 14 | const self = this; 15 | self.i = 0; 16 | self.nextObjectActive = false; 17 | self.nextObject = function(callback) { 18 | if (self.nextObjectActive) { 19 | throw 'nextObject called concurrently!'; 20 | } 21 | self.nextObjectActive = true; 22 | if (self.i === self.failOn) { 23 | self.nextObjectActive = false; 24 | return setImmediate(function() { 25 | return callback('simulated error'); 26 | }); 27 | } 28 | if (self.i >= data.length) { 29 | return setImmediate(function() { 30 | self.nextObjectActive = false; 31 | return callback(null); 32 | }); 33 | } 34 | const result = data[self.i++]; 35 | 36 | return setTimeout(function() { 37 | self.nextObjectActive = false; 38 | return callback(null, result); 39 | }, Math.random() * 5); 40 | }; 41 | } 42 | 43 | const collection = { 44 | find: function(criteria) { 45 | return new Cursor(); 46 | } 47 | }; 48 | 49 | describe('broadband', function() { 50 | let completions = 0; 51 | it('receives all results only once with random timing, never runs nextObject concurrently', function(done) { 52 | const seen = {}; 53 | this.timeout(7000); 54 | 55 | const cursor = collection.find({}); 56 | return broadband(cursor, 4, function(page, callback) { 57 | assert(page); 58 | assert(!seen[page._id]); 59 | seen[page._id] = true; 60 | return setTimeout(function() { 61 | return callback(null); 62 | }, Math.random() * 50); 63 | }, function(err) { 64 | completions++; 65 | assert(completions === 1); 66 | assert(!err); 67 | assert(Object.keys(seen).length === data.length); 68 | return done(); 69 | }); 70 | }); 71 | it('handles an error on the 80th result gracefully', function(done) { 72 | let completions = 0; 73 | const limit = 4; 74 | let received = 0; 75 | let completed = 0; 76 | this.timeout(7000); 77 | 78 | const cursor = collection.find({}); 79 | cursor.failOn = 80; 80 | let running = 0; 81 | let maxRunning = 0; 82 | return broadband(cursor, limit, function(page, callback) { 83 | received++; 84 | running++; 85 | if (running > maxRunning) { 86 | maxRunning = running; 87 | } 88 | return setTimeout(function() { 89 | running--; 90 | completed++; 91 | return callback(null); 92 | }, Math.random() * 50); 93 | }, function(err) { 94 | completions++; 95 | assert(completions === 1); 96 | assert(err); 97 | assert(received === cursor.failOn); 98 | assert((completed >= cursor.failOn) && (completed < cursor.failOn + limit)); 99 | assert(maxRunning === limit); 100 | return done(); 101 | }); 102 | }); 103 | }); 104 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # broadband 2 | ========= 3 | [![CircleCI](https://circleci.com/gh/apostrophecms/broadband/tree/master.svg?style=svg)](https://circleci.com/gh/apostrophecms/broadband/tree/master) 4 | 5 | 6 | 7 | Given a MongoDB query cursor, process the results in parallel, up to the specified limit. 8 | 9 | ```javascript 10 | var broadband = require('broadband'); 11 | var cursor = mongoCollection.find({}); 12 | 13 | return broadband(cursor, 8, function(doc, callback) { 14 | // Up to 8 of these will be invoked simultaneously 15 | // Do something with doc, then... 16 | return callback(null); 17 | }, function(err) { 18 | // All done 19 | }); 20 | ``` 21 | 22 | ## Why? 23 | 24 | We wanted to work with MongoDB queries the way we work with [async.eachLimit](https://github.com/caolan/async#eachLimit), but without yanking everything into memory at once with `toArray`. 25 | 26 | Specifically, we wanted to resize some images in parallel, rather than waiting to do them one at a time. We have a MongoDB collection with information about all of the images. But there are a lot of them, so we don't want to yank all of that information into memory up front. 27 | 28 | `broadband` wraps MongoDB's `Cursor.nextObject` with a queueing mechanism that allows several results to be processed at once, but only up to the limit you specify. You don't run out of memory due to too many image processes, you don't wait too long, and you don't have to load the entire array into memory at once. Everybody gets a medal. 29 | 30 | ## What about errors? 31 | 32 | If an error occurs, `broadband` will: 33 | 34 | 1. Stop starting new iterator callbacks. 35 | 2. Wait for any outstanding iterator callbacks to finish. 36 | 3. Invoke the final callback (its third argument) with the first error it received. 37 | 38 | ## Using broadband without mongodb 39 | 40 | You can pass any object with a `nextObject` method as the "cursor." That method should invoke its callback with `(err, object)`. If there is no error, `object` should be the next object retrieved from your data source. If there are no more objects, pass `null` as `object`. 41 | 42 | ## About P'unk Avenue and Apostrophe 43 | 44 | `broadband` was created at [P'unk Avenue](https://punkave.com) for use in many projects built with Apostrophe, an open-source content management system built on node.js. If you like `broadband` you should definitely [check out apostrophecms.com](https://apostrophecms.com). 45 | 46 | ## Support 47 | 48 | Feel free to open issues on [github](http://github.com/apostrophecms/broadband). 49 | 50 | 51 | 52 | ## Changelog 53 | 54 | ### 1.1.0 55 | 56 | - Adds support for both MongoDB 2 and 3 via the cursor `next` and `nextObject` methods. 57 | - Adds JS linting to the tests. 58 | 59 | ### 1.0.0 60 | 61 | - Declared 1.0.0 stable as this has long been a component of Apostrophe. Updated lodash dependency to satisfy `npm audit`. 62 | 63 | ### 0.1.1 64 | 65 | - Fixed a rare race condition which caused `broadband` to invoke its final callback more than once. 66 | 67 | ### 0.1.0 68 | 69 | - Initial release. With shiny unit tests, of course. 70 | --------------------------------------------------------------------------------