├── .travis.yml ├── .gitignore ├── Makefile ├── test ├── dropdb.js ├── common.js └── querystream.work.test.js ├── mongoose-querystream-worker.js ├── package.json ├── LICENSE └── README.md /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 0.10 4 | - 0.12 5 | - 4 6 | - node 7 | services: 8 | - mongodb 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | 10 | pids 11 | logs 12 | results 13 | node_modules 14 | 15 | npm-debug.log 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | TESTS = $(shell find test/ -name '*.test.js') 3 | 4 | test: 5 | @node test/dropdb.js 6 | @./node_modules/.bin/mocha $(T) --async-only $(TESTS) 7 | @node test/dropdb.js 8 | 9 | .PHONY: test -------------------------------------------------------------------------------- /test/dropdb.js: -------------------------------------------------------------------------------- 1 | // Lifted from https://raw.github.com/LearnBoost/mongoose/master/test/dropdb.js 2 | 3 | var start = require('./common'); 4 | var db = start(); 5 | db.once('open', function () { 6 | // drop the default test database 7 | db.db.dropDatabase(function () { 8 | db.close(); 9 | }); 10 | }); -------------------------------------------------------------------------------- /mongoose-querystream-worker.js: -------------------------------------------------------------------------------- 1 | var QueryStream = require('mongoose/lib/querystream'), 2 | streamWorker = require('stream-worker'), 3 | DEFAULT_CONCURRENCY_LIMIT = 10; 4 | 5 | QueryStream.prototype.concurrency = function (max) { 6 | this._concurrency = max; 7 | return this; 8 | }; 9 | 10 | QueryStream.prototype.work = function (worker, options, done) { 11 | options = options || { }; 12 | if (!options.concurrency) options.concurrency = this._concurrency || DEFAULT_CONCURRENCY_LIMIT; 13 | return streamWorker(this, worker, options, done); 14 | }; 15 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongoose-querystream-worker", 3 | "version": "3.0.1", 4 | "description": "Execute an async function per document in a streamed query, pausing the stream when a concurrency limit is saturated", 5 | "main": "mongoose-querystream-worker.js", 6 | "scripts": { 7 | "test": "make test" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/goodeggs/mongoose-querystream-worker.git" 12 | }, 13 | "keywords": [ 14 | "mongoose", 15 | "querystream", 16 | "worker", 17 | "queue" 18 | ], 19 | "author": "Good Eggs ", 20 | "license": "MIT", 21 | "readmeFilename": "README.md", 22 | "gitHead": "a39913da83703f67e608426e275aad9effff8fbc", 23 | "devDependencies": { 24 | "mocha": "^2.0.0", 25 | "mongoose": "^4.0.0" 26 | }, 27 | "dependencies": { 28 | "bluebird": "~3.3.0", 29 | "stream-worker": "^2.0.1" 30 | }, 31 | "publishConfig": { 32 | "registry": "https://registry.npmjs.org/" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Good Eggs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | mongoose-querystream-worker [![build status](https://secure.travis-ci.org/goodeggs/mongoose-querystream-worker.png)](http://travis-ci.org/goodeggs/mongoose-querystream-worker) 2 | =========================== 3 | 4 | DEPRECATED - In light of [Mongoose QueryStreams](http://mongoosejs.com/docs/api.html#query_Query-stream) being deprecated in favor of [Mongoose QueryCursors](http://mongoosejs.com/docs/api.html#query_Query-cursor), may as well just depend on [stream-worker](https://github.com/goodeggs/stream-worker) directly, rather than maintaining this lightweight syntatic layer. 5 | 6 | Execute an async function per document in a streamed query, pausing the stream when a concurrency limit is saturated. Think [async.queue](https://github.com/caolan/async#queue) but for [Mongoose QueryStreams](http://mongoosejs.com/docs/api.html#query_Query-stream). Built on top of [stream-worker](https://github.com/goodeggs/stream-worker). 7 | 8 | ```js 9 | require('mongoose-querystream-worker'); 10 | 11 | /* Promises: */ 12 | 13 | Model.find().stream().concurrency(n).work(function (doc) { 14 | /* ... work with the doc ... */ 15 | return doc.save(); /* returns a promise */ 16 | }, {promises: true}) 17 | .then(function() { 18 | /* ... all workers have finished ... */ 19 | }, function(err) { 20 | /* ... something went wrong ... */ 21 | }); 22 | 23 | /* Callbacks: */ 24 | 25 | Model.find().stream().concurrency(n).work( 26 | function (doc, done) { 27 | /* ... work with the doc ... */ 28 | }, 29 | function (err) { 30 | /* ... all workers have finished ... */ 31 | } 32 | ); 33 | ``` 34 | -------------------------------------------------------------------------------- /test/common.js: -------------------------------------------------------------------------------- 1 | // Lifted from https://raw.github.com/LearnBoost/mongoose/master/test/common.js 2 | 3 | /** 4 | * Module dependencies. 5 | */ 6 | 7 | var mongoose = require('mongoose') 8 | , Mongoose = mongoose.Mongoose 9 | , Collection = mongoose.Collection 10 | , assert = require('assert') 11 | , startTime = Date.now() 12 | , queryCount = 0 13 | , opened = 0 14 | , closed = 0; 15 | 16 | if (process.env.D === '1') 17 | mongoose.set('debug', true); 18 | 19 | /** 20 | * Override all Collection related queries to keep count 21 | */ 22 | 23 | [ 'ensureIndex' 24 | , 'findAndModify' 25 | , 'findOne' 26 | , 'find' 27 | , 'insert' 28 | , 'save' 29 | , 'update' 30 | , 'remove' 31 | , 'count' 32 | , 'distinct' 33 | , 'isCapped' 34 | , 'options' 35 | ].forEach(function (method) { 36 | 37 | var oldMethod = Collection.prototype[method]; 38 | 39 | Collection.prototype[method] = function () { 40 | queryCount++; 41 | return oldMethod.apply(this, arguments); 42 | }; 43 | 44 | }); 45 | 46 | /** 47 | * Override Collection#onOpen to keep track of connections 48 | */ 49 | 50 | var oldOnOpen = Collection.prototype.onOpen; 51 | 52 | Collection.prototype.onOpen = function(){ 53 | opened++; 54 | return oldOnOpen.apply(this, arguments); 55 | }; 56 | 57 | /** 58 | * Override Collection#onClose to keep track of disconnections 59 | */ 60 | 61 | var oldOnClose = Collection.prototype.onClose; 62 | 63 | Collection.prototype.onClose = function(){ 64 | closed++; 65 | return oldOnClose.apply(this, arguments); 66 | }; 67 | 68 | /** 69 | * Create a connection to the test database. 70 | * You can set the environmental variable MONGOOSE_TEST_URI to override this. 71 | * 72 | * @api private 73 | */ 74 | 75 | module.exports = function (options) { 76 | options || (options = {}); 77 | var uri; 78 | 79 | if (options.uri) { 80 | uri = options.uri; 81 | delete options.uri; 82 | } else { 83 | uri = module.exports.uri; 84 | } 85 | 86 | var noErrorListener = !! options.noErrorListener; 87 | delete options.noErrorListener; 88 | 89 | var conn = mongoose.createConnection(uri, options); 90 | 91 | if (noErrorListener) return conn; 92 | 93 | conn.on('error', function (err) { 94 | assert.ok(err); 95 | }); 96 | 97 | return conn; 98 | }; 99 | 100 | /*! 101 | * testing uri 102 | */ 103 | 104 | module.exports.uri = process.env.MONGOOSE_TEST_URI || 'mongodb://localhost/mongoose_test'; 105 | 106 | /** 107 | * expose mongoose 108 | */ 109 | 110 | module.exports.mongoose = mongoose; 111 | 112 | /** 113 | * expose mongod version helper 114 | */ 115 | 116 | module.exports.mongodVersion = function (cb) { 117 | var db = module.exports(); 118 | 119 | db.on('error', cb); 120 | 121 | db.on('open', function () { 122 | db.db.admin(function (err, admin) { 123 | if (err) return cb(err); 124 | admin.serverStatus(function (err, info) { 125 | if (err) return cb(err); 126 | var version = info.version.split('.').map(function(n){return parseInt(n, 10) }); 127 | cb(null, version); 128 | }); 129 | }); 130 | }) 131 | } -------------------------------------------------------------------------------- /test/querystream.work.test.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Test dependencies. 3 | */ 4 | require('..'); 5 | 6 | var start = require('./common') 7 | , assert = require('assert') 8 | , mongoose = start.mongoose 9 | , utils = require('mongoose/lib/utils') 10 | , random = utils.random 11 | , Schema = mongoose.Schema 12 | , Promise = require('bluebird') 13 | 14 | var names = ('Aaden Aaron Adrian Aditya Agustin Jim Bob Jonah Frank Sally Lucy').split(' '); 15 | 16 | /** 17 | * Setup. 18 | */ 19 | 20 | var Person = new Schema({ 21 | name: String 22 | }); 23 | 24 | mongoose.model('PersonForStream', Person); 25 | var collection = 'personforstream_' + random(); 26 | 27 | describe('query stream worker:', function(){ 28 | before(function (done) { 29 | var db = start() 30 | , P = db.model('PersonForStream', collection) 31 | 32 | var people = names.map(function (name) { 33 | return { name: name }; 34 | }); 35 | 36 | P.create(people, function (err) { 37 | assert.ifError(err); 38 | db.close(); 39 | done(); 40 | }); 41 | }); 42 | 43 | it('invokes a worker (promise-style) for each doc', function(done){ 44 | var db = start() 45 | , P = db.model('PersonForStream', collection) 46 | , i = 0 47 | 48 | return P.find().stream().work(function(doc) { 49 | i++ 50 | }, {promises : true}).then(function() { 51 | assert.equal(i, names.length); 52 | done(); 53 | }); 54 | }); 55 | 56 | it('invokes a worker (callback-style) for each doc', function(done){ 57 | var db = start() 58 | , P = db.model('PersonForStream', collection) 59 | , i = 0 60 | 61 | var stream = P.find().stream().work( 62 | function(doc, done) { 63 | i++; 64 | done(); 65 | },{promises : false}, 66 | function(err) { 67 | assert.equal(i, names.length); 68 | done(); 69 | } 70 | ); 71 | }); 72 | 73 | it('limits concurrency', function(testDone){ 74 | var db = start() 75 | , P = db.model('PersonForStream', collection) 76 | , workers = [] 77 | , docCount = 0 78 | , concurrencyLimit = 2 79 | , i = 0 80 | 81 | function worker (doc, done) { 82 | workers.push({done: done}); 83 | } 84 | 85 | function workFinished (err) { 86 | testDone(); 87 | } 88 | 89 | var stream = P.find().stream(); 90 | stream.concurrency(concurrencyLimit).work(worker, {}, testDone); 91 | 92 | function checkWorkers () { 93 | assert(workers.length <= concurrencyLimit, 'the concurrency limit is never exceeded'); 94 | 95 | var oldDocCount = ++docCount; 96 | if(docCount <= concurrencyLimit) { 97 | process.nextTick(function () { 98 | assert(docCount > oldDocCount, 'not yet saturated, expect another worker to be started'); 99 | }); 100 | } else if (docCount == concurrencyLimit) { 101 | process.nextTick(function () { 102 | assert(docCount == oldDocCount, 'now we\'re saturated, no more workers'); 103 | 104 | // good, now free up a worker 105 | workers.pop().done(); 106 | }); 107 | } else { 108 | // check passed, burn the rest of the stream 109 | while(workers.length) { 110 | workers.pop().done(); 111 | } 112 | } 113 | } 114 | 115 | stream.on('data', function() { 116 | // nextTick to make sure worker has started 117 | process.nextTick(checkWorkers); 118 | }); 119 | }); 120 | }); 121 | --------------------------------------------------------------------------------