├── .gitignore ├── LICENSE ├── README.md ├── SECURITY.md ├── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2013 Mathias Buus 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # parallel-transform 2 | 3 | [Transform stream](http://nodejs.org/api/stream.html#stream_class_stream_transform_1) for Node.js that allows you to run your transforms 4 | in parallel without changing the order of the output. 5 | 6 | npm install parallel-transform 7 | 8 | It is easy to use 9 | 10 | ``` js 11 | var transform = require('parallel-transform'); 12 | 13 | var stream = transform(10, function(data, callback) { // 10 is the parallism level 14 | setTimeout(function() { 15 | callback(null, data); 16 | }, Math.random() * 1000); 17 | }); 18 | 19 | for (var i = 0; i < 10; i++) { 20 | stream.write(''+i); 21 | } 22 | stream.end(); 23 | 24 | stream.on('data', function(data) { 25 | console.log(data); // prints 0,1,2,... 26 | }); 27 | stream.on('end', function() { 28 | console.log('stream has ended'); 29 | }); 30 | ``` 31 | 32 | If you run the above example you'll notice that it runs in parallel 33 | (does not take ~1 second between each print) and that the order is preserved 34 | 35 | ## Stream options 36 | 37 | All transforms are Node 0.10 streams. Per default they are created with the options `{objectMode:true}`. 38 | If you want to use your own stream options pass them as the second parameter 39 | 40 | ``` js 41 | var stream = transform(10, {objectMode:false}, function(data, callback) { 42 | // data is now a buffer 43 | callback(null, data); 44 | }); 45 | 46 | fs.createReadStream('filename').pipe(stream).pipe(process.stdout); 47 | ``` 48 | 49 | ### Unordered 50 | Passing the option `{ordered:false}` will output the data as soon as it's processed by a transform, without waiting to respect the order. 51 | 52 | ## License 53 | 54 | MIT -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | ## Security contact information 2 | 3 | To report a security vulnerability, please use the 4 | [Tidelift security contact](https://tidelift.com/security). 5 | Tidelift will coordinate the fix and disclosure. 6 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var Transform = require('readable-stream').Transform; 2 | var inherits = require('inherits'); 3 | var cyclist = require('cyclist'); 4 | var util = require('util'); 5 | 6 | var ParallelTransform = function(maxParallel, opts, ontransform) { 7 | if (!(this instanceof ParallelTransform)) return new ParallelTransform(maxParallel, opts, ontransform); 8 | 9 | if (typeof maxParallel === 'function') { 10 | ontransform = maxParallel; 11 | opts = null; 12 | maxParallel = 1; 13 | } 14 | if (typeof opts === 'function') { 15 | ontransform = opts; 16 | opts = null; 17 | } 18 | 19 | if (!opts) opts = {}; 20 | if (!opts.highWaterMark) opts.highWaterMark = Math.max(maxParallel, 16); 21 | if (opts.objectMode !== false) opts.objectMode = true; 22 | 23 | Transform.call(this, opts); 24 | 25 | this._maxParallel = maxParallel; 26 | this._ontransform = ontransform; 27 | this._destroyed = false; 28 | this._flushed = false; 29 | this._ordered = opts.ordered !== false; 30 | this._buffer = this._ordered ? cyclist(maxParallel) : []; 31 | this._top = 0; 32 | this._bottom = 0; 33 | this._ondrain = null; 34 | }; 35 | 36 | inherits(ParallelTransform, Transform); 37 | 38 | ParallelTransform.prototype.destroy = function() { 39 | if (this._destroyed) return; 40 | this._destroyed = true; 41 | this.emit('close'); 42 | }; 43 | 44 | ParallelTransform.prototype._transform = function(chunk, enc, callback) { 45 | var self = this; 46 | var pos = this._top++; 47 | 48 | this._ontransform(chunk, function(err, data) { 49 | if (self._destroyed) return; 50 | if (err) { 51 | self.emit('error', err); 52 | self.push(null); 53 | self.destroy(); 54 | return; 55 | } 56 | if (self._ordered) { 57 | self._buffer.put(pos, (data === undefined || data === null) ? null : data); 58 | } 59 | else { 60 | self._buffer.push(data); 61 | } 62 | self._drain(); 63 | }); 64 | 65 | if (this._top - this._bottom < this._maxParallel) return callback(); 66 | this._ondrain = callback; 67 | }; 68 | 69 | ParallelTransform.prototype._flush = function(callback) { 70 | this._flushed = true; 71 | this._ondrain = callback; 72 | this._drain(); 73 | }; 74 | 75 | ParallelTransform.prototype._drain = function() { 76 | if (this._ordered) { 77 | while (this._buffer.get(this._bottom) !== undefined) { 78 | var data = this._buffer.del(this._bottom++); 79 | if (data === null) continue; 80 | this.push(data); 81 | } 82 | } 83 | else { 84 | while (this._buffer.length > 0) { 85 | var data = this._buffer.pop(); 86 | this._bottom++; 87 | if (data === null) continue; 88 | this.push(data); 89 | } 90 | } 91 | 92 | 93 | if (!this._drained() || !this._ondrain) return; 94 | 95 | var ondrain = this._ondrain; 96 | this._ondrain = null; 97 | ondrain(); 98 | }; 99 | 100 | ParallelTransform.prototype._drained = function() { 101 | var diff = this._top - this._bottom; 102 | return this._flushed ? !diff : diff < this._maxParallel; 103 | }; 104 | 105 | module.exports = ParallelTransform; 106 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "parallel-transform", 3 | "version": "1.2.0", 4 | "repository": "git://github.com/mafintosh/parallel-transform", 5 | "license": "MIT", 6 | "description": "Transform stream that allows you to run your transforms in parallel without changing the order", 7 | "keywords": [ 8 | "transform", 9 | "stream", 10 | "parallel", 11 | "preserve", 12 | "order" 13 | ], 14 | "author": "Mathias Buus Madsen ", 15 | "dependencies": { 16 | "cyclist": "^1.0.1", 17 | "inherits": "^2.0.3", 18 | "readable-stream": "^2.1.5" 19 | } 20 | } 21 | --------------------------------------------------------------------------------