├── .travis.yml ├── package.json ├── LICENSE ├── index.js ├── test.js └── README.md /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "async-iterators", 3 | "version": "0.2.2", 4 | "description": "utility functions for async iterators", 5 | "homepage": "https://github.com/mirkokiefer/async-iterators", 6 | "main": "index.js", 7 | "scripts": { 8 | "test": "node_modules/.bin/mocha -R spec -t 60000" 9 | }, 10 | "author": "Mirko Kiefer ", 11 | "license": "MIT", 12 | "devDependencies": { 13 | "mocha": "~1.11.0" 14 | }, 15 | "engines" : { "node" : ">=0.10" } 16 | } 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Mirko Kiefer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 2 | var EventEmitter = require('events').EventEmitter 3 | 4 | var iterators = {} 5 | 6 | iterators.forEach = function(iterator, fn, cb) { 7 | iterator.next(function(err, res) { 8 | if (res === undefined) return cb(err, undefined) 9 | fn(err, res) 10 | iterators.forEach(iterator, fn, cb) 11 | }) 12 | } 13 | 14 | iterators.forEachAsync = function(iterator, fn, cb) { 15 | iterator.next(function(err, res) { 16 | if (res === undefined) return cb(err, undefined) 17 | fn(err, res, function() { 18 | iterators.forEachAsync(iterator, fn, cb) 19 | }) 20 | }) 21 | } 22 | 23 | iterators.map = function(iterator, fn) { 24 | return { 25 | next: function(cb) { 26 | iterator.next(function(err, res) { 27 | if ((res === undefined) || err) return cb(err, undefined) 28 | var mappedRes = fn(err, res) 29 | cb(err, mappedRes) 30 | }) 31 | } 32 | } 33 | } 34 | 35 | iterators.mapAsync = function(iterator, fn) { 36 | return { 37 | next: function(cb) { 38 | iterator.next(function(err, res) { 39 | if ((res === undefined) || err) return cb(err, undefined) 40 | fn(err, res, function(err, mappedRes) { 41 | cb(err, mappedRes) 42 | }) 43 | }) 44 | } 45 | } 46 | } 47 | 48 | iterators.filter = function(iterator, fn) { 49 | var next = function(cb) { 50 | iterator.next(function(err, res) { 51 | if (res === undefined) return cb(err, undefined) 52 | if (fn(err, res)) { 53 | cb(null, res) 54 | } else { 55 | next(cb) 56 | } 57 | }) 58 | } 59 | return { 60 | next: next 61 | } 62 | } 63 | 64 | iterators.filterAsync = function(iterator, fn) { 65 | var next = function(cb) { 66 | iterator.next(function(err, res) { 67 | if (res === undefined) return cb(err, undefined) 68 | fn(err, res, function(err, passedFilter) { 69 | if (passedFilter) { 70 | cb(null, res) 71 | } else { 72 | next(cb) 73 | } 74 | }) 75 | }) 76 | } 77 | return { 78 | next: next 79 | } 80 | } 81 | 82 | iterators.buffer = function(iterator, size) { 83 | var buffer = [] 84 | var bufferingInProgress = false 85 | var hasEnded = false 86 | var bufferEvents = new EventEmitter() 87 | 88 | var readBuffer = function(cb) { 89 | if (buffer.length) { 90 | cb(null, buffer.shift()) 91 | } else { 92 | bufferEvents.once('data', function() { 93 | readBuffer(cb) 94 | }) 95 | } 96 | if (!bufferingInProgress && !hasEnded && buffer.length < size) { 97 | fillBuffer() 98 | } 99 | } 100 | 101 | var fillBuffer = function(cb) { 102 | bufferingInProgress = true 103 | if ((buffer.length >= size) || hasEnded) { 104 | bufferingInProgress = false 105 | return 106 | } 107 | iterator.next(function(err, res) { 108 | if (res === undefined) hasEnded = true 109 | buffer.push(res) 110 | bufferEvents.emit('data') 111 | fillBuffer(cb) 112 | }) 113 | } 114 | 115 | var publicObj = { 116 | bufferFillRatio: function() { return buffer.length / size }, 117 | next: function(cb) { 118 | readBuffer(function(err, res) { 119 | cb(err, res) 120 | }) 121 | } 122 | } 123 | return publicObj 124 | } 125 | 126 | iterators.fromArray = function(array, cb) { 127 | var i = 0 128 | return { 129 | next: function(cb) { 130 | if (i == array.length) return cb(null, undefined) 131 | var value = array[i] 132 | i++ 133 | cb(null, value) 134 | } 135 | } 136 | } 137 | 138 | iterators.fromReadableStream = function(readable) { 139 | var isReadable = true 140 | var hasEnded = false 141 | 142 | readable.on('readable', function() { 143 | isReadable = true 144 | }) 145 | readable.on('end', function() { 146 | hasEnded = true 147 | }) 148 | 149 | var next = function(cb) { 150 | if (hasEnded) { 151 | return cb(null, undefined) 152 | } 153 | if (isReadable) { 154 | var res = readable.read() 155 | if (res === null) { 156 | isReadable = false 157 | return next(cb) 158 | } 159 | cb(null, res) 160 | } else { 161 | var onEnd = function() { next(cb) } 162 | readable.once('readable', function() { 163 | readable.removeListener('end', onEnd) 164 | next(cb) 165 | }) 166 | readable.once('end', onEnd) 167 | } 168 | } 169 | 170 | return {next: next} 171 | } 172 | 173 | iterators.toArray = function(iterator, cb) { 174 | var array = [] 175 | iterators.forEach(iterator, function(err, each) { 176 | array.push(each) 177 | }, function() { 178 | cb(null, array) 179 | }) 180 | } 181 | 182 | iterators.range = function(iterator, opts) { 183 | var from = opts.from 184 | var to = opts.to 185 | var pos = -1 186 | var next = function(cb) { 187 | iterator.next(function(err, value) { 188 | pos++ 189 | if (pos < from) return next(cb) 190 | if (pos > to) return cb(null, undefined) 191 | cb(err, value) 192 | }) 193 | } 194 | return {next: next} 195 | } 196 | 197 | iterators.toWritableStream = function(iterator, writeStream, encoding, cb) { 198 | write(cb); 199 | function write(cb) { 200 | iterator.next(function(err, res) { 201 | if (res === undefined) return writeStream.write('', encoding, cb) 202 | if (writeStream.write(res, encoding)) { 203 | write(cb) 204 | } else { 205 | writeStream.once('drain', function() { write(cb) }) 206 | } 207 | }) 208 | } 209 | } 210 | 211 | module.exports = iterators 212 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | 2 | var assert = require('assert') 3 | var iterators = require('./index') 4 | var fs = require('fs') 5 | var stream = require('stream') 6 | 7 | var numbers = [] 8 | for (var i = 1; i < 100; i++) { 9 | numbers.push(i) 10 | } 11 | var doubleFn = function(each) { return each * 2 } 12 | var numbersDoubled = numbers.map(doubleFn) 13 | var evenFn = function(each) { return (each % 2) == 0 } 14 | var evenNumbers = numbers.filter(evenFn) 15 | 16 | var createMockAsyncIterator = function() { 17 | var index = -1 18 | var next = function(cb) { 19 | setTimeout(function() { 20 | index++ 21 | if (!numbers[index]) return cb(null, undefined) 22 | cb(null, numbers[index]) 23 | }, 1) 24 | } 25 | return {next: next} 26 | } 27 | 28 | var runForEachIteratorTest = function(iterator, cb) { 29 | var index = 0 30 | iterators.forEach(iterator, function(err, each) { 31 | assert.equal(each, numbers[index]) 32 | index++ 33 | }, function() { 34 | assert.equal(index, numbers.length) 35 | cb() 36 | }) 37 | } 38 | 39 | describe('async-iterators', function() { 40 | it('should run forEach on an iterator', function(done) { 41 | var iterator = createMockAsyncIterator() 42 | runForEachIteratorTest(iterator, done) 43 | }) 44 | it('should run forEachAsync on an iterator', function(done) { 45 | var iterator = createMockAsyncIterator() 46 | var index = 0 47 | iterators.forEachAsync(iterator, function(err, each, cb) { 48 | assert.equal(each, numbers[index]) 49 | index++ 50 | cb() 51 | }, function() { 52 | assert.equal(index, numbers.length) 53 | done() 54 | }) 55 | }) 56 | it('should pipe an iterator to an array', function(done) { 57 | var iterator = createMockAsyncIterator() 58 | iterators.toArray(iterator, function(err, res) { 59 | assert.deepEqual(res, numbers) 60 | done() 61 | }) 62 | }) 63 | it('should create a map iterator and pipe to array', function(done) { 64 | var iterator = createMockAsyncIterator() 65 | var doublingIterator = iterators.map(iterator, function(err, each) { 66 | return doubleFn(each) 67 | }) 68 | iterators.toArray(doublingIterator, function(err, res) { 69 | assert.deepEqual(res, numbersDoubled) 70 | done() 71 | }) 72 | }) 73 | it('should create an asyncMap iterator', function(done) { 74 | var iterator = createMockAsyncIterator() 75 | var doublingIterator = iterators.mapAsync(iterator, function(err, each, cb) { 76 | cb(null, doubleFn(each)) 77 | }) 78 | iterators.toArray(doublingIterator, function(err, res) { 79 | assert.deepEqual(res, numbersDoubled) 80 | done() 81 | }) 82 | }) 83 | it('should create a filter iterator', function(done) { 84 | var iterator = createMockAsyncIterator() 85 | var filterIterator = iterators.filter(iterator, function(err, each) { 86 | return evenFn(each) 87 | }) 88 | iterators.toArray(filterIterator, function(err, res) { 89 | assert.deepEqual(res, evenNumbers) 90 | done() 91 | }) 92 | }) 93 | it('should create an async filter iterator', function(done) { 94 | var iterator = createMockAsyncIterator() 95 | var filterIterator = iterators.filterAsync(iterator, function(err, each, cb) { 96 | cb(null, evenFn(each)) 97 | }) 98 | iterators.toArray(filterIterator, function(err, res) { 99 | assert.deepEqual(res, evenNumbers) 100 | done() 101 | }) 102 | }) 103 | it('should create a buffering iterator', function(done) { 104 | var iterator = createMockAsyncIterator() 105 | var bufferIterator = iterators.buffer(iterator, 10) 106 | var bufferFillRatio = 0 107 | var slowMapIterator = iterators.mapAsync(bufferIterator, function(err, res, cb) { 108 | setTimeout(function() { 109 | bufferFillRatio += bufferIterator.bufferFillRatio() / numbers.length 110 | cb(null, res) 111 | }, 2) 112 | }) 113 | iterators.toArray(slowMapIterator, function(err, res) { 114 | assert.deepEqual(res, numbers) 115 | console.log(bufferFillRatio) 116 | assert.ok(bufferFillRatio > 0.5) 117 | done() 118 | }) 119 | }) 120 | it('should create an array iterator', function(done) { 121 | var arrayIterator = iterators.fromArray(numbers) 122 | iterators.toArray(arrayIterator, function(err, res) { 123 | assert.deepEqual(res, numbers) 124 | done() 125 | }) 126 | }) 127 | it('should create a readable stream iterator', function(done) { 128 | function MockStream(opt) { 129 | stream.Readable.call(this, opt) 130 | this.index = 0 131 | } 132 | require('util').inherits(MockStream, stream.Readable) 133 | 134 | MockStream.prototype._read = function() { 135 | mockStream.push(numbers[this.index]) 136 | this.index++ 137 | if (this.index == numbers.length) mockStream.push(null) 138 | } 139 | 140 | var mockStream = new MockStream(({objectMode: true, highWaterMark: 2})) 141 | 142 | var streamIterator = iterators.fromReadableStream(mockStream) 143 | iterators.toArray(streamIterator, function(err, res) { 144 | assert.deepEqual(res, numbers) 145 | done() 146 | }) 147 | }) 148 | it('should create a range iterator', function(done) { 149 | var iterator = createMockAsyncIterator() 150 | var rangeIterator = iterators.range(iterator, {from: 10, to: 19}) 151 | iterators.toArray(rangeIterator, function(err, res) { 152 | assert.deepEqual(res, numbers.slice(10, 20)) 153 | done() 154 | }) 155 | }) 156 | it('should create a range iterator with no end', function(done) { 157 | var iterator = createMockAsyncIterator() 158 | var rangeIterator = iterators.range(iterator, {from: 90}) 159 | iterators.toArray(rangeIterator, function(err, res) { 160 | assert.deepEqual(res, numbers.slice(90)) 161 | done() 162 | }) 163 | }) 164 | it('should create a range iterator with no start', function(done) { 165 | var iterator = createMockAsyncIterator() 166 | var rangeIterator = iterators.range(iterator, {to: 19}) 167 | iterators.toArray(rangeIterator, function(err, res) { 168 | assert.deepEqual(res, numbers.slice(0, 20)) 169 | done() 170 | }) 171 | }) 172 | it('should write an iterator to a writable stream', function(done) { 173 | var path = __dirname + '/output.txt' 174 | var writeStream = fs.createWriteStream(path) 175 | var iterator = createMockAsyncIterator() 176 | var stringIterator = iterators.map(iterator, function(err, res) { 177 | return res.toString() 178 | }) 179 | iterators.toWritableStream(stringIterator, writeStream, 'utf8', function() { 180 | var output = fs.readFileSync(path, {encoding: 'utf8'}) 181 | fs.unlinkSync(path) 182 | assert.deepEqual(output, numbers.join('')) 183 | done() 184 | }) 185 | }) 186 | }) 187 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # async-iterators 2 | 3 | [![NPM](https://nodei.co/npm/async-iterators.png)](https://nodei.co/npm/async-iterators/) 4 | 5 | Useful abstractions and utility functions for async iterators in Node.js. 6 | 7 | An async iterator is an object with a `next(cb)` method. 8 | Invoking the method should return the next item of an underlying data source. 9 | The callback should be a function of type `function(err, value)`. 10 | If the iterator has no more data to read, it will call the callback with `value == undefined`. 11 | 12 | Async iterators can easily be created from Node.js [Readable Streams](http://nodejs.org/api/stream.html#stream_class_stream_readable) by using [stream-iterator](https://github.com/mirkokiefer/stream-iterator). 13 | 14 | An example with a pointless iterator that asynchronously returns the numbers from 1 to 100: 15 | 16 | ``` js 17 | var iterators = require('async-iterators') 18 | 19 | function createExampleIterator = function() { 20 | var i = 0 21 | return { 22 | next: function(cb) { 23 | i++ 24 | if (i == 100) return cb(null, undefined) 25 | cb(null, i) 26 | } 27 | } 28 | } 29 | 30 | var myIterator = createExampleIterator() 31 | 32 | // wrap myIterator with a map iterator that doubles all results 33 | var doublingIterator = iterators.map(iterator, function(err, each) { 34 | return each * 2 35 | }) 36 | 37 | // pipe the iterator to an array 38 | iterators.toArray(doublingIterator, function(err, res) { 39 | console.log(res) 40 | }) 41 | ``` 42 | 43 | ## Documentation 44 | ### Iterator Sources 45 | - [fromArray](#fromArray) 46 | - [fromReadableStream](#fromReadableStream) 47 | 48 | ### Transforming Iterators 49 | - [map](#map) / [mapAsync](#mapAsync) 50 | - [filter](#filter) / [filterAsync](#filterAsync) 51 | - [range](#range) 52 | - [buffer](#buffer) 53 | 54 | ### Iterator Targets 55 | - [toArray](#toArray) 56 | - [toWritableStream](#toWritableStream) 57 | 58 | ### Utilities 59 | - [forEach](#forEach) 60 | 61 | ## Iterator Sources 62 | 63 | 64 | 65 | ### fromArray(array) 66 | 67 | Creates an iterator from an array. 68 | 69 | ``` js 70 | var arrayIterator = iterators.fromArray(numbers) 71 | ``` 72 | 73 | 74 | 75 | ### fromReadableStream(readableStream) 76 | 77 | Creates an iterator from a [Readable Stream](http://nodejs.org/api/stream.html#stream_class_stream_readable). 78 | 79 | ``` js 80 | var readStream = fs.createReadStream('input.txt', {encoding: 'utf8'}) 81 | var streamIterator = iterators.fromReadableStream(readStream) 82 | ``` 83 | 84 | ## Transforming Iterators 85 | 86 | 87 | 88 | ### map(iterator, mapFn) 89 | 90 | Create an iterator that applies a map function to transform each value of the source iterator. 91 | 92 | ``` js 93 | var mapIterator = iterators.map(someNumberIterator, function(err, each) { 94 | return each * 2 95 | }) 96 | 97 | // pipe the iterator to an array: 98 | iterators.toArray(mapIterator, function(err, res) { 99 | console.log(res) 100 | }) 101 | ``` 102 | 103 | 104 | 105 | ### mapAsync(iterator, mapFn) 106 | 107 | ``` js 108 | var mapIterator = iterators.map(someNumberIterator, function(err, each, cb) { 109 | cb(null, each * 2) 110 | }) 111 | ``` 112 | 113 | Create an iterator that filters the values of the source iterator using a filter function. 114 | 115 | 116 | 117 | ### filter(iterator, filterFn) 118 | 119 | ``` js 120 | var evenNumbersIterator = iterators.filter(someNumberIterator, function(err, each) { 121 | return (each % 2) == 0 122 | }) 123 | ``` 124 | 125 | 126 | 127 | ### filterAsync(iterator, filterFn) 128 | 129 | ``` js 130 | var evenNumbersIterator = iterators.filter(someNumberIterator, function(err, each, cb) { 131 | cb(null, (each % 2) == 0) 132 | }) 133 | ``` 134 | 135 | 136 | 137 | ### range(iterator, range) 138 | 139 | Creates an iterator that only iteratores over the specified range. 140 | 141 | `range` is specified as `{from: startIndex, to: endIndex}` where `from` and `to` are both inclusive. 142 | 143 | ``` js 144 | var rangeIterator = iterators.range(iterator, {from: 10, to: 19}) 145 | ``` 146 | 147 | 148 | 149 | ### buffer(iterator, bufferSize) 150 | 151 | Creates an iterator with an internal buffer that is always filled until `bufferSize`. 152 | The buffer can abviously only grow if the buffer iterator is read slower than the underlying iterator source can return data. 153 | 154 | The current buffer fill ratio can be inspected at any time using `bufferFillRatio()` which returns a number between 0..1. 155 | 156 | The buffer size can be changed using `setBufferSize(bufferSize)`. 157 | 158 | ``` js 159 | var bufferedIterator = iterators.buffer(someIterator, 10) 160 | 161 | // inspect buffer size 162 | console.log(bufferedIterator.bufferFillRatio()) 163 | 164 | // change the buffer size later 165 | bufferedIterator.setBufferSize(100) 166 | ``` 167 | 168 | ## Iterator Targets 169 | 170 | 171 | 172 | ### toArray(iterator, cb) 173 | 174 | Reads the source iterator and writes the results to an array. 175 | 176 | ``` js 177 | iterators.toArray(someIterator, function(err, array) { 178 | console.log(array) 179 | }) 180 | ``` 181 | 182 | 183 | 184 | ### toWritableStream(iterator, writeStream, encoding, cb) 185 | 186 | Reads the source iterator and writes the result to a [Writable Stream](http://nodejs.org/api/stream.html#stream_class_stream_writable). 187 | 188 | ``` js 189 | var writeStream = fs.createWriteStream('output.txt') 190 | iterators.toWritableStream(iterator, writeStream, 'utf8', function() { 191 | console.log('done') 192 | }) 193 | ``` 194 | 195 | ## Utilities 196 | 197 | ### forEach(iterator, fn, cb) 198 | 199 | Reads the source iterator and invokes `fn` for each value of the iterator. 200 | 201 | ``` js 202 | iterators.forEach(someIterator, function(err, data) { 203 | console.log(data) 204 | }, function() { 205 | console.log('end') 206 | }) 207 | ``` 208 | 209 | 210 | 211 | ### forEachAsync(iterator, fn, cb) 212 | 213 | Reads the source iterator and invokes `fn` for each value of the iterator. 214 | Only once the callback is invoked the next value is read from the source iterator. 215 | 216 | ``` js 217 | iterators.forEachAsync(someIterator, function(err, data, cb) { 218 | console.log(data) 219 | setTimeout(cb, 100) 220 | }, function() { 221 | console.log('end') 222 | }) 223 | ``` 224 | 225 | ## Other libraries 226 | 227 | Some libraries using the async iterator pattern: 228 | 229 | - [stream-iterator](https://github.com/mirkokiefer/stream-iterator) - wrap any stream into an async iterator 230 | - [node-leveldown](https://github.com/rvagg/node-leveldown#iteratornextcallback) - allows you to iterate over entries in LevelDB 231 | 232 | ## Contributors 233 | This project was created by Mirko Kiefer ([@mirkokiefer](https://github.com/mirkokiefer)). 234 | --------------------------------------------------------------------------------