├── .gitignore ├── .jshintrc ├── .travis.yml ├── AUTHORS ├── LICENSE ├── README.md ├── lib └── redis-rstream.js ├── package-lock.json ├── package.json └── test ├── basic.mocha.js └── mocha.opts /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | 10 | pids 11 | logs 12 | results 13 | 14 | node_modules 15 | npm-debug.log -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "indent": 2, 3 | 4 | "predef": [ 5 | "Arguments", 6 | "alert", 7 | "chai", 8 | "console", 9 | "exports", 10 | "localStorage", 11 | "module", 12 | "parseInt", 13 | "sessionStorage", 14 | "window", 15 | "$", 16 | "__dirname", 17 | "__filename", 18 | "suite", 19 | "test", 20 | "before", 21 | "beforeEach", 22 | "after", 23 | "afterEach" 24 | ], 25 | 26 | "bitwise": true, 27 | "curly": false, 28 | "eqeqeq": true, 29 | "forin": true, 30 | "immed": true, 31 | "latedef": false, 32 | "newcap": true, 33 | "noarg": true, 34 | "noempty": true, 35 | "nonew": true, 36 | "plusplus": false, 37 | "regexp": true, 38 | "strict": false, 39 | "trailing": true, 40 | "undef": true, 41 | 42 | "asi": false, 43 | "boss": false, 44 | "debug": false, 45 | "eqnull": true, 46 | "es5": true, 47 | "evil": false, 48 | "globalstrict": true, 49 | "iterator": false, 50 | "lastsemic": false, 51 | "laxbreak": false, 52 | "loopfunc": false, 53 | "onecase": false, 54 | "proto": false, 55 | "regexdash": false, 56 | "scripturl": false, 57 | "shadow": false, 58 | "sub": false, 59 | "supernew": false, 60 | 61 | "browser": true, 62 | "couch": false, 63 | "devel": false, 64 | "dojo": false, 65 | "jquery": false, 66 | "mootools": false, 67 | "node": true, 68 | "nonstandard": false, 69 | "prototypejs": false, 70 | "rhino": false, 71 | "wsh": false, 72 | 73 | "nomen": false, 74 | "onevar": false, 75 | "passfail": false, 76 | "white": true 77 | } 78 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "6" 4 | - "8" 5 | - "10" 6 | - "node" 7 | services: 8 | - redis-server -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # Authors ordered by first contribution. 2 | 3 | Jeff Barczewski (https://github.com/jeffbski) 4 | Carlos Rodriguez (https://github.com/carlos8f) 5 | 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2012 Jeff Barczewski 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # redis-rstream 2 | 3 | redis-rstream is a node.js redis read stream which streams binary or utf8 data in chunks from a redis key using an existing redis client. (streams2) Tested with mranney/node_redis client. 4 | 5 | [![Build Status](https://secure.travis-ci.org/jeffbski/redis-rstream.png?branch=master)](http://travis-ci.org/jeffbski/redis-rstream) 6 | 7 | ## Installation 8 | 9 | ```bash 10 | npm install redis-rstream 11 | ``` 12 | 13 | You will also need the `redis` client (`npm install redis`) or other compatible library. You an also optionally install `hiredis` along with `redis` for additional performance. 14 | 15 | ## Usage 16 | 17 | - `redisRStream(client, key, [options])` - Construct a read stream instance by passing in `client`, redis `key`, and options. Be sure to enable an option in your redis client to return Buffers for the data, like `detect_buffers: true` so that binary data will be read properly. The default `options.chunkSize` (the size of the data packets in the stream) is 64KB, this is ignored if using the streams2 read(chunkSize) since the provided chunkSize will be used instead. You can limit how many pending reads are allowed for this read stream, by specifying `options.maxPendingReads` which defaults to 2. You also have the option to specify a range with `options.startOffset` (inclusive) and `options.endOffset` (non-inclusive), streaming only the chosen segment. 18 | 19 | ```javascript 20 | var redis = require('redis'); 21 | var redisRStream = require('redis-rstream'); // factory 22 | var client = redis.createClient(null, null, {detect_buffers: true}); // create client using your options and auth 23 | redisRStream(client, 'keyToStreamFrom') // create instance of read stream w/default 64KB chunk size 24 | .pipe(...) 25 | ``` 26 | 27 | Tested with mranney/node_redis client, but should work with any client that implements: 28 | 29 | - `getrange(key, start, stop, cb)` - where key is a Buffer and the data returned is a Buffer 30 | 31 | 32 | ## Goals 33 | 34 | - Simple read stream which can use existing redis client (and especially mranney/node_redis) 35 | - Remove all the complexity of managing a stream and reading in chunks from a redis key 36 | - Create normal pausable node.js read stream which can be piped or used as any other stream 37 | - uses streams2 from node 0.10+. 38 | 39 | ## Why 40 | 41 | mranney/node_redis does not have direct ability to read a key as a stream, so rather than writing this logic again and again, wrap this up into a read stream so we simply point it to a key and it streams. 42 | 43 | Other redis stream implementations use their own direct network connections to redis, but I would prefer to use an existing connection for all my redis work which makes authentication and things lke failover easier to deal with. 44 | 45 | ## Get involved 46 | 47 | If you have input or ideas or would like to get involved, you may: 48 | 49 | - contact me via twitter @jeffbski - 50 | - open an issue on github to begin a discussion - 51 | - fork the repo and send a pull request (ideally with tests) - 52 | 53 | ## License 54 | 55 | - [MIT license](http://github.com/jeffbski/redis-rstream/raw/master/LICENSE) 56 | -------------------------------------------------------------------------------- /lib/redis-rstream.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /*global setImmediate:false */ 3 | 4 | var Stream = require('stream'); 5 | var util = require('util'); 6 | 7 | // node 0.10+ has Writable stream so use it if available 8 | // otherwise use readable-stream module 9 | var Readable = Stream.Readable; 10 | 11 | function RedisRStream(client, key, options) { 12 | if (!client || !key) throw new Error('RedisRStream requires client and key'); 13 | // allow call without new 14 | if (!(this instanceof RedisRStream)) return new RedisRStream(client, key, options); 15 | Readable.call(this, options); 16 | this._redisClient = client; 17 | this._redisKey = !!client.getrangeBuffer ? key : new Buffer(key); // using Buffer key so redis returns buffers 18 | this._redisChunkSize = (options && options.chunkSize) ? options.chunkSize : 64 * 1024; // default 64KB 19 | this._redisMaxPendingReads = (options && options.maxPendingReads) ? options.maxPendingReads : 2; 20 | this._redisOffset = (options && options.startOffset) ? options.startOffset : 0; 21 | this._redisStartOffset = (options && options.startOffset) ? options.startOffset : 0; 22 | this._redisEndOffset = (options && options.endOffset) ? options.endOffset : 0; 23 | this._redisLength = 0; 24 | this._redisEnded = false; 25 | this._redisPendingReads = 0; 26 | } 27 | 28 | util.inherits(RedisRStream, Readable); 29 | 30 | RedisRStream.prototype._read = function _read(size) { 31 | var self = this; 32 | if (self._redisPendingReads >= self._redisMaxPendingReads) return; 33 | size = size || self._redisChunkSize; 34 | var startOffset = self._redisOffset; 35 | var endOffset = startOffset + size - 1; 36 | if(self._redisEndOffset !== 0) { 37 | // -1 is due to the inclusive nature or redis getrange 38 | endOffset = Math.min(self._redisEndOffset - 1, endOffset); 39 | } 40 | self._redisOffset = endOffset + 1; 41 | self._redisPendingReads += 1; 42 | var getrangeCallback = function (err, buff) { 43 | self._redisPendingReads -= 1; 44 | if(buff) { 45 | self._redisLength += buff.length; 46 | } 47 | if (err) return self.emit('error', err); 48 | if (!buff.length) { 49 | if (!self._redisEnded) { 50 | self._redisEnded = true; 51 | self.push(null); // ended 52 | } 53 | return; 54 | } 55 | try { 56 | if (self.push(buff)) { // continue reading 57 | if(self._redisEndOffset !== 0 && self._redisLength >= (self._redisEndOffset - self._redisStartOffset) || buff.length < (endOffset - startOffset)) { 58 | if (!self._redisEnded) { 59 | self._redisEnded = true; 60 | self.push(null); // ended 61 | } 62 | return; 63 | } 64 | process.nextTick(function () { self._read(size); }); 65 | } 66 | } catch (err) { 67 | self._redisEnded = true; 68 | self.emit('error', err); 69 | return; 70 | } 71 | }; 72 | if (self._redisClient.getrangeBuffer) { 73 | self._redisClient.getrangeBuffer(self._redisKey, startOffset, endOffset, getrangeCallback) 74 | } else { 75 | self._redisClient.getrange(self._redisKey, startOffset, endOffset, getrangeCallback) 76 | } 77 | }; 78 | 79 | module.exports = RedisRStream; 80 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "redis-rstream", 3 | "version": "1.0.1", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "amdefine": { 8 | "version": "0.0.8", 9 | "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-0.0.8.tgz", 10 | "integrity": "sha1-NNyMmB5qyzvhhTvvjw7JSjnVW6A=", 11 | "dev": true 12 | }, 13 | "assertion-error": { 14 | "version": "1.1.0", 15 | "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", 16 | "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", 17 | "dev": true 18 | }, 19 | "balanced-match": { 20 | "version": "1.0.0", 21 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", 22 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", 23 | "dev": true 24 | }, 25 | "brace-expansion": { 26 | "version": "1.1.11", 27 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", 28 | "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", 29 | "dev": true, 30 | "requires": { 31 | "balanced-match": "^1.0.0", 32 | "concat-map": "0.0.1" 33 | } 34 | }, 35 | "browser-stdout": { 36 | "version": "1.3.1", 37 | "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", 38 | "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", 39 | "dev": true 40 | }, 41 | "chai": { 42 | "version": "4.1.2", 43 | "resolved": "https://registry.npmjs.org/chai/-/chai-4.1.2.tgz", 44 | "integrity": "sha1-D2RYS6ZC8PKs4oBiefTwbKI61zw=", 45 | "dev": true, 46 | "requires": { 47 | "assertion-error": "^1.0.1", 48 | "check-error": "^1.0.1", 49 | "deep-eql": "^3.0.0", 50 | "get-func-name": "^2.0.0", 51 | "pathval": "^1.0.0", 52 | "type-detect": "^4.0.0" 53 | } 54 | }, 55 | "chai-stack": { 56 | "version": "1.3.1", 57 | "resolved": "https://registry.npmjs.org/chai-stack/-/chai-stack-1.3.1.tgz", 58 | "integrity": "sha1-WASRQweLmANaYnLPy73N5VFYSOs=", 59 | "dev": true, 60 | "requires": { 61 | "amdefine": "~0.0.4", 62 | "chai": ">=0.3.4" 63 | } 64 | }, 65 | "check-error": { 66 | "version": "1.0.2", 67 | "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", 68 | "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", 69 | "dev": true 70 | }, 71 | "commander": { 72 | "version": "2.15.1", 73 | "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", 74 | "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", 75 | "dev": true 76 | }, 77 | "concat-map": { 78 | "version": "0.0.1", 79 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 80 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", 81 | "dev": true 82 | }, 83 | "debug": { 84 | "version": "3.1.0", 85 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", 86 | "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", 87 | "dev": true, 88 | "requires": { 89 | "ms": "2.0.0" 90 | } 91 | }, 92 | "deep-eql": { 93 | "version": "3.0.1", 94 | "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", 95 | "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", 96 | "dev": true, 97 | "requires": { 98 | "type-detect": "^4.0.0" 99 | } 100 | }, 101 | "diff": { 102 | "version": "3.5.0", 103 | "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", 104 | "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", 105 | "dev": true 106 | }, 107 | "double-ended-queue": { 108 | "version": "2.1.0-0", 109 | "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz", 110 | "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=", 111 | "dev": true 112 | }, 113 | "escape-string-regexp": { 114 | "version": "1.0.5", 115 | "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", 116 | "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", 117 | "dev": true 118 | }, 119 | "fs.realpath": { 120 | "version": "1.0.0", 121 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 122 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", 123 | "dev": true 124 | }, 125 | "get-func-name": { 126 | "version": "2.0.0", 127 | "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", 128 | "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", 129 | "dev": true 130 | }, 131 | "glob": { 132 | "version": "7.1.2", 133 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", 134 | "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", 135 | "dev": true, 136 | "requires": { 137 | "fs.realpath": "^1.0.0", 138 | "inflight": "^1.0.4", 139 | "inherits": "2", 140 | "minimatch": "^3.0.4", 141 | "once": "^1.3.0", 142 | "path-is-absolute": "^1.0.0" 143 | } 144 | }, 145 | "growl": { 146 | "version": "1.10.5", 147 | "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", 148 | "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", 149 | "dev": true 150 | }, 151 | "has-flag": { 152 | "version": "3.0.0", 153 | "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", 154 | "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", 155 | "dev": true 156 | }, 157 | "he": { 158 | "version": "1.1.1", 159 | "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", 160 | "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", 161 | "dev": true 162 | }, 163 | "inflight": { 164 | "version": "1.0.6", 165 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 166 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", 167 | "dev": true, 168 | "requires": { 169 | "once": "^1.3.0", 170 | "wrappy": "1" 171 | } 172 | }, 173 | "inherits": { 174 | "version": "2.0.3", 175 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", 176 | "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", 177 | "dev": true 178 | }, 179 | "minimatch": { 180 | "version": "3.0.4", 181 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", 182 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", 183 | "dev": true, 184 | "requires": { 185 | "brace-expansion": "^1.1.7" 186 | } 187 | }, 188 | "minimist": { 189 | "version": "0.0.8", 190 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", 191 | "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", 192 | "dev": true 193 | }, 194 | "mkdirp": { 195 | "version": "0.5.1", 196 | "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", 197 | "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", 198 | "dev": true, 199 | "requires": { 200 | "minimist": "0.0.8" 201 | } 202 | }, 203 | "mocha": { 204 | "version": "5.2.0", 205 | "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.2.0.tgz", 206 | "integrity": "sha512-2IUgKDhc3J7Uug+FxMXuqIyYzH7gJjXECKe/w43IGgQHTSj3InJi+yAA7T24L9bQMRKiUEHxEX37G5JpVUGLcQ==", 207 | "dev": true, 208 | "requires": { 209 | "browser-stdout": "1.3.1", 210 | "commander": "2.15.1", 211 | "debug": "3.1.0", 212 | "diff": "3.5.0", 213 | "escape-string-regexp": "1.0.5", 214 | "glob": "7.1.2", 215 | "growl": "1.10.5", 216 | "he": "1.1.1", 217 | "minimatch": "3.0.4", 218 | "mkdirp": "0.5.1", 219 | "supports-color": "5.4.0" 220 | } 221 | }, 222 | "ms": { 223 | "version": "2.0.0", 224 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 225 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", 226 | "dev": true 227 | }, 228 | "once": { 229 | "version": "1.4.0", 230 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 231 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 232 | "dev": true, 233 | "requires": { 234 | "wrappy": "1" 235 | } 236 | }, 237 | "path-is-absolute": { 238 | "version": "1.0.1", 239 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 240 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", 241 | "dev": true 242 | }, 243 | "pathval": { 244 | "version": "1.1.0", 245 | "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", 246 | "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", 247 | "dev": true 248 | }, 249 | "redis": { 250 | "version": "2.8.0", 251 | "resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz", 252 | "integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==", 253 | "dev": true, 254 | "requires": { 255 | "double-ended-queue": "^2.1.0-0", 256 | "redis-commands": "^1.2.0", 257 | "redis-parser": "^2.6.0" 258 | } 259 | }, 260 | "redis-commands": { 261 | "version": "1.3.5", 262 | "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.3.5.tgz", 263 | "integrity": "sha512-foGF8u6MXGFF++1TZVC6icGXuMYPftKXt1FBT2vrfU9ZATNtZJ8duRC5d1lEfE8hyVe3jhelHGB91oB7I6qLsA==", 264 | "dev": true 265 | }, 266 | "redis-parser": { 267 | "version": "2.6.0", 268 | "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz", 269 | "integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs=", 270 | "dev": true 271 | }, 272 | "supports-color": { 273 | "version": "5.4.0", 274 | "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", 275 | "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==", 276 | "dev": true, 277 | "requires": { 278 | "has-flag": "^3.0.0" 279 | } 280 | }, 281 | "type-detect": { 282 | "version": "4.0.8", 283 | "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", 284 | "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", 285 | "dev": true 286 | }, 287 | "wrappy": { 288 | "version": "1.0.2", 289 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 290 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", 291 | "dev": true 292 | } 293 | } 294 | } 295 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "redis-rstream", 3 | "description": "redis-rstream - node.js redis read stream which streams binary or utf8 data in chunks from a redis key using an existing redis client (streams2)", 4 | "version": "1.0.1", 5 | "author": "Jeff Barczewski ", 6 | "repository": { 7 | "type": "git", 8 | "url": "git://github.com/jeffbski/redis-rstream.git" 9 | }, 10 | "bugs": { 11 | "url": "http://github.com/jeffbski/redis-rstream/issues" 12 | }, 13 | "license": "MIT", 14 | "main": "lib/redis-rstream", 15 | "engines": { 16 | "node": ">=0.10" 17 | }, 18 | "dependencies": {}, 19 | "devDependencies": { 20 | "chai": "^4.1.2", 21 | "chai-stack": "~1.3.1", 22 | "mocha": "^5.2.0", 23 | "redis": "^2.8.0" 24 | }, 25 | "scripts": { 26 | "test": "./node_modules/mocha/bin/mocha ./test/*.mocha.js" 27 | }, 28 | "keywords": [ 29 | "stream", 30 | "streams2", 31 | "read", 32 | "redis", 33 | "binary" 34 | ], 35 | "optionalDependencies": {} 36 | } 37 | -------------------------------------------------------------------------------- /test/basic.mocha.js: -------------------------------------------------------------------------------- 1 | /*global suite:false test:false */ 2 | 'use strict'; 3 | 4 | var chai = require('chai-stack'); 5 | var crypto = require('crypto'); 6 | var redisRStream = require('..'); // require('redis-rstream'); 7 | var redis = require('redis'); 8 | var client = redis.createClient(null, null, { detect_buffers: true }); 9 | 10 | 11 | var t = chai.assert; 12 | 13 | var KEY = 'foo'; 14 | 15 | suite('basic'); 16 | 17 | var TEST_DATA_SIZE = 2 * 1024 * 1024 + 25; // 2,025KB 18 | var TEST_DATA = crypto.randomBytes(TEST_DATA_SIZE); 19 | var EXPECTED_DIGEST = crypto.createHash('sha1').update(TEST_DATA).digest('base64'); 20 | var START_OFFSET = 101; 21 | var LENGTH = 64*1024*2 + 5; // Two default chunks and 5 odd bytes 22 | var END_OFFSET = START_OFFSET + LENGTH; 23 | var EXPECTED_DIGEST2 = crypto.createHash('sha1').update(TEST_DATA.slice(START_OFFSET, END_OFFSET)).digest('base64'); 24 | 25 | function setupDataForKey(cb) { 26 | client.set(KEY, TEST_DATA, cb); 27 | } 28 | 29 | function cleanup(cb) { 30 | client.del(KEY, cb); 31 | } 32 | 33 | before(function (done) { setupDataForKey(done); }); 34 | after(function (done) { cleanup(done); }); 35 | after(function () { client.quit(); }) 36 | 37 | test('basic use streams2 binary data from key', function (done) { 38 | var stream = redisRStream(client, KEY); 39 | var accum = []; 40 | stream 41 | .on('error', function (err) { done(err); }) 42 | .on('readable', function () { 43 | var data; 44 | while ((data = stream.read()) !== null) { 45 | accum.push(data); 46 | } 47 | }) 48 | .on('end', function () { 49 | var allBuffer = Buffer.concat(accum); 50 | var allDigest = crypto.createHash('sha1').update(allBuffer).digest('base64'); 51 | t.equal(allDigest, EXPECTED_DIGEST); 52 | done(); 53 | }); 54 | }); 55 | 56 | test('basic use stream binary data from key, defaults to 64KB chunks', function (done) { 57 | var stream = redisRStream(client, KEY); 58 | var accum = []; 59 | stream 60 | .on('error', function (err) { done(err); }) 61 | .on('data', function (data) { 62 | accum.push(data); 63 | }) 64 | .on('end', function () { 65 | var allBuffer = Buffer.concat(accum); 66 | var allDigest = crypto.createHash('sha1').update(allBuffer).digest('base64'); 67 | t.equal(allDigest, EXPECTED_DIGEST); 68 | done(); 69 | }); 70 | }); 71 | 72 | test('basic use stream binary data from key and limited offset, defaults to 64KB chunks', function (done) { 73 | var stream = redisRStream(client, KEY, {startOffset: START_OFFSET, endOffset: END_OFFSET}); 74 | var accum = []; 75 | stream 76 | .on('error', function (err) { done(err); }) 77 | .on('data', function (data) { 78 | accum.push(data); 79 | }) 80 | .on('end', function () { 81 | var allBuffer = Buffer.concat(accum); 82 | var allDigest = crypto.createHash('sha1').update(allBuffer).digest('base64'); 83 | t.equal(allBuffer.length, LENGTH); 84 | t.equal(allDigest, EXPECTED_DIGEST2); 85 | done(); 86 | }); 87 | }); 88 | 89 | 90 | 91 | test('all arguments missing for factory, throws error', function () { 92 | function throwsErr() { 93 | var stream = redisRStream(); 94 | } 95 | t.throws(throwsErr, /RedisRStream requires client and key/); 96 | }); 97 | 98 | test('client null, throws error', function () { 99 | function throwsErr() { 100 | var stream = redisRStream(null, KEY); 101 | } 102 | t.throws(throwsErr, /RedisRStream requires client and key/); 103 | }); 104 | 105 | test('key null, throws error', function () { 106 | function throwsErr() { 107 | var stream = redisRStream(client, null); 108 | } 109 | t.throws(throwsErr, /RedisRStream requires client and key/); 110 | }); 111 | -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --reporter spec 2 | --ui qunit --------------------------------------------------------------------------------