├── test ├── fixtures │ ├── emptydoc.txt │ ├── text.txt │ └── mongo.png └── index.js ├── index.js ├── .gitignore ├── Makefile ├── package.json ├── LICENSE ├── History.md ├── lib ├── index.js ├── readstream.js └── writestream.js └── Readme.md /test/fixtures/emptydoc.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/fixtures/text.txt: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | module.exports = exports = require('./lib'); 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swo 2 | *.swp 3 | *.swn 4 | node_modules/ 5 | .DS_Store 6 | -------------------------------------------------------------------------------- /test/fixtures/mongo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aheckmann/gridfs-stream/HEAD/test/fixtures/mongo.png -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | TESTS = test/*.js 3 | 4 | test: 5 | @./node_modules/mocha/bin/mocha --reporter list $(TESTFLAGS) $(TESTS) 6 | 7 | .PHONY: test 8 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "author": "Aaron Heckmann ", 3 | "name": "gridfs-stream", 4 | "description": "Writable/Readable Nodejs compatible GridFS streams", 5 | "version": "1.1.1", 6 | "keywords": [ 7 | "mongodb", 8 | "mongoose", 9 | "gridfs" 10 | ], 11 | "scripts": { 12 | "test": "make test" 13 | }, 14 | "dependencies": { 15 | "flushwritable": "^1.0.0" 16 | }, 17 | "devDependencies": { 18 | "mocha": "*", 19 | "mongodb": "2.0.15", 20 | "checksum": "~0.1.1" 21 | }, 22 | "optionalDependencies": {}, 23 | "engines": { 24 | "node": ">= 0.4.2" 25 | }, 26 | "repository": { 27 | "type": "git", 28 | "url": "git://github.com/aheckmann/gridfs-stream.git" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | (The MIT License) 2 | 3 | Copyright (c) 2012 [Aaron Heckmann](aaron.heckmann+github@gmail.com) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | 'Software'), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /History.md: -------------------------------------------------------------------------------- 1 | 1.1.1 / 2015-04-03 2 | ================== 3 | 4 | * fixed; Not working with Mongoose 4.0.1 / Mongo JS 2.0 #72 5 | * fixed; Fix a bug of GridWriteStream constructor #71 [pine](https://github.com/pine) 6 | 7 | 1.1.0 / 2015-02-15 8 | ================== 9 | 10 | * fixed; Patch stream2 writable v2 #67 [triccardi-systran](https://github.com/triccardi-systran) 11 | 12 | 1.0.1 / 2015-02-15 13 | ================== 14 | 15 | * fixed; Add findOne function and fix Grid.collection not reflecting name #64 [healiha](https://github.com/healiha) 16 | 17 | 1.0.0 / 2015-02-11 18 | ================== 19 | 20 | * first version working with mongodb 2.x driver [riaan53](https://github.com/riaan53) 21 | 22 | 0.5.3 / 2014-10-23 23 | ================== 24 | 25 | * fixed; 'exist' method does not respect 'root' option #50 [nachoalthabe](https://github.com/nachoalthabe) 26 | 27 | 0.5.2 / 2014-10-23 28 | ================== 29 | 30 | * fixed; multiple, multi-chunk stream support #47 [ceari](https://github.com/ceari) 31 | 32 | 0.5.1 / 2014-05-14 33 | ================== 34 | 35 | * fixed; support 0-byte files #39 [PNPhillips](https://github.com/PNPhillips) 36 | * fixed; do not overwrite the filename with empty string #38 [vsivsi](https://github.com/vsivsi) 37 | 38 | 0.5.0 / 2014-05-03 39 | ================== 40 | 41 | * added; range read support #37 from [xissy](https://github.com/xissy) 42 | * added; support for file existance queries #32 [jaalfaro](https://github.com/jaalfaro) 43 | * docs; show that Mongo Db needs to be initalized #33 [marcusoftnet](https://github.com/marcusoftnet) 44 | 45 | 0.4.1 / 2014-01-22 46 | ================== 47 | 48 | * fixed; Changing chunk_size to chunkSize #27 from [khous](https://github.com/khous) 49 | * fixed; Server is undefined, should be mongo.Server instead #23 from [sahat](https://github.com/sahat) 50 | 51 | 0.4.0 / 2013-04-07 52 | ================== 53 | 54 | * changed; ReadStream/WriteStream api #4 #6 #7 #10 #11 #15 [Reggino](https://github.com/Reggino) 55 | * fixed; parameter ambiguity (objectId / filename) for GridReadStream, GridWriteStream, grid#remove() #11 [Reggino](https://github.com/Reggino) 56 | * fixed; Gridfs should be able to store 12 lettered file names Issue #11 [Reggino](https://github.com/Reggino) 57 | * fixed; ReadStream pause() / resume() issue #12 #13 [Reggino](https://github.com/Reggino) 58 | * fixed; #4 59 | * fixed; #6 60 | * fixed; #7 61 | * fixed; #10 62 | 63 | 0.3.2 / 2012-10-25 64 | ================== 65 | 66 | * add; passing File object when emmiting close event [diogogmt](https://github.com/diogogmt) 67 | * update readme 68 | 69 | 0.3.1 / 2012-09-12 70 | ================== 71 | 72 | * add; remove method 73 | 74 | 0.3.0 / 2012-09-12 75 | ================== 76 | 77 | * refactor api 78 | * added; grid#files & grid#collection() 79 | 80 | 0.2.1 / 2012-09-12 81 | ================== 82 | 83 | * explicit id creation 84 | 85 | 0.2.0 / 2012-09-11 86 | ================== 87 | 88 | * update driver to 1.1.7 89 | * auto cast to ObjectId when possible 90 | 91 | 0.1.1 / 2012-08-31 92 | ================== 93 | 94 | * updated; mongodb version to 1.1.5 95 | 96 | 0.1.0 / 06-13-2012 97 | ================== 98 | 99 | * fixed; node-mongodb-native compatibility issues (#1) 100 | 101 | 0.0.5 / 05-18-2012 102 | ================== 103 | 104 | * fixed; node v4 compatibility 105 | 106 | 0.0.4 / 05-11-2012 107 | ================== 108 | 109 | * fixed; do not flush until open 110 | * updated; mongodb driver 111 | 112 | 0.0.3 / 05-10-2012 113 | ================== 114 | 115 | * fixed; only open once 116 | 117 | 0.0.2 / 05-10-2012 118 | ================== 119 | 120 | * add GridWriteStream progress event 121 | 122 | 0.0.1 / 05-09-2012 123 | ================== 124 | 125 | * initial release 126 | 127 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | // gridfs-stream 2 | 3 | /** 4 | * Module dependencies. 5 | */ 6 | 7 | var GridWriteStream = require('./writestream') 8 | var GridReadStream = require('./readstream') 9 | 10 | /** 11 | * Grid constructor 12 | * 13 | * @param {mongo.Db} db - an open mongo.Db instance 14 | * @param {mongo} [mongo] - the native driver you are using 15 | */ 16 | 17 | function Grid (db, mongo) { 18 | if (!(this instanceof Grid)) { 19 | return new Grid(db, mongo); 20 | } 21 | 22 | mongo || (mongo = Grid.mongo ? Grid.mongo : undefined); 23 | 24 | if (!mongo) throw new Error('missing mongo argument\nnew Grid(db, mongo)'); 25 | if (!db) throw new Error('missing db argument\nnew Grid(db, mongo)'); 26 | 27 | // the db must already be open b/c there is no `open` event emitted 28 | // in old versions of the driver 29 | this.db = db; 30 | this.mongo = mongo; 31 | this.curCol = this.mongo.GridStore ? this.mongo.GridStore.DEFAULT_ROOT_COLLECTION : 'fs'; 32 | } 33 | 34 | /** 35 | * Creates a writable stream. 36 | * 37 | * @param {Object} [options] 38 | * @return Stream 39 | */ 40 | 41 | Grid.prototype.createWriteStream = function (options) { 42 | return new GridWriteStream(this, options); 43 | } 44 | 45 | /** 46 | * Creates a readable stream. Pass at least a filename or _id option 47 | * 48 | * @param {Object} options 49 | * @return Stream 50 | */ 51 | 52 | Grid.prototype.createReadStream = function (options) { 53 | return new GridReadStream(this, options); 54 | } 55 | 56 | /** 57 | * The collection used to store file data in mongodb. 58 | * @return {Collection} 59 | */ 60 | 61 | Object.defineProperty(Grid.prototype, 'files', { 62 | get: function () { 63 | if (this._col) return this._col; 64 | return this.collection(); 65 | } 66 | }); 67 | 68 | /** 69 | * Changes the default collection to `name` or to the default mongodb gridfs collection if not specified. 70 | * 71 | * @param {String|undefined} name root gridfs collection name 72 | * @return {Collection} 73 | */ 74 | 75 | Grid.prototype.collection = function (name) { 76 | this.curCol = name || this.curCol || this.mongo.GridStore.DEFAULT_ROOT_COLLECTION; 77 | return this._col = this.db.collection(this.curCol + ".files"); 78 | } 79 | 80 | /** 81 | * Removes a file by passing any options, at least an _id or filename 82 | * 83 | * @param {Object} options 84 | * @param {Function} callback 85 | */ 86 | 87 | Grid.prototype.remove = function (options, callback) { 88 | var _id; 89 | if (options._id) { 90 | _id = this.tryParseObjectId(options._id) || options._id; 91 | } 92 | if (!_id) { 93 | _id = options.filename; 94 | } 95 | return this.mongo.GridStore.unlink(this.db, _id, options, callback); 96 | } 97 | 98 | /** 99 | * Checks if a file exists by passing a filename 100 | * 101 | * @param {Object} options 102 | * @param {Function} callback 103 | */ 104 | 105 | Grid.prototype.exist = function (options, callback) { 106 | var _id; 107 | if (options._id) { 108 | _id = this.tryParseObjectId(options._id) || options._id; 109 | } 110 | if (!_id) { 111 | _id = options.filename; 112 | } 113 | return this.mongo.GridStore.exist(this.db, _id, options.root, callback); 114 | } 115 | 116 | /** 117 | * Find file by passing any options, at least an _id or filename 118 | * 119 | * @param {Object} options 120 | * @param {Function} callback 121 | */ 122 | 123 | Grid.prototype.findOne = function (options, callback) { 124 | if ('function' != typeof callback) { 125 | throw new Error('findOne requires a callback function'); 126 | } 127 | var find = {}; 128 | for (var n in options) { 129 | if (n != 'root') { 130 | find[n] = options[n]; 131 | } 132 | } 133 | if (find._id) { 134 | find._id = this.tryParseObjectId(find._id) || find._id; 135 | } 136 | var collection = options.root && options.root != this.curCol ? this.db.collection(options.root + ".files") : this.files; 137 | if (!collection) { 138 | return callback(new Error('No collection specified')); 139 | } 140 | collection.find(find, function(err, cursor) { 141 | if (err) { return callback(err); } 142 | if (!cursor) { return callback(new Error('Collection not found')); } 143 | cursor.nextObject(callback); 144 | }); 145 | } 146 | 147 | /** 148 | * Attemps to parse `string` into an ObjectId 149 | * 150 | * @param {GridReadStream} self 151 | * @param {String|ObjectId} string 152 | * @return {ObjectId|Boolean} 153 | */ 154 | 155 | Grid.prototype.tryParseObjectId = function tryParseObjectId (string) { 156 | try { 157 | return new this.mongo.ObjectID(string); 158 | } catch (_) { 159 | return false; 160 | } 161 | } 162 | 163 | /** 164 | * expose 165 | */ 166 | 167 | module.exports = exports = Grid; 168 | -------------------------------------------------------------------------------- /lib/readstream.js: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * Module dependencies 4 | */ 5 | 6 | var util = require('util'); 7 | var Readable = require('stream').Readable; 8 | 9 | /** 10 | * expose 11 | * @ignore 12 | */ 13 | 14 | module.exports = exports = GridReadStream; 15 | 16 | /** 17 | * GridReadStream 18 | * 19 | * @param {Grid} grid 20 | * @param {Object} options 21 | */ 22 | 23 | function GridReadStream (grid, options) { 24 | if (!(this instanceof GridReadStream)) 25 | return new GridReadStream(grid, options); 26 | 27 | Readable.call(this); 28 | this._opened = false; 29 | this._opening = false; 30 | this._closing = false; 31 | this._end = false; 32 | this._needToPush = false; 33 | 34 | this._grid = grid; 35 | 36 | // a bit backwards compatible 37 | if (typeof options === 'string') { 38 | options = { filename: options }; 39 | } 40 | 41 | this.options = options || {}; 42 | 43 | if(options._id) { 44 | this.id = grid.tryParseObjectId(options._id); 45 | 46 | if(!this.id) { 47 | this.id = options._id; 48 | } 49 | } 50 | 51 | this.name = this.options.filename || ''; 52 | this.mode = 'r'; 53 | 54 | // If chunk size specified use it for read chunk size otherwise default to 255k (GridStore default). chunkSize and chunk_size in mongodb api so check both. 55 | this._chunkSize = this.options.chunkSize || this.options.chunk_size || 1024 * 255; 56 | 57 | this.range = this.options.range || { startPos: 0, endPos: undefined }; 58 | if (typeof(this.range.startPos) === 'undefined') { 59 | this.range.startPos = 0; 60 | } 61 | 62 | this._currentPos = this.range.startPos; 63 | 64 | var options = {}; 65 | for (var i in this.options) { options[i] = this.options[i]; } 66 | options.root || (options.root = this._grid.curCol); 67 | 68 | this._store = new grid.mongo.GridStore(grid.db, this.id || new grid.mongo.ObjectID(), this.name, this.mode, options); 69 | // Workaround for Gridstore issue https://github.com/mongodb/node-mongodb-native/pull/930 70 | if (!this.id) { 71 | // var REFERENCE_BY_FILENAME = 0, 72 | this._store.referenceBy = 0; 73 | } 74 | 75 | var self = this; 76 | 77 | //Close the store once `end` received 78 | this.on('end', function() { 79 | self._end = true; 80 | self._close() 81 | }); 82 | 83 | process.nextTick(function() { 84 | self._open(); 85 | }); 86 | } 87 | 88 | /** 89 | * Inherit from stream.Readable 90 | * @ignore 91 | */ 92 | 93 | util.inherits(GridReadStream, Readable); 94 | 95 | /** 96 | * _open 97 | * 98 | * @api private 99 | */ 100 | 101 | GridReadStream.prototype._open = function _open () { 102 | if (this._opening) return; 103 | this._opening = true; 104 | 105 | var self = this; 106 | 107 | // Open the store 108 | this._store.open(function (err, gs) { 109 | if (err) return self._error(err); 110 | 111 | // Find the length of the file by setting the head to the end of the file and requesting the position 112 | self._store.seek(0, self._grid.mongo.GridStore.IO_SEEK_END, function(err) { 113 | if (err) return self._error(err); 114 | 115 | // Request the position of the end of the file 116 | self._store.tell(function(err, position) { 117 | if (err) return self._error(err); 118 | 119 | // Calculate the correct end position either from EOF or end of range. Also handle incorrect range request. 120 | if (!self.range.endPos || self.range.endPos > position-1) {self.range.endPos = position - 1}; 121 | 122 | // Set the read head to the beginning of the file or start position if specified 123 | self._store.seek(self.range.startPos, self._grid.mongo.GridStore.IO_SEEK_SET, function(err) { 124 | if (err) return self._error(err); 125 | 126 | // The store is now open 127 | self.emit('open'); 128 | self._opened = true; 129 | 130 | // If `_read()` was already called then we need to start pushing data to the stream. Otherwise `_read()` will handle this once called from stream. 131 | if (self._needToPush) self._push(); 132 | }); 133 | }); 134 | }); 135 | }); 136 | } 137 | 138 | /** 139 | * _read 140 | * 141 | * @api private 142 | */ 143 | 144 | // `_read()` will be called when the stream wants to pull more data in 145 | // The advisory `size` argument is ignored in this case and user specified use or default to 255kk. 146 | GridReadStream.prototype._read = function _read (size) { 147 | var self = this; 148 | 149 | // Set `_needToPush` to true because the store may still be closed if data is immediately piped. Once the store is open `_needToPush` is checked and _push() called if necessary. 150 | self._needToPush = true; 151 | 152 | // The store must be open 153 | if (!this._opened) return; 154 | 155 | // Read data from GridStore and push to stream 156 | self._push(); 157 | } 158 | 159 | /** 160 | * _push 161 | * 162 | * @api private 163 | */ 164 | 165 | GridReadStream.prototype._push = function _push () { 166 | var self = this; 167 | 168 | // Do not continue if the store is closed 169 | if (!this._opened) return self._error('Unable to push data. Expected gridstore to be open'); 170 | 171 | // Check if EOF, if the full requested range has been pushed or if the stream must be destroyed. If so than push EOF-signalling `null` chunk 172 | if ( !this._store.eof() && (self._currentPos <= self.range.endPos) && !this._end) { 173 | 174 | // Determine the chunk size for the read from GridStore 175 | // Use default chunk size or user specified 176 | var readChunkSize = self._chunkSize 177 | // Override the chunk size if the chunk size is more than the size that is left until EOF/range 178 | if (self.range.endPos-self._currentPos < self._chunkSize) {readChunkSize = self.range.endPos - self._currentPos + 1}; 179 | 180 | // Read the chunk from GridStore. Head moves automatically after each read. 181 | self._store.read(readChunkSize,function(err, data) { 182 | 183 | // If error stop and close the store 184 | if (err) return self._error(err); 185 | 186 | // Advance the current position of the read head 187 | self._currentPos += data.length; 188 | 189 | // Push data 190 | if (!self._end) self.push(data) 191 | }) 192 | 193 | 194 | } else { 195 | // Push EOF-signalling `null` chunk 196 | this._end = true; 197 | self.push(null); 198 | } 199 | } 200 | 201 | /** 202 | * _close 203 | * 204 | * @api private 205 | */ 206 | 207 | GridReadStream.prototype._close = function _close () { 208 | var self = this; 209 | if (!self._opened) return; 210 | if (self._closing) return; 211 | this._closing = true; 212 | 213 | // Close the store and emit `close` event 214 | self._store.close(function (err) { 215 | if (err) return self._error(err); 216 | self.emit('close'); 217 | }); 218 | } 219 | 220 | /** 221 | * _error 222 | * 223 | * @api private 224 | */ 225 | 226 | GridReadStream.prototype._error = function _error (err) { 227 | // Set end true so that no further reads from GridStore are possible and close the store 228 | this._end = true; 229 | 230 | // Emit the error event 231 | this.emit('error', err); 232 | 233 | // Close the gridstore if an error is received. 234 | this._close() 235 | } 236 | 237 | /** 238 | * destroy 239 | * 240 | * @api public 241 | */ 242 | 243 | GridReadStream.prototype.destroy = function destroy () { 244 | // Set end true so that no further reads from GridSotre are possible and close the store 245 | this._end = true; 246 | this._close(); 247 | } 248 | -------------------------------------------------------------------------------- /lib/writestream.js: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * Module dependencies 4 | */ 5 | 6 | var util = require('util'); 7 | //var Writable = require('stream').Writable; 8 | 9 | // This is a workaround to implement a _flush method for Writable (like for Transform) to emit the 'finish' event only after all data has been flushed to the underlying system (GridFS). See https://www.npmjs.com/package/flushwritable and https://github.com/joyent/node/issues/7348 10 | var FlushWritable = require('flushwritable'); 11 | 12 | /** 13 | * expose 14 | * @ignore 15 | */ 16 | 17 | module.exports = exports = GridWriteStream; 18 | 19 | /** 20 | * GridWriteStream 21 | * 22 | * @param {Grid} grid 23 | * @param {Object} options (optional) 24 | */ 25 | 26 | function GridWriteStream (grid, options) { 27 | if (!(this instanceof GridWriteStream)) 28 | return new GridWriteStream(grid, options); 29 | 30 | FlushWritable.call(this); 31 | this._opened = false; 32 | this._opening = false; 33 | this._writable = true; 34 | this._closing = false; 35 | this._destroyed = false; 36 | this._errorEmitted = false; 37 | this._grid = grid; 38 | 39 | // a bit backwards compatible 40 | if (typeof options === 'string') { 41 | options = { filename: options }; 42 | } 43 | this.options = options || {}; 44 | if(this.options._id) { 45 | this.id = grid.tryParseObjectId(this.options._id); 46 | 47 | if(!this.id) { 48 | this.id = this.options._id; 49 | } 50 | } 51 | 52 | this.name = this.options.filename; // This may be undefined, that's okay 53 | 54 | if (!this.id) { 55 | //_id not passed or unparsable? This is a new file! 56 | this.id = new grid.mongo.ObjectID(); 57 | this.name = this.name || ''; // A new file needs a name 58 | } 59 | 60 | this.mode = 'w'; //Mongodb v2 driver have disabled w+ because of possible data corruption. So only allow `w` for now. 61 | 62 | // The value of this.name may be undefined. GridStore treats that as a missing param 63 | // in the call signature, which is what we want. 64 | this._store = new grid.mongo.GridStore(grid.db, this.id, this.name, this.mode, this.options); 65 | 66 | this._delayedWrite = null; 67 | this._delayedFlush = null; 68 | this._delayedClose = null; 69 | 70 | var self = this; 71 | 72 | self._open(); 73 | } 74 | 75 | /** 76 | * Inherit from stream.Writable (FlushWritable for workaround to defer finish until all data flushed) 77 | * @ignore 78 | */ 79 | 80 | util.inherits(GridWriteStream, FlushWritable); 81 | 82 | // private api 83 | 84 | /** 85 | * _open 86 | * 87 | * @api private 88 | */ 89 | 90 | GridWriteStream.prototype._open = function () { 91 | if (this._opened) return; 92 | if (this._opening) return; 93 | this._opening = true; 94 | 95 | var self = this; 96 | this._store.open(function (err, gs) { 97 | self._opening = false; 98 | if (err) return self._error(err); 99 | self._opened = true; 100 | self.emit('open'); 101 | 102 | // If _close was called during _store opening, then it was delayed until now, so do the close now 103 | if (self._delayedClose) { 104 | var closed = self._delayedClose.cb; 105 | self._delayedClose = null; 106 | return self._closeInternal(closed); 107 | } 108 | 109 | // If _flush was called during _store opening, then it was delayed until now, so do the flush now (it's necessarily an empty GridFS file, no _write could have been called and have finished) 110 | if (self._delayedFlush) { 111 | var flushed = self._delayedFlush; 112 | self._delayedFlush = null; 113 | return self._flushInternal(flushed); 114 | } 115 | 116 | // If _write was called during _store opening, then it was delayed until now, so do the write now (_flush could not have been called yet as _write has not finished yet) 117 | if (self._delayedWrite) { 118 | var delayedWrite = self._delayedWrite; 119 | self._delayedWrite = null; 120 | return self._writeInternal(delayedWrite.chunk, delayedWrite.encoding, delayedWrite.done); 121 | } 122 | }); 123 | } 124 | 125 | /** 126 | * _writeInternal 127 | * 128 | * @api private 129 | */ 130 | 131 | GridWriteStream.prototype._writeInternal = function (chunk, encoding, done) { 132 | // If destroy or error no more data will be written. 133 | if (!this._writable) return; 134 | 135 | var self = this; 136 | // Write the chunk to the GridStore. The write head automatically moves along with each write. 137 | this._store.write(chunk, function (err, store) { 138 | if (err) return self._error(err); 139 | 140 | // Emit the write head position 141 | self.emit('progress', store.position); 142 | 143 | // We are ready to receive a new chunk from the writestream - call done(). 144 | done(); 145 | }); 146 | } 147 | 148 | /** 149 | * _write 150 | * 151 | * @api private 152 | */ 153 | 154 | GridWriteStream.prototype._write = function (chunk, encoding, done) { 155 | if (this._opening) { 156 | // if we are still opening the store, then delay the write until it is open. 157 | this._delayedWrite = {chunk: chunk, encoding: encoding, done: done}; 158 | return; 159 | } 160 | 161 | // otherwise, do the write now 162 | this._writeInternal(chunk, encoding, done); 163 | } 164 | 165 | /** 166 | * _flushInternal 167 | * 168 | * @api private 169 | */ 170 | 171 | GridWriteStream.prototype._flushInternal = function (flushed) { 172 | this._close(flushed); 173 | } 174 | 175 | /** 176 | * _flush 177 | * 178 | * @api private 179 | */ 180 | 181 | GridWriteStream.prototype._flush = function (flushed) { 182 | // _flush is called when all _write() have finished (even if no _write() was called (empty GridFS file)) 183 | 184 | if (this._opening) { 185 | // if we are still opening the store, then delay the flush until it is open. 186 | this._delayedFlush = flushed; 187 | return; 188 | } 189 | 190 | // otherwise, do the flush now 191 | this._flushInternal(flushed); 192 | } 193 | 194 | 195 | /** 196 | * _closeInternal 197 | * 198 | * @api private 199 | */ 200 | 201 | GridWriteStream.prototype._closeInternal = function (cb) { 202 | if (!this._opened) return; 203 | if (this._closing) return; 204 | this._closing = true; 205 | 206 | var self = this; 207 | this._store.close(function (err, file) { 208 | self._closing = false; 209 | self._opened = false; 210 | if (err) return self._error(err); 211 | self.emit('close', file); 212 | 213 | if (cb) cb(); 214 | }); 215 | } 216 | 217 | /** 218 | * _close 219 | * 220 | * @api private 221 | */ 222 | 223 | GridWriteStream.prototype._close = function _close (cb) { 224 | if (this._opening) { 225 | // if we are still opening the store, then delay the close until it is open. 226 | this._delayedClose = { cb: cb }; 227 | return; 228 | } 229 | 230 | // otherwise, do the close now 231 | this._closeInternal(cb); 232 | } 233 | 234 | /** 235 | * _error 236 | * 237 | * @api private 238 | */ 239 | 240 | GridWriteStream.prototype._error = function _error (err) { 241 | // Stop receiving more data to write, emit `error` and close the store 242 | if (this._errorEmitted) return; 243 | this._errorEmitted = true; 244 | 245 | this._writable = false; 246 | this.emit('error', err); 247 | this._close(); 248 | } 249 | 250 | // public api 251 | 252 | /** 253 | * destroy 254 | * 255 | * @api public 256 | */ 257 | 258 | GridWriteStream.prototype.destroy = function destroy (err) { 259 | // Abort the write stream, even if write not completed 260 | if (this._destroyed) return; 261 | this._destroyed = true; 262 | 263 | var self = this; 264 | process.nextTick(function() { 265 | self._error(err); 266 | }); 267 | } 268 | 269 | 270 | /** 271 | * destroySoon 272 | * 273 | * @api public 274 | * @deprecated just use destroy() 275 | */ 276 | 277 | GridWriteStream.prototype.destroySoon = function destroySoon () { 278 | return this.destroy(); 279 | }; -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # gridfs-stream 2 | 3 | Easily stream files to and from MongoDB [GridFS](http://www.mongodb.org/display/DOCS/GridFS). 4 | 5 | ## Please note 6 | 7 | gridfs-stream v1.x uses [Stream2 API from nodejs v0.10](http://nodejs.org/docs/v0.10.36/api/stream.html) (and the mongodb v2.x driver). It provides more robust and easier to use streams. If for some reason you need nodejs v0.8 streams, please switch to the [gridfs-stream 0.x branch](https://github.com/aheckmann/gridfs-stream/tree/0.x) 8 | 9 | ## Description 10 | 11 | ```js 12 | var mongo = require('mongodb'); 13 | var Grid = require('gridfs-stream'); 14 | 15 | // create or use an existing mongodb-native db instance 16 | var db = new mongo.Db('yourDatabaseName', new mongo.Server("127.0.0.1", 27017)); 17 | var gfs = Grid(db, mongo); 18 | 19 | // streaming to gridfs 20 | var writestream = gfs.createWriteStream({ 21 | filename: 'my_file.txt' 22 | }); 23 | fs.createReadStream('/some/path').pipe(writestream); 24 | 25 | // streaming from gridfs 26 | var readstream = gfs.createReadStream({ 27 | filename: 'my_file.txt' 28 | }); 29 | 30 | //error handling, e.g. file does not exist 31 | readstream.on('error', function (err) { 32 | console.log('An error occurred!', err); 33 | throw err; 34 | }); 35 | 36 | readstream.pipe(response); 37 | ``` 38 | 39 | Alternatively you could read the file using an _id. This is often a better option, since filenames don't have to be unique within the collection. e.g. 40 | 41 | ```js 42 | var readstream = gfs.createReadStream({ 43 | _id: '50e03d29edfdc00d34000001' 44 | }); 45 | 46 | ``` 47 | 48 | Created streams are compatible with other Node streams so piping anywhere is easy. 49 | 50 | ## install 51 | 52 | ``` 53 | npm install gridfs-stream 54 | ``` 55 | 56 | ## use 57 | 58 | ```js 59 | var mongo = require('mongodb'); 60 | var Grid = require('gridfs-stream'); 61 | 62 | // create or use an existing mongodb-native db instance. 63 | // for this example we'll just create one: 64 | var db = new mongo.Db('yourDatabaseName', new mongo.Server("127.0.0.1", 27017)); 65 | 66 | // make sure the db instance is open before passing into `Grid` 67 | db.open(function (err) { 68 | if (err) return handleError(err); 69 | var gfs = Grid(db, mongo); 70 | 71 | // all set! 72 | }) 73 | ``` 74 | 75 | The `gridfs-stream` module exports a constructor that accepts an open [mongodb-native](https://github.com/mongodb/node-mongodb-native/) db and the [mongodb-native](https://github.com/mongodb/node-mongodb-native/) driver you are using. _The db must already be opened before calling `createWriteStream` or `createReadStream`._ 76 | 77 | Now we're ready to start streaming. 78 | 79 | ## createWriteStream 80 | 81 | To stream data to GridFS we call `createWriteStream` passing any options. 82 | 83 | ```js 84 | var writestream = gfs.createWriteStream([options]); 85 | fs.createReadStream('/some/path').pipe(writestream); 86 | ``` 87 | 88 | Options may contain zero or more of the following options, for more information see [GridStore](http://mongodb.github.com/node-mongodb-native/api-generated/gridstore.html): 89 | ```js 90 | { 91 | _id: '50e03d29edfdc00d34000001', // a MongoDb ObjectId 92 | filename: 'my_file.txt', // a filename 93 | mode: 'w', // default value: w 94 | 95 | //any other options from the GridStore may be passed too, e.g.: 96 | 97 | chunkSize: 1024, 98 | content_type: 'plain/text', // For content_type to work properly, set "mode"-option to "w" too! 99 | root: 'my_collection', 100 | metadata: { 101 | ... 102 | } 103 | } 104 | ``` 105 | 106 | ### Events 107 | 108 | The `writeStream` is a fully compliant [Stream2 Writable Stream](http://nodejs.org/docs/v0.10.36/api/stream.html#stream_class_stream_writable), it emits all the associated events (`drain`, `finish`, `pipe`, `unpipe`, `error`), as well as additional special events (`open`, `close`). 109 | 110 | `finish` is emitted after the file has been completely written to GridFS. 111 | 112 | `open` is emitted after the GridStore is successfully opened. 113 | 114 | `close` is emitted after the GridStore is successfully closed, which means the file is fully written to GridFS, and the file object is passed as the first argument. 115 | 116 | ```js 117 | writestream.on('close', function (file) { 118 | // do something with `file` 119 | console.log(file.filename); 120 | }); 121 | ``` 122 | 123 | ### Methods 124 | 125 | The `writeStream` has additional methods: 126 | 127 | `destroy([err])`: 128 | Destroy the `writeStream` as soon as possible: stop writing incoming data, close the _store. An `error` event will be emitted, as well as a `close` event. 129 | It's up to you to cleanup the GridStore if it's not desired to keep half written files in GridFS (the `close` event returns a GridStore `file` which can be used to delete the file, or mark it failed). 130 | 131 | ## createReadStream 132 | 133 | To stream data out of GridFS we call `createReadStream` passing any options, at least an `_id` or `filename`. 134 | 135 | ```js 136 | var readstream = gfs.createReadStream(options); 137 | readstream.pipe(response); 138 | ``` 139 | 140 | See the options of `createWriteStream` for more information. 141 | 142 | To get partial data with `createReadStream`, use `range` option. e.g. 143 | ```js 144 | var readstream = gfs.createReadStream({ 145 | _id: '50e03d29edfdc00d34000001', 146 | range: { 147 | startPos: 100, 148 | endPos: 500000 149 | } 150 | }); 151 | ``` 152 | 153 | ## removing files 154 | 155 | Files can be removed by passing options (at least an `_id` or `filename`) to the `remove()` method. 156 | 157 | ```js 158 | gfs.remove(options, function (err, gridStore) { 159 | if (err) return handleError(err); 160 | console.log('success'); 161 | }); 162 | ``` 163 | 164 | See the options of `createWriteStream` for more information. 165 | 166 | ## check if file exists 167 | 168 | Check if a file exist by passing options (at least an `_id` or `filename`) to the `exist()` method. 169 | 170 | ```js 171 | gfs.exist(options, function (err, found) { 172 | if (err) return handleError(err); 173 | found ? console.log('File exists') : console.log('File does not exist'); 174 | }); 175 | ``` 176 | 177 | See the options of `createWriteStream` for more information. 178 | 179 | ## accessing file metadata 180 | 181 | All file meta-data (file name, upload date, contentType, etc) are stored in a special mongodb collection separate from the actual file data. This collection can be queried directly: 182 | 183 | ```js 184 | var gfs = Grid(conn.db); 185 | gfs.files.find({ filename: 'myImage.png' }).toArray(function (err, files) { 186 | if (err) ... 187 | console.log(files); 188 | }) 189 | ``` 190 | 191 | Alternatively you can use the ```gfs.findOne```-shorthand to find a single file 192 | 193 | ```js 194 | gfs.findOne({ _id: '54da7b013706c1e7ab25f9fa'}, function (err, file) { 195 | console.log(file); 196 | }); 197 | ``` 198 | 199 | ## using with mongoose 200 | 201 | ```js 202 | var mongoose = require('mongoose'); 203 | var Grid = require('gridfs-stream'); 204 | 205 | var conn = mongoose.createConnection(..); 206 | conn.once('open', function () { 207 | var gfs = Grid(conn.db, mongoose.mongo); 208 | 209 | // all set! 210 | }) 211 | ``` 212 | 213 | You may optionally assign the driver directly to the `gridfs-stream` module so you don't need to pass it along each time you construct a grid: 214 | 215 | ```js 216 | var mongoose = require('mongoose'); 217 | var Grid = require('gridfs-stream'); 218 | Grid.mongo = mongoose.mongo; 219 | 220 | var conn = mongoose.createConnection(..); 221 | conn.once('open', function () { 222 | var gfs = Grid(conn.db); 223 | 224 | // all set! 225 | }) 226 | ``` 227 | 228 | [LICENSE](https://github.com/aheckmann/gridfs-stream/blob/master/LICENSE) 229 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | 2 | // fixture/logo.png 3 | var assert = require('assert') 4 | , Stream = require('stream') 5 | , fs = require('fs') 6 | , mongo = require('mongodb') 7 | , Grid = require('../') 8 | , crypto = require('crypto') 9 | , checksum = require('checksum') 10 | , tmpDir = __dirname + '/tmp/' 11 | , fixturesDir = __dirname + '/fixtures/' 12 | , imgReadPath = fixturesDir + 'mongo.png' 13 | , txtReadPath =fixturesDir + 'text.txt' 14 | , emptyReadPath = fixturesDir + 'emptydoc.txt' 15 | , largeBlobPath = tmpDir + '1mbBlob' 16 | , server 17 | , db 18 | 19 | 20 | describe('test', function(){ 21 | var id; 22 | before(function (done) { 23 | server = new mongo.Server('localhost', 27017); 24 | db = new mongo.Db('gridstream_test', server, {w:1}); 25 | if (!fs.existsSync(tmpDir)) { 26 | fs.mkdirSync(tmpDir); 27 | } 28 | fs.writeFile(largeBlobPath, crypto.randomBytes(1024*1024), function (err) { 29 | if (err) { 30 | done(err); 31 | } 32 | db.open(done) 33 | }); 34 | }); 35 | 36 | describe('Grid', function () { 37 | it('should be a function', function () { 38 | assert('function' == typeof Grid); 39 | }); 40 | it('should create instances without the new keyword', function () { 41 | var x = Grid(2,3); 42 | assert(x instanceof Grid); 43 | }); 44 | it('should store the arguments', function () { 45 | var x = new Grid(4, 5); 46 | assert.equal(x.db, 4); 47 | assert.equal(x.mongo, 5); 48 | }); 49 | it('should require mongo argument', function(){ 50 | assert.throws(function () { 51 | new Grid(3) 52 | }, /missing mongo argument/); 53 | }) 54 | it('should require db argument', function(){ 55 | assert.throws(function () { 56 | new Grid(null, 3) 57 | }, /missing db argument/); 58 | }) 59 | describe('files', function(){ 60 | it('returns a collection', function(){ 61 | var g = new Grid(db, mongo); 62 | assert(g.files instanceof mongo.Collection); 63 | }) 64 | }) 65 | describe('collection()', function(){ 66 | it('changes the files collection', function(){ 67 | var g = new Grid(db, mongo); 68 | assert.equal('function', typeof g.collection); 69 | assert(g.collection() instanceof mongo.Collection); 70 | assert.equal(g.collection(), g.files); 71 | var old = g.collection(); 72 | g.collection('changed'); 73 | assert(g.collection() instanceof mongo.Collection); 74 | assert.ok(g.collection() == g.files); 75 | assert.ok(g.collection() != old); 76 | assert.equal(g.collection(), g.files); 77 | assert.equal(g.collection().collectionName, 'changed.files'); 78 | }) 79 | }); 80 | }); 81 | 82 | describe('createWriteStream', function(){ 83 | it('should be a function', function () { 84 | var x = Grid(1, mongo); 85 | assert('function' == typeof x.createWriteStream); 86 | }); 87 | }) 88 | 89 | describe('GridWriteStream', function(){ 90 | var g 91 | , ws 92 | 93 | before(function(){ 94 | Grid.mongo = mongo; 95 | g = Grid(db); 96 | ws = g.createWriteStream({ filename: 'logo.png' }); 97 | }); 98 | 99 | it('should be an instance of Stream', function(){ 100 | assert(ws instanceof Stream); 101 | }) 102 | it('should be an instance of Stream.Writable', function(){ 103 | assert(ws instanceof Stream.Writable); 104 | }) 105 | it('should should be writable', function(){ 106 | assert(ws.writable); 107 | }); 108 | it('should store the grid', function(){ 109 | assert(ws._grid == g) 110 | }); 111 | it('should have an id', function(){ 112 | assert(ws.id) 113 | }) 114 | it('id should be an ObjectId', function(){ 115 | assert(ws.id instanceof mongo.ObjectID); 116 | }); 117 | it('should have a name', function(){ 118 | assert(ws.name == 'logo.png') 119 | }) 120 | describe('options', function(){ 121 | it('should have one key', function(){ 122 | assert(Object.keys(ws.options).length === 1); 123 | }); 124 | it('should have filename option', function(){ 125 | assert(ws.options.filename === 'logo.png'); 126 | }); 127 | }) 128 | it('mode should default to w', function(){ 129 | assert(ws.mode == 'w'); 130 | }) 131 | describe('store', function(){ 132 | it('should be an instance of mongo.GridStore', function(){ 133 | assert(ws._store instanceof mongo.GridStore) 134 | }) 135 | }) 136 | describe('#methods', function(){ 137 | describe('write', function(){ 138 | it('should be a function', function(){ 139 | assert('function' == typeof ws.write) 140 | }) 141 | }) 142 | describe('end', function(){ 143 | it('should be a function', function(){ 144 | assert('function' == typeof ws.end) 145 | }) 146 | }) 147 | describe('destroy', function(){ 148 | it('should be a function', function(){ 149 | assert('function' == typeof ws.destroy) 150 | }) 151 | }) 152 | }); 153 | it('should provide piping from a readableStream into GridFS', function(done){ 154 | var readStream = fs.createReadStream(imgReadPath, { bufferSize: 1024 }); 155 | var ws = g.createWriteStream({ filename: 'logo.png'}); 156 | 157 | // used in readable stream test 158 | id = ws.id; 159 | 160 | var progress = 0; 161 | var finished = false; 162 | var opened = false; 163 | var closed = false; 164 | var file; 165 | 166 | ws.on('progress', function (size) { 167 | progress = size; 168 | }); 169 | 170 | ws.on('open', function () { 171 | opened = true; 172 | }); 173 | 174 | ws.on('close', function (file_) { 175 | closed = true; 176 | file = file_; 177 | }); 178 | 179 | ws.on('finish', function () { 180 | assert(opened); 181 | assert(progress > 0); 182 | assert(closed); 183 | assert(file.filename === 'logo.png'); 184 | assert(file._id === id); 185 | assert(file.length === fs.readFileSync(imgReadPath).length); 186 | done(); 187 | }); 188 | 189 | var pipe = readStream.pipe(ws); 190 | }); 191 | it('should provide Error and File object on WriteStream close event', function(done){ 192 | var readStream = fs.createReadStream(imgReadPath, { bufferSize: 1024 }); 193 | var ws = g.createWriteStream({ 194 | mode: 'w', 195 | filename: 'closeEvent.png', 196 | content_type: "image/png" 197 | }); 198 | // used in readable stream test 199 | id = ws.id; 200 | 201 | var progress = 0; 202 | 203 | ws.on('progress', function (size) { 204 | progress = size; 205 | }); 206 | 207 | ws.on('close', function (file) { 208 | assert(file.filename === 'closeEvent.png') 209 | assert(file.contentType === 'image/png') 210 | assert(progress > 0); 211 | done(); 212 | }); 213 | var pipe = readStream.pipe(ws); 214 | }); 215 | 216 | 217 | //W+ not supported in new mongodb v2 gridstore driver 218 | it.skip('should pipe more data to an existing GridFS file', function(done){ 219 | function pipe (id, cb) { 220 | if (!cb) cb = id, id = null; 221 | var readStream = fs.createReadStream(txtReadPath); 222 | var ws = g.createWriteStream({ 223 | _id: id, 224 | mode: 'w+' }); 225 | ws.on('close', function () { 226 | cb(ws.id); 227 | }); 228 | readStream.pipe(ws); 229 | } 230 | 231 | pipe(function (id) { 232 | pipe(id, function (id) { 233 | // read the file out. it should consist of two copies of original 234 | mongo.GridStore.read(db, id, function (err, txt) { 235 | if (err) return done(err); 236 | assert.equal(txt.length, fs.readFileSync(txtReadPath).length*2); 237 | done(); 238 | }); 239 | }); 240 | }) 241 | }); 242 | 243 | it('should be able to store a 12-letter file name', function() { 244 | var ws = g.createWriteStream({ filename: '12345678.png' }); 245 | assert.equal(ws.name,'12345678.png'); 246 | }); 247 | 248 | it("shouldn't clobber filename when rewriting to an existing file by id", function(done){ 249 | var ws = g.createWriteStream({ 250 | mode: 'w', 251 | filename: 'filename.txt', 252 | content_type: 'text/plain' 253 | }); 254 | var rewrite_id = ws.id; 255 | ws.write("Some text\n"); 256 | ws.end(); 257 | 258 | ws.on('close', function () { 259 | // Rewrite the same file by _id 260 | var ws2 = g.createWriteStream({ 261 | _id: rewrite_id, 262 | mode: 'w' 263 | }); 264 | ws2.write("Some more text\n"); 265 | ws2.end(); 266 | 267 | ws2.on('close', function () { 268 | g.exist({ _id: rewrite_id }, function (err, result) { 269 | if (err) return done(err); 270 | assert.ok(result); 271 | g.exist({ filename: "filename.txt" }, function (err, result) { 272 | if (err) return done(err); 273 | assert.ok(result); 274 | done(); 275 | }); 276 | }); 277 | }); 278 | }); 279 | }); 280 | 281 | it('should be able to store an empty file', function(done){ 282 | var readStream = fs.createReadStream(emptyReadPath); 283 | var ws = g.createWriteStream({ 284 | mode: 'w', 285 | filename: 'closeEvent.txt', 286 | content_type: "text/plain" 287 | }); 288 | 289 | ws.on('close', function (file) { 290 | assert(file.filename === 'closeEvent.txt') 291 | assert(file.contentType === 'text/plain') 292 | done(); 293 | }); 294 | var pipe = readStream.pipe(ws); 295 | }); 296 | 297 | it('should create files with an _id of arbitrary type', function(done){ 298 | var readStream = fs.createReadStream(imgReadPath, { bufferSize: 1024 }); 299 | var ws = g.createWriteStream({ _id: 'an_arbitrary_id', filename: 'file.img'}); 300 | 301 | ws.on('close', function (file) { 302 | assert(file._id === 'an_arbitrary_id'); 303 | done(); 304 | }); 305 | 306 | var pipe = readStream.pipe(ws); 307 | }); 308 | 309 | it('should emit finish after the file exists', function(done){ 310 | var readStream = fs.createReadStream(imgReadPath); 311 | var ws = g.createWriteStream({ filename: 'logo.png'}); 312 | 313 | ws.on('finish', function () { 314 | var rs = g.createReadStream({_id: id}); 315 | var file = fixturesDir + 'byid.png'; 316 | var writeStream = fs.createWriteStream(file); 317 | 318 | rs.on('error', function (err) { 319 | // should not happen 320 | assert(false); 321 | }); 322 | 323 | writeStream.on('finish', function () { 324 | var buf1 = fs.readFileSync(imgReadPath); 325 | var buf2 = fs.readFileSync(file); 326 | 327 | assert(buf1.length === buf2.length); 328 | 329 | for (var i = 0, len = buf1.length; i < len; ++i) { 330 | assert(buf1[i] == buf2[i]); 331 | } 332 | 333 | fs.unlinkSync(file); 334 | done(); 335 | }); 336 | 337 | rs.pipe(writeStream); 338 | }); 339 | 340 | var pipe = readStream.pipe(ws); 341 | }); 342 | 343 | it('should emit one error on destroy()', function(done){ 344 | var readStream = fs.createReadStream(imgReadPath, { bufferSize: 1024 }); 345 | var ws = g.createWriteStream({ filename: 'logo.png'}); 346 | 347 | var error = new Error('test error from destroy'); 348 | var errorCounter = 0; 349 | 350 | ws.on('error', function (err) { 351 | errorCounter += 1; 352 | assert(errorCounter === 1); 353 | assert(err === error); 354 | done(); 355 | }); 356 | 357 | ws.on('progress', function (progress) { 358 | ws.destroy(error); 359 | ws.destroy(); // test multiple destroy call 360 | }); 361 | 362 | var pipe = readStream.pipe(ws); 363 | }); 364 | 365 | it('should emit error on destroy() on nextTick', function(done){ 366 | var ws = g.createWriteStream({ filename: 'logo.png'}); 367 | 368 | ws.destroy(new Error('early destroy')); 369 | 370 | ws.on('error', function (err) { 371 | done(); 372 | }); 373 | }); 374 | 375 | it('should emit close if open is emitted on destroy()', function(done){ 376 | var ws = g.createWriteStream({ filename: 'logo.png'}); 377 | 378 | var opened = false; 379 | var error = false; 380 | ws.on('open', function () { 381 | opened = true; 382 | }); 383 | ws.on('close', function () { 384 | assert(opened); 385 | assert(error); 386 | done(); 387 | }); 388 | 389 | ws.destroy(); 390 | 391 | ws.on('error', function (err) { 392 | error = true; 393 | }); 394 | }); 395 | 396 | it('should create GridWriteStream without options.', function(done){ 397 | var ws = g.createWriteStream(); 398 | 399 | ws.on('close', function () { 400 | done(); 401 | }); 402 | 403 | ws.destroy(); 404 | 405 | ws.on('error', function (err) { 406 | assert(!err); 407 | }); 408 | }); 409 | }); 410 | 411 | 412 | describe('createReadStream', function(){ 413 | it('should be a function', function () { 414 | var x = Grid(1); 415 | assert('function' == typeof x.createReadStream); 416 | }); 417 | }); 418 | 419 | describe('GridReadStream', function(){ 420 | var g 421 | , rs 422 | 423 | before(function(){ 424 | g = Grid(db); 425 | rs = g.createReadStream({ 426 | filename: 'logo.png' 427 | }); 428 | }); 429 | 430 | it('should create an instance of Stream', function(){ 431 | assert(rs instanceof Stream); 432 | }); 433 | it('should should be readable', function(){ 434 | assert(rs.readable); 435 | }); 436 | it('should store the grid', function(){ 437 | assert(rs._grid == g) 438 | }); 439 | it('should have a name', function(){ 440 | assert(rs.name == 'logo.png') 441 | }) 442 | it('should not have an id', function(){ 443 | assert.equal(rs.id, null) 444 | }) 445 | describe('options', function(){ 446 | it('should have no defaults', function(){ 447 | // NOTE: filename is required to avoid a throw here, because you can't create a valid 448 | // read stream for a non-existing file. 449 | assert(Object.keys(g.createReadStream({filename: 'logo.png'}).options).length === 1); 450 | }); 451 | }) 452 | it('mode should default to r', function(){ 453 | assert(rs.mode == 'r'); 454 | assert(rs._store.mode == 'r'); 455 | }) 456 | 457 | describe('store', function(){ 458 | it('should be an instance of mongo.GridStore', function(){ 459 | assert(rs._store instanceof mongo.GridStore) 460 | }) 461 | }) 462 | describe('#methods', function(){ 463 | describe('setEncoding', function(){ 464 | it('should be a function', function(){ 465 | assert('function' == typeof rs.setEncoding) 466 | // TODO test actual encodings 467 | }) 468 | }) 469 | describe('pause', function(){ 470 | it('should be a function', function(){ 471 | assert('function' == typeof rs.pause) 472 | }) 473 | }) 474 | describe('destroy', function(){ 475 | it('should be a function', function(){ 476 | assert('function' == typeof rs.destroy) 477 | }) 478 | }) 479 | describe('resume', function(){ 480 | it('should be a function', function(){ 481 | assert('function' == typeof rs.resume) 482 | }) 483 | }) 484 | describe('pipe', function(){ 485 | it('should be a function', function(){ 486 | assert('function' == typeof rs.pipe) 487 | }) 488 | }) 489 | }); 490 | it('should provide piping to a writable stream by name', function(done){ 491 | var file = fixturesDir + 'byname.png'; 492 | var rs = g.createReadStream({ 493 | filename: 'logo.png' 494 | }); 495 | var writeStream = fs.createWriteStream(file); 496 | 497 | var opened = false; 498 | var ended = false; 499 | 500 | rs.on('open', function () { 501 | opened = true; 502 | }); 503 | 504 | rs.on('error', function (err) { 505 | throw err; 506 | }); 507 | 508 | rs.on('end', function () { 509 | ended = true; 510 | }); 511 | 512 | writeStream.on('close', function () { 513 | // check they are identical 514 | var buf1 = fs.readFileSync(imgReadPath); 515 | var buf2 = fs.readFileSync(file); 516 | 517 | assert(buf1.length === buf2.length); 518 | 519 | for (var i = 0, len = buf1.length; i < len; ++i) { 520 | assert(buf1[i] == buf2[i]); 521 | } 522 | 523 | assert(opened); 524 | assert(ended); 525 | 526 | fs.unlinkSync(file); 527 | done(); 528 | }); 529 | 530 | rs.pipe(writeStream); 531 | }); 532 | 533 | it('should provide piping to a writable stream by id', function(done){ 534 | var file = fixturesDir + 'byid.png'; 535 | var rs = g.createReadStream({ 536 | _id: id 537 | }); 538 | var writeStream = fs.createWriteStream(file); 539 | assert(rs.id instanceof mongo.ObjectID); 540 | assert(rs.id == String(id)) 541 | 542 | var opened = false; 543 | var ended = false; 544 | 545 | rs.on('open', function () { 546 | opened = true; 547 | }); 548 | 549 | rs.on('error', function (err) { 550 | throw err; 551 | }); 552 | 553 | rs.on('end', function () { 554 | ended = true; 555 | }); 556 | 557 | writeStream.on('close', function () { 558 | //check they are identical 559 | assert(opened); 560 | assert(ended); 561 | 562 | var buf1 = fs.readFileSync(imgReadPath); 563 | var buf2 = fs.readFileSync(file); 564 | 565 | assert(buf1.length === buf2.length); 566 | 567 | for (var i = 0, len = buf1.length; i < len; ++i) { 568 | assert(buf1[i] == buf2[i]); 569 | } 570 | 571 | fs.unlinkSync(file); 572 | done(); 573 | }); 574 | 575 | rs.pipe(writeStream); 576 | }); 577 | 578 | it('should provide piping to a writable stream with a range by id', function(done){ 579 | var file = fixturesDir + 'byid.png'; 580 | var rs = g.createReadStream({ 581 | _id: id, 582 | range: { 583 | startPos: 1000, 584 | endPos: 10000 585 | } 586 | }); 587 | var writeStream = fs.createWriteStream(file); 588 | assert(rs.id instanceof mongo.ObjectID); 589 | assert(rs.id == String(id)) 590 | 591 | var opened = false; 592 | var ended = false; 593 | 594 | rs.on('open', function () { 595 | opened = true; 596 | }); 597 | 598 | rs.on('error', function (err) { 599 | throw err; 600 | }); 601 | 602 | rs.on('end', function () { 603 | ended = true; 604 | }); 605 | 606 | writeStream.on('close', function () { 607 | //check they are identical 608 | assert(opened); 609 | assert(ended); 610 | 611 | var buf1 = fs.readFileSync(imgReadPath); 612 | var buf2 = fs.readFileSync(file); 613 | 614 | assert(buf2.length === rs.options.range.endPos - rs.options.range.startPos + 1); 615 | 616 | for (var i = 0, len = buf2.length; i < len; ++i) { 617 | assert(buf1[i + rs.options.range.startPos] == buf2[i]); 618 | } 619 | 620 | fs.unlinkSync(file); 621 | done(); 622 | }); 623 | 624 | rs.pipe(writeStream); 625 | }); 626 | 627 | it('should read files with an _id of arbitrary type', function(done){ 628 | var rs = g.createReadStream({ _id: 'an_arbitrary_id'}); 629 | 630 | rs.on('open', function () { 631 | assert(rs.id === 'an_arbitrary_id'); 632 | done(); 633 | }); 634 | 635 | }); 636 | 637 | it('should allow checking for existence of files', function(done){ 638 | g.exist({ _id: id }, function (err, result) { 639 | if (err) return done(err); 640 | assert.ok(result); 641 | done(); 642 | }); 643 | }); 644 | 645 | it('should allow checking for non existence of files', function(done){ 646 | g.exist({ filename: 'does-not-exists.1234' }, function (err, result) { 647 | if (err) return done(err); 648 | assert.ok(!result); 649 | done(); 650 | }); 651 | }); 652 | 653 | // See #51 654 | it('should allow checking for existence of files in an alternate root collection', function(done){ 655 | var alternateFileOptions = {filename: 'alternateLogo.png', root: 'alternate' }; 656 | var readStream = g.createReadStream({filename: 'logo.png'}); 657 | var writeStream = g.createWriteStream(alternateFileOptions); 658 | readStream.pipe(writeStream); 659 | writeStream.on('close', function () { 660 | g.exist(alternateFileOptions, function (err, result) { 661 | if (err) return done(err); 662 | assert.ok(result); 663 | done(); 664 | }); 665 | }); 666 | }); 667 | 668 | it('should get a specific file', function(done){ 669 | g.findOne({ _id: id }, function(err, result) { 670 | if (err) return done(err); 671 | assert.ok(result); 672 | done(); 673 | }); 674 | }); 675 | 676 | // See #72 677 | it('should be able to find a file in an alternate root collection', function (done){ 678 | g.findOne({filename: 'alternateLogo.png', root: 'alternate' }, function (err, file) { 679 | assert.equal(err, null); 680 | done(); 681 | }); 682 | }); 683 | 684 | it('should allow removing files', function(done){ 685 | g.remove({ _id: id }, function (err) { 686 | if (err) return done(err); 687 | g.files.findOne({ _id: id }, function (err, doc) { 688 | if (err) return done(err); 689 | assert.ok(!doc); 690 | done(); 691 | }) 692 | }); 693 | }) 694 | 695 | it('should be possible to pause a stream after constructing it', function (done) { 696 | var rs = g.createReadStream({ filename: 'logo.png' }); 697 | rs.pause(); 698 | setTimeout(function () { 699 | rs.resume(); 700 | }, 1000); 701 | 702 | rs.on('data', function (data) { 703 | rs.destroy(); 704 | done(); 705 | }); 706 | }); 707 | 708 | //issue #46 709 | it('should be able handle multiple streams with multiple chunks', function (done) { 710 | var doneCounter = 0; 711 | var totalCounter = 100; 712 | var checksums = []; 713 | 714 | 715 | this.timeout(10000); 716 | function doTest (i) { 717 | var copyFileName = tmpDir + 'logo' + i + '.png'; 718 | var readStream = g.createReadStream({filename: '1mbBlob'}); 719 | var writeStream = fs.createWriteStream(copyFileName); 720 | readStream.pipe(writeStream); 721 | writeStream.on('close', function () { 722 | checksum.file(copyFileName, function (err, sum) { 723 | checksums.push(sum); 724 | fs.unlinkSync(copyFileName); 725 | if (++doneCounter == totalCounter) { 726 | assert(checksums.filter(function (value, index, self) { 727 | return self.indexOf(value) === index; 728 | }).length === 1); 729 | done(); 730 | } 731 | }); 732 | }); 733 | } 734 | 735 | var writeStream = g.createWriteStream({filename: '1mbBlob'}); 736 | fs.createReadStream(largeBlobPath).pipe(writeStream); 737 | writeStream.on('close', function() { 738 | for (var i = totalCounter; i-- > 0;) { 739 | doTest(i); 740 | } 741 | }); 742 | }); 743 | 744 | it('should be able to set the encoding of a readstream', function (done) { 745 | var rs = g.createReadStream({ filename: 'logo.png' }); 746 | rs.setEncoding('utf8'); 747 | 748 | rs.on('data', function (data) { 749 | assert.equal(typeof data, 'string'); 750 | rs.destroy(); 751 | done(); 752 | }); 753 | }); 754 | 755 | it('should be able to pause/resume after a chunk is sent to be able to throttle the stream', function (done) { 756 | var rs = g.createReadStream({ filename: '1mbBlob' }); 757 | var numChuksSent = 0 758 | 759 | // Pause stream after one chunk has been sent 760 | rs.on('data', function (data) { 761 | numChuksSent += 1; 762 | rs.pause(); 763 | }); 764 | 765 | // Only one chunk should have been sent because it was paused after that. 1mbBlob contains 5 with default gridstream chunk size 766 | setTimeout(function () { 767 | assert.equal( numChuksSent, 1 ); 768 | rs.resume(); 769 | }, 500); 770 | 771 | // Now there should be 2 772 | setTimeout(function () { 773 | assert.equal( numChuksSent, 2 ); 774 | done() 775 | }, 1000); 776 | 777 | }); 778 | }); 779 | 780 | after(function (done) { 781 | fs.unlink(largeBlobPath, function (err) { 782 | if (err) { 783 | done(err); 784 | } 785 | fs.rmdir(tmpDir, function () { 786 | db.dropDatabase(function () { 787 | db.close(true, done); 788 | }); 789 | }); 790 | }); 791 | }); 792 | }); 793 | --------------------------------------------------------------------------------