├── .gitignore ├── .travis.yml ├── Makefile ├── README.md ├── index.js ├── package.json └── test ├── all_npm.json └── index.test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | node_modules/* 3 | *.swp 4 | *.swo 5 | npm-debug.log 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" 4 | before_script: 5 | - npm install 6 | notifications: 7 | email: false 8 | services: mongodb 9 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | test: 2 | @echo "StreamToMongo - Launching tests." 3 | @ NODE_ENV="test" node_modules/.bin/mocha --reporter spec -t 10000 4 | @echo "Tests finished" 5 | 6 | .PHONY: test 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # StreamToMongo 2 | Stream objects straight into a MongoDB database. 3 | 4 | [![Build 5 | Status](https://travis-ci.org/czzarr/node-stream-to-mongo.png)](https://travis-ci.org/czzarr/node-stream-to-mongo) 6 | 7 | ## Install 8 | `npm install stream-to-mongo` 9 | 10 | ## Main usecase 11 | Seeding a database from large HTTP response or JSON file. 12 | 13 | ## Example 14 | ```javascript 15 | var request = require('request') 16 | var parser = require('JSONStream').parse('rows.*.doc') 17 | var options = { db: 'mongodb://localhost:27017/test-stream', collection: 'docs' } 18 | var streamToMongo = require('stream-to-mongo')(options); 19 | 20 | request('http://isaacs.couchone.com/registry/_all_docs') 21 | .pipe(parser) 22 | .pipe(streamToMongo); 23 | ``` 24 | ## Test 25 | Needs a MongoDB instance running. 26 | `make test` 27 | 28 | ## License 29 | 30 | MIT 31 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var Writable = require('stream').Writable; 2 | var util = require('util'); 3 | var MongoClient = require('mongodb').MongoClient; 4 | 5 | util.inherits(StreamToMongo, Writable); 6 | 7 | module.exports = StreamToMongo; 8 | 9 | function StreamToMongo(options) { 10 | if(!(this instanceof StreamToMongo)) { 11 | return new StreamToMongo(options); 12 | } 13 | Writable.call(this, { objectMode: true }); 14 | this.options = options; 15 | } 16 | 17 | 18 | StreamToMongo.prototype._write = function (obj, encoding, done) { 19 | var self = this; 20 | 21 | // Custom action definition 22 | var action = this.options.action || function insert (obj, cb) { 23 | this.collection.insert(obj, {w: 1}, cb); 24 | }; 25 | 26 | if (!this.db) { 27 | MongoClient.connect(this.options.db, function (err, db) { 28 | if (err) throw err; 29 | self.db = db; 30 | self.on('finish', function () { 31 | self.db.close(); 32 | }); 33 | self.collection = db.collection(self.options.collection); 34 | action.call(self, obj, done); 35 | }); 36 | } else { 37 | action.call(self, obj, done); 38 | } 39 | }; 40 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "stream-to-mongo", 3 | "version": "0.1.5", 4 | "description": "Write a stream of objects to a Mongo database", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "make test" 8 | }, 9 | "keywords": [ 10 | "mongo", 11 | "write", 12 | "stream" 13 | ], 14 | "author": "Stanislas Marion", 15 | "license": "MIT", 16 | "dependencies": { 17 | "mongodb": "2" 18 | }, 19 | "devDependencies": { 20 | "mocha": "~1.12.0", 21 | "chai": "~1.7.2", 22 | "JSONStream": "~0.6.4" 23 | }, 24 | "directories": { 25 | "test": "test" 26 | }, 27 | "repository": { 28 | "type": "git", 29 | "url": "git://github.com/czzarr/node-stream-to-mongo.git" 30 | }, 31 | "bugs": { 32 | "url": "https://github.com/czzarr/node-stream-to-mongo/issues" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /test/index.test.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'); 2 | var chai = require('chai'); 3 | var MongoClient = require('mongodb').MongoClient; 4 | var StreamToMongo = require('../'); 5 | var parser = require('JSONStream').parse('rows.*.id') 6 | var options = { db: 'mongodb://localhost:27017/test-stream', collection: 'docs' } 7 | var collection = options.collection; 8 | var streamToMongo = new StreamToMongo(options); 9 | 10 | chai.should(); 11 | 12 | describe('Stream to Mongo', function () { 13 | it('should insert docs into Mongo', function (done) { 14 | fs.createReadStream('test/all_npm.json') 15 | .pipe(parser) 16 | .pipe(streamToMongo); 17 | streamToMongo.on('finish', function() { 18 | MongoClient.connect(options.db, function (err, db) { 19 | db.collection(options.collection).count(function (err, count) { 20 | count.should.equal(4028); 21 | done(); 22 | }); 23 | }); 24 | }); 25 | }); 26 | }); 27 | 28 | after(function(done) { 29 | MongoClient.connect(options.db, function (err, db) { 30 | db.collection(collection).drop(done); 31 | }); 32 | }); 33 | --------------------------------------------------------------------------------