├── .gitignore
├── .travis.yml
├── Makefile
├── README.md
├── index.js
├── package.json
└── test
    ├── all_npm.json
    └── index.test.js


/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | node_modules/*
3 | *.swp
4 | *.swo
5 | npm-debug.log
6 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: node_js
2 | node_js:
3 |   - "0.10"
4 | before_script:
5 |   - npm install
6 | notifications:
7 |   email: false
8 | services: mongodb
9 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | test:
2 | 	@echo "StreamToMongo - Launching tests."
3 | 	@ NODE_ENV="test" node_modules/.bin/mocha --reporter spec -t 10000
4 | 	@echo "Tests finished"
5 | 
6 | .PHONY: test
7 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # StreamToMongo
 2 | Stream objects straight into a MongoDB database.
 3 | 
 4 | [![Build
 5 | Status](https://travis-ci.org/czzarr/node-stream-to-mongo.png)](https://travis-ci.org/czzarr/node-stream-to-mongo)
 6 | 
 7 | ## Install
 8 | `npm install stream-to-mongo`
 9 | 
10 | ## Main usecase 
11 | Seeding a database from large HTTP response or JSON file.
12 | 
13 | ## Example
14 | ```javascript
15 | var request = require('request')
16 | var parser = require('JSONStream').parse('rows.*.doc')
17 | var options = { db: 'mongodb://localhost:27017/test-stream', collection: 'docs' }
18 | var streamToMongo = require('stream-to-mongo')(options);
19 | 
20 | request('http://isaacs.couchone.com/registry/_all_docs')
21 |   .pipe(parser)
22 |   .pipe(streamToMongo);
23 | ```
24 | ## Test
25 | Needs a MongoDB instance running.
26 | `make test`
27 | 
28 | ## License
29 | 
30 | MIT
31 | 


--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
 1 | var Writable = require('stream').Writable;
 2 | var util = require('util');
 3 | var MongoClient = require('mongodb').MongoClient;
 4 | 
 5 | util.inherits(StreamToMongo, Writable);
 6 | 
 7 | module.exports = StreamToMongo;
 8 | 
 9 | function StreamToMongo(options) {
10 |   if(!(this instanceof StreamToMongo)) {
11 |     return new StreamToMongo(options);
12 |   }
13 |   Writable.call(this, { objectMode: true });
14 |   this.options = options;
15 | }
16 | 
17 | 
18 | StreamToMongo.prototype._write = function (obj, encoding, done) {
19 |   var self = this;
20 | 
21 |   // Custom action definition
22 |   var action = this.options.action || function insert (obj, cb) {
23 |     this.collection.insert(obj, {w: 1}, cb);
24 |   };
25 | 
26 |   if (!this.db) {
27 |     MongoClient.connect(this.options.db, function (err, db) {
28 |       if (err) throw err;
29 |       self.db = db;
30 |       self.on('finish', function () {
31 |         self.db.close();
32 |       });
33 |       self.collection = db.collection(self.options.collection);
34 |       action.call(self, obj, done);
35 |     });
36 |   } else {
37 |     action.call(self, obj, done);
38 |   }
39 | };
40 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "stream-to-mongo",
 3 |   "version": "0.1.5",
 4 |   "description": "Write a stream of objects to a Mongo database",
 5 |   "main": "index.js",
 6 |   "scripts": {
 7 |     "test": "make test"
 8 |   },
 9 |   "keywords": [
10 |     "mongo",
11 |     "write",
12 |     "stream"
13 |   ],
14 |   "author": "Stanislas Marion",
15 |   "license": "MIT",
16 |   "dependencies": {
17 |     "mongodb": "2"
18 |   },
19 |   "devDependencies": {
20 |     "mocha": "~1.12.0",
21 |     "chai": "~1.7.2",
22 |     "JSONStream": "~0.6.4"
23 |   },
24 |   "directories": {
25 |     "test": "test"
26 |   },
27 |   "repository": {
28 |     "type": "git",
29 |     "url": "git://github.com/czzarr/node-stream-to-mongo.git"
30 |   },
31 |   "bugs": {
32 |     "url": "https://github.com/czzarr/node-stream-to-mongo/issues"
33 |   }
34 | }
35 | 


--------------------------------------------------------------------------------
/test/index.test.js:
--------------------------------------------------------------------------------
 1 | var fs = require('fs');
 2 | var chai = require('chai');
 3 | var MongoClient = require('mongodb').MongoClient;
 4 | var StreamToMongo = require('../');
 5 | var parser = require('JSONStream').parse('rows.*.id')
 6 | var options = { db: 'mongodb://localhost:27017/test-stream', collection: 'docs' }
 7 | var collection = options.collection;
 8 | var streamToMongo = new StreamToMongo(options);
 9 | 
10 | chai.should();
11 | 
12 | describe('Stream to Mongo', function () {
13 |   it('should insert docs into Mongo', function (done) {
14 |     fs.createReadStream('test/all_npm.json')
15 |       .pipe(parser)
16 |       .pipe(streamToMongo);
17 |     streamToMongo.on('finish', function() {
18 |       MongoClient.connect(options.db, function (err, db) {
19 |         db.collection(options.collection).count(function (err, count) {
20 |           count.should.equal(4028);
21 |           done();
22 |         });
23 |       });
24 |     });
25 |   });
26 | });
27 | 
28 | after(function(done) {
29 |   MongoClient.connect(options.db, function (err, db) {
30 |     db.collection(collection).drop(done);
31 |   });
32 | });
33 | 


--------------------------------------------------------------------------------