├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── lib └── json-stream.js ├── package.json └── test ├── json-stream-test.js ├── pipe-test.js └── throw-in-readable-test.js /.gitignore: -------------------------------------------------------------------------------- 1 | npm-debug.log 2 | node_modules 3 | .DS_Store 4 | .*.sw[op] 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" 4 | - "0.11" 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2011 by Maciej Małecki 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # json-stream [![Build Status](https://secure.travis-ci.org/mmalecki/json-stream.png?branch=master)](http://travis-ci.org/mmalecki/json-stream) 2 | New line-delimeted JSON parser with a stream interface. 3 | 4 | ## Installation 5 | 6 | npm install json-stream 7 | 8 | ## Usage 9 | ```js 10 | var JSONStream = require('json-stream'); 11 | 12 | var stream = JSONStream(); 13 | 14 | stream.on('data', function (chunk) { 15 | console.dir(chunk); 16 | }); 17 | stream.write('{"a":'); 18 | stream.write('42}\n'); 19 | stream.write('{"hel'); 20 | stream.write('lo": "world"}\n'); 21 | ``` 22 | 23 | Will output: 24 | ``` 25 | { a: 42 } 26 | { hello: 'world' } 27 | ``` 28 | 29 | If invalid JSON gets written, it's silently ignored. 30 | -------------------------------------------------------------------------------- /lib/json-stream.js: -------------------------------------------------------------------------------- 1 | var util = require('util'), 2 | TransformStream = require('stream').Transform; 3 | 4 | module.exports = function (options) { 5 | return new JSONStream(options); 6 | }; 7 | 8 | var JSONStream = module.exports.JSONStream = function (options) { 9 | options = options || {}; 10 | TransformStream.call(this, options); 11 | this._writableState.objectMode = false; 12 | this._readableState.objectMode = true; 13 | this._async = options.async || false; 14 | }; 15 | util.inherits(JSONStream, TransformStream); 16 | 17 | JSONStream.prototype._transform = function (data, encoding, callback) { 18 | if (!Buffer.isBuffer(data)) data = new Buffer(data); 19 | if (this._buffer) { 20 | data = Buffer.concat([this._buffer, data]); 21 | } 22 | 23 | var ptr = 0, start = 0; 24 | while (++ptr <= data.length) { 25 | if (data[ptr] === 10 || ptr === data.length) { 26 | var line; 27 | try { 28 | line = JSON.parse(data.slice(start, ptr)); 29 | } 30 | catch (ex) { } 31 | if (line) { 32 | this.push(line); 33 | line = null; 34 | } 35 | if (data[ptr] === 10) start = ++ptr; 36 | } 37 | } 38 | 39 | this._buffer = data.slice(start); 40 | return this._async 41 | ? void setImmediate(callback) 42 | : void callback(); 43 | }; 44 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "json-stream", 3 | "version": "1.0.0", 4 | "description": "New line-delimeted JSON parser with a stream interface", 5 | "author": "Maciej Małecki ", 6 | "main": "./lib/json-stream", 7 | "scripts": { 8 | "test": "node test/pipe-test.js && node test/json-stream-test.js && node test/throw-in-readable-test.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "https://github.com/mmalecki/json-stream.git" 13 | }, 14 | "license": "MIT" 15 | } 16 | -------------------------------------------------------------------------------- /test/json-stream-test.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'), 2 | JSONStream = require('../'); 3 | 4 | function write(stream) { 5 | var writes = []; 6 | for (var i = 1; i < arguments.length; i++) { 7 | writes[i - 1] = arguments[i]; 8 | } 9 | writes.forEach(function (write) { 10 | stream.write(write); 11 | }); 12 | stream.end(); 13 | } 14 | 15 | function expect(stream, events) { 16 | var chunks = [], endCalled = false; 17 | stream.on('readable', function () { 18 | var chunk = stream.read(); 19 | if (chunk) { 20 | chunks.push(chunk); 21 | } 22 | }); 23 | stream.on('end', function () { 24 | endCalled = true; 25 | }); 26 | process.on('exit', function () { 27 | assert.deepEqual(chunks, events); 28 | assert(endCalled); 29 | }); 30 | } 31 | 32 | var stream = JSONStream(); 33 | expect(stream, [ { a: 42 } ]); 34 | write(stream, '{"a": 42}\n'); 35 | 36 | stream = JSONStream(); 37 | expect(stream, [ { a: 42 } ]); 38 | write(stream, '{"a":', '42}\n'); 39 | 40 | stream = JSONStream(); 41 | expect(stream, [ { a: 42, b: 1337 } ]); 42 | write(stream, '{"a":', '42', ',"b": 1337', '}\n'); 43 | 44 | stream = JSONStream(); 45 | expect(stream, [ { a: 42, b: 1337 }, { hello: 'world' } ]); 46 | write(stream, '{"a":', '42', ',"b": 1337', '}\n{"hel', 'lo": "wor', 'ld"}\n'); 47 | 48 | stream = JSONStream(); 49 | expect(stream, [ { a: 42 }, { hello: 'world' } ]); 50 | write(stream, '{"a":', '42}\n{ blah blah blah }\n{"hel', 'lo": "wor', 'ld"}\n'); 51 | 52 | stream = JSONStream(); 53 | expect(stream, [ { a: 42 }, { hello: 'world' } ]); 54 | write(stream, '{"a":', '42}\n{ blah blah', 'blah }\n{"hel', 'lo": "wor', 'ld"}\n'); 55 | 56 | stream = JSONStream(); 57 | expect(stream, [ { å: '⇢ utf8!', b: 1337 } ]); 58 | write(stream, '{"å": "⇢ utf8!", "b": 1337 }\n'); 59 | 60 | stream = JSONStream({ async: true }); 61 | expect(stream, [ { å: '⇢ utf8!', b: 1337 } ]); 62 | write(stream, '{"å": "⇢ utf8!", "b": 1337 }\n'); 63 | -------------------------------------------------------------------------------- /test/pipe-test.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'), 2 | ReadableStream = require('stream').Readable, 3 | JSONStream = require('../'); 4 | 5 | var source = new ReadableStream(), 6 | dest = JSONStream(), 7 | chunks = [], 8 | endCalled = false; 9 | 10 | var wantedChunks = [ 11 | { a: 42 }, 12 | { hello: 'world' } 13 | ]; 14 | 15 | dest.on('readable', function () { 16 | var chunk = dest.read(); 17 | if (chunk) { 18 | chunks.push(chunk); 19 | } 20 | }); 21 | 22 | dest.on('end', function () { 23 | endCalled = true; 24 | }); 25 | 26 | process.on('exit', function () { 27 | assert.deepEqual(chunks, wantedChunks); 28 | assert(endCalled); 29 | }); 30 | 31 | var source = new ReadableStream(); 32 | source._read = function () { 33 | }; 34 | source.pipe(dest); 35 | source.push('{"a": 4'); 36 | source.push('2}\nblah'); 37 | source.push('\n{"hello"'); 38 | source.push(': "world"}\n'); 39 | source.push(null); 40 | -------------------------------------------------------------------------------- /test/throw-in-readable-test.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'), 2 | JSONStream = require('../'); 3 | 4 | var stream = JSONStream(); 5 | 6 | stream.on('readable', function () { 7 | throw new Error('This should crash and burn.'); 8 | }); 9 | 10 | assert.throws(function () { 11 | stream.write('{"a":"b"}\n'); 12 | }, 'write with a throwing `readable` handler should throw'); 13 | --------------------------------------------------------------------------------