├── .gitignore ├── README.md ├── index.js ├── lib ├── parser.js └── stringifier.js ├── package.json └── test ├── parser.js └── stringifier.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JSONLines for Node.js 2 | 3 | Parse [JSONLines](http://jsonlines.org) with Node.js. 4 | 5 | ## Installation 6 | 7 | ```sh 8 | npm install --save jsonlines 9 | ``` 10 | 11 | ## Usage 12 | 13 | ```javascript 14 | var jsonlines = require('jsonlines') 15 | var parser = jsonlines.parse() 16 | 17 | parser.on('data', function (data) { 18 | console.log('Got json:', data) 19 | }) 20 | 21 | parser.on('end', function () { 22 | console.log('No more data') 23 | }) 24 | 25 | parser.write('{ "test": "This is a test!" }\n') 26 | parser.write('{ "jsonlines": "is awesome" }') 27 | parser.end() 28 | ``` 29 | 30 | ```javascript 31 | var jsonlines = require('jsonlines') 32 | var stringifier = jsonlines.stringify() 33 | 34 | stringifier.pipe(process.stdout) 35 | 36 | stringifier.write({ test: 'This is a test!' }) 37 | stringifier.write({ jsonlines: 'is awesome' }) 38 | stringifier.end() 39 | ``` 40 | 41 | ## API 42 | 43 | ### `.parse([options])` 44 | 45 | Returns a transform stream that turns newline separated json into a stream of 46 | javascript values. 47 | 48 | `options` is an optional object with the keys documented below. 49 | 50 | ### `.stringify()` 51 | 52 | Returns a transform stream that turns javascript values into a stream of newline 53 | separated json. 54 | 55 | ## Options 56 | 57 | ### `emitInvalidLine` 58 | 59 | If true, instead of emitting an error and cancelling the stream when an invalid line is proccessed, an `invalid-line` event is emitted with the same error. This is very useful when processing text that have mixed plain text and json data. 60 | 61 | Example: 62 | 63 | ```js 64 | var jsonlines = require('jsonlines') 65 | var parser = jsonlines.parse({ emitInvalidLines: true }) 66 | 67 | parser.on('data', function (data) { 68 | console.log('Got json:', data) 69 | }) 70 | 71 | parser.on('invalid-line', function (err) { 72 | console.log('Got text:', err.source) 73 | }) 74 | 75 | parser.write('{ "test": "This is a test!" }\n') 76 | parser.write('This is some plain text\n') 77 | parser.write('{ "jsonlines": "is awesome" }') 78 | parser.end() 79 | ``` 80 | 81 | Output: 82 | 83 | ```text 84 | Got json: { test: 'This is a test!' } 85 | Got text: This is some plain text 86 | Got json: { jsonlines: 'is awesome' } 87 | ``` 88 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var Parser = require('./lib/parser') 2 | var Stringifier = require('./lib/stringifier') 3 | 4 | exports.parse = function parse (options) { 5 | return new Parser(options) 6 | } 7 | 8 | exports.stringify = function stringify () { 9 | return new Stringifier() 10 | } 11 | -------------------------------------------------------------------------------- /lib/parser.js: -------------------------------------------------------------------------------- 1 | var Transform = require('stream').Transform 2 | 3 | function Parser (options) { 4 | if (!(this instanceof Parser)) { 5 | throw new TypeError('Cannot call a class as a function') 6 | } 7 | 8 | options = options || {} 9 | 10 | Transform.call(this, { objectMode: true }) 11 | this._memory = '' 12 | this._emitInvalidLines = (options.emitInvalidLines || false) 13 | } 14 | 15 | Parser.prototype = Object.create(Transform.prototype) 16 | 17 | Parser.prototype._handleLines = function (lines, cb) { 18 | for (var i = 0; i < lines.length; i++) { 19 | if (lines[i] === '') continue 20 | 21 | var err = null 22 | var json = null 23 | try { 24 | json = JSON.parse(lines[i]) 25 | } catch (_err) { 26 | _err.source = lines[i] 27 | err = _err 28 | } 29 | 30 | if (err) { 31 | if (this._emitInvalidLines) { 32 | this.emit('invalid-line', err) 33 | } else { 34 | return cb(err) 35 | } 36 | } else { 37 | this.push(json) 38 | } 39 | } 40 | 41 | cb(null) 42 | } 43 | 44 | Parser.prototype._transform = function (chunk, encoding, cb) { 45 | var lines = (this._memory + chunk.toString()).split('\n') 46 | 47 | this._memory = lines.pop() 48 | this._handleLines(lines, cb) 49 | } 50 | 51 | Parser.prototype._flush = function (cb) { 52 | if (!this._memory) return cb(null) 53 | 54 | var line = this._memory 55 | 56 | this._memory = '' 57 | this._handleLines([ line ], cb) 58 | } 59 | 60 | module.exports = Parser 61 | -------------------------------------------------------------------------------- /lib/stringifier.js: -------------------------------------------------------------------------------- 1 | var Transform = require('stream').Transform 2 | 3 | function Stringifier () { 4 | if (!(this instanceof Stringifier)) { 5 | throw new TypeError('Cannot call a class as a function') 6 | } 7 | 8 | Transform.call(this, { objectMode: true }) 9 | } 10 | 11 | Stringifier.prototype = Object.create(Transform.prototype) 12 | 13 | Stringifier.prototype._transform = function (data, _, cb) { 14 | var value 15 | 16 | try { 17 | value = JSON.stringify(data) 18 | } catch (err) { 19 | err.source = data 20 | return cb(err) 21 | } 22 | 23 | cb(null, value + '\n') 24 | } 25 | 26 | module.exports = Stringifier 27 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jsonlines", 3 | "version": "0.1.1", 4 | "license": "MIT", 5 | "author": "Linus Unnebäck ", 6 | "main": "index.js", 7 | "repository": { 8 | "type": "git", 9 | "url": "http://github.com/LinusU/node-jsonlines.git" 10 | }, 11 | "scripts": { 12 | "test": "standard && mocha" 13 | }, 14 | "devDependencies": { 15 | "concat-stream": "^1.5.0", 16 | "mocha": "^2.2.5", 17 | "standard": "^5.0.2" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /test/parser.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | var jsonlines = require('../') 4 | 5 | var assert = require('assert') 6 | var concat = require('concat-stream') 7 | 8 | describe('Parser', function () { 9 | it('should handle simple jsonlines', function (done) { 10 | var parser = jsonlines.parse() 11 | var expected = [ 12 | { a: 453345, b: 23423 }, 13 | { c: 843222, d: 19534 }, 14 | { e: 656564, f: 76521 } 15 | ] 16 | 17 | parser.pipe(concat({ encoding: 'object' }, function (result) { 18 | assert.deepEqual(result, expected) 19 | done() 20 | })) 21 | 22 | for (var line of expected) { 23 | parser.write(JSON.stringify(line) + '\n') 24 | } 25 | 26 | parser.end() 27 | }) 28 | 29 | it('should handle strange chunks', function (done) { 30 | var parser = jsonlines.parse() 31 | var expected = [ 32 | { test: 'This is a test!' }, 33 | { key: 'value' } 34 | ] 35 | 36 | parser.pipe(concat({ encoding: 'object' }, function (result) { 37 | assert.deepEqual(result, expected) 38 | done() 39 | })) 40 | 41 | parser.write('{ "tes') 42 | parser.write('t": "This is a test!" ') 43 | parser.write('}\n{ "key": "value" }') 44 | parser.write('\n\n\n') 45 | 46 | parser.end() 47 | }) 48 | 49 | it('should ignore empty lines', function (done) { 50 | var parser = jsonlines.parse() 51 | var expected = [ 52 | { first: true, last: false }, 53 | { first: false, last: true } 54 | ] 55 | 56 | parser.pipe(concat({ encoding: 'object' }, function (result) { 57 | assert.deepEqual(result, expected) 58 | done() 59 | })) 60 | 61 | for (var line of expected) { 62 | parser.write(JSON.stringify(line) + '\n\n\n\n\n') 63 | } 64 | 65 | parser.end() 66 | }) 67 | 68 | it('should handle plain values', function (done) { 69 | var parser = jsonlines.parse() 70 | var expected = [ true, false, true, true, false ] 71 | 72 | parser.pipe(concat({ encoding: 'object' }, function (result) { 73 | assert.deepEqual(result, expected) 74 | done() 75 | })) 76 | 77 | for (var line of expected) { 78 | parser.write(JSON.stringify(line) + '\n') 79 | } 80 | 81 | parser.end() 82 | }) 83 | 84 | it('should emit an error on malformed json', function (done) { 85 | var parser = jsonlines.parse() 86 | var line = '{ "broken": _ }' 87 | 88 | parser.on('error', function (err) { 89 | assert.equal(err.source, line) 90 | done() 91 | }) 92 | 93 | parser.end(line) 94 | }) 95 | 96 | it('should emit invalid lines', function (done) { 97 | var parser = jsonlines.parse({ emitInvalidLines: true }) 98 | var data = '"works"\nbroken\n"ok"' 99 | 100 | parser.once('data', function (data) { 101 | assert.equal(data, 'works') 102 | 103 | parser.once('invalid-line', function (err) { 104 | assert.equal(err.source, 'broken') 105 | 106 | parser.once('data', function (data) { 107 | assert.equal(data, 'ok') 108 | done() 109 | }) 110 | }) 111 | }) 112 | 113 | parser.end(data) 114 | }) 115 | }) 116 | -------------------------------------------------------------------------------- /test/stringifier.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | var jsonlines = require('../') 4 | 5 | var assert = require('assert') 6 | var concat = require('concat-stream') 7 | 8 | describe('Stringifier', function () { 9 | it('should handle simple jsonlines', function (done) { 10 | var stringifier = jsonlines.stringify() 11 | var source = [ 12 | { a: 453345, b: 23423 }, 13 | { c: 843222, d: 19534 }, 14 | { e: 656564, f: 76521 } 15 | ] 16 | var expected = source.map(function (obj) { 17 | return JSON.stringify(obj) 18 | }).join('\n') 19 | 20 | stringifier.pipe(concat({ encoding: 'buffer' }, function (result) { 21 | assert.deepEqual(result.toString().trim(), expected) 22 | done() 23 | })) 24 | 25 | for (var line of source) { 26 | stringifier.write(line) 27 | } 28 | 29 | stringifier.end() 30 | }) 31 | 32 | it('should handle plain values', function (done) { 33 | var stringifier = jsonlines.stringify() 34 | var source = [ true, false, true, true, false ] 35 | var expected = 'true\nfalse\ntrue\ntrue\nfalse' 36 | 37 | stringifier.pipe(concat({ encoding: 'buffer' }, function (result) { 38 | assert.deepEqual(result.toString().trim(), expected) 39 | done() 40 | })) 41 | 42 | for (var line of source) { 43 | stringifier.write(line) 44 | } 45 | 46 | stringifier.end() 47 | }) 48 | 49 | it('should emit an error on bad values', function (done) { 50 | var stringifier = jsonlines.stringify() 51 | var broken = { a: 1 } 52 | 53 | // Make the value bad 54 | broken.self = broken 55 | 56 | stringifier.on('error', function (err) { 57 | assert.equal(err.source, broken) 58 | done() 59 | }) 60 | 61 | stringifier.end(broken) 62 | }) 63 | }) 64 | --------------------------------------------------------------------------------