├── .gitignore
├── collaborators.md
├── cli.js
├── index.js
├── package.json
├── readme.md
└── test.js
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
--------------------------------------------------------------------------------
/collaborators.md:
--------------------------------------------------------------------------------
1 | ## Collaborators
2 |
3 | ldjson-stream is only possible due to the excellent work of the following collaborators:
4 |
5 |
8 |
--------------------------------------------------------------------------------
/cli.js:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 |
3 | var fs = require('fs')
4 | var ndjson = require('./index.js')
5 | var minimist = require('minimist')
6 |
7 | var args = minimist(process.argv.slice(2))
8 |
9 | var inputStream
10 |
11 | var first = args._[0]
12 | if (!first) {
13 | console.error('Usage: ndjson [input] ')
14 | process.exit(1)
15 | }
16 |
17 | if (first === '-') inputStream = process.stdin
18 | else inputStream = fs.createReadStream(first)
19 |
20 | var parse = ndjson.parse(args)
21 | var serializer = ndjson.serialize(args)
22 |
23 | inputStream.pipe(parse).pipe(serializer).pipe(process.stdout)
24 |
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | var through = require('through2')
2 | var split = require('split2')
3 | var EOL = require('os').EOL
4 | var stringify = require('json-stringify-safe')
5 |
6 | module.exports = parse
7 | module.exports.serialize = module.exports.stringify = serialize
8 | module.exports.parse = parse
9 |
10 | function parse (opts) {
11 | opts = opts || {}
12 | opts.strict = opts.strict !== false
13 |
14 | function parseRow (row) {
15 | try {
16 | if (row) return JSON.parse(row)
17 | } catch (e) {
18 | if (opts.strict) {
19 | this.emit('error', new Error('Could not parse row ' + row.slice(0, 50) + '...'))
20 | }
21 | }
22 | }
23 |
24 | return split(parseRow, opts)
25 | }
26 |
27 | function serialize (opts) {
28 | return through.obj(opts, function(obj, enc, cb) {
29 | cb(null, stringify(obj) + EOL)
30 | })
31 | }
32 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "ndjson",
3 | "version": "1.5.0",
4 | "description": "streaming newline delimited json parser + serializer",
5 | "main": "index.js",
6 | "scripts": {
7 | "test": "tape test.js"
8 | },
9 | "bin": {
10 | "ndjson": "cli.js"
11 | },
12 | "author": "max ogden",
13 | "license": "BSD-3-Clause",
14 | "dependencies": {
15 | "json-stringify-safe": "^5.0.1",
16 | "minimist": "^1.2.0",
17 | "split2": "^2.1.0",
18 | "through2": "^2.0.3"
19 | },
20 | "devDependencies": {
21 | "concat-stream": "^1.5.0",
22 | "tape": "^4.6.3"
23 | },
24 | "repository": {
25 | "type": "git",
26 | "url": "git://github.com/maxogden/ndjson.git"
27 | },
28 | "bugs": {
29 | "url": "https://github.com/maxogden/ndjson/issues"
30 | },
31 | "homepage": "https://github.com/maxogden/ndjson",
32 | "keywords": [
33 | "ndjson",
34 | "ldjson"
35 | ]
36 | }
37 |
--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | # Development on this npm package is moved to https://github.com/ndjson/ndjson.js
2 |
3 |
4 | # ndjson
5 |
6 | streaming [newline delimited json](https://en.wikipedia.org/wiki/Line_Delimited_JSON) parser + serializer. Available as a JS API or a command line tool
7 |
8 | [](https://nodei.co/npm/ndjson/)
9 |
10 | ## usage
11 |
12 | ```
13 | var ndjson = require('ndjson')
14 | ```
15 |
16 | #### ndjson.parse(opts)
17 |
18 | returns a transform stream that accepts newline delimited json and emits objects
19 |
20 | example newline delimited json:
21 |
22 | `data.txt`:
23 |
24 | ```
25 | {"foo": "bar"}
26 | {"hello": "world"}
27 | ```
28 |
29 | If you want to discard non-valid JSON messages, you can call `ndjson.parse({strict: false})`
30 |
31 | usage:
32 |
33 | ```js
34 | fs.createReadStream('data.txt')
35 | .pipe(ndjson.parse())
36 | .on('data', function(obj) {
37 | // obj is a javascript object
38 | })
39 | ```
40 |
41 | #### ndjson.serialize() / ndjson.stringify()
42 |
43 | returns a transform stream that accepts json objects and emits newline delimited json
44 |
45 | example usage:
46 |
47 | ```js
48 | var serialize = ndjson.serialize()
49 | serialize.on('data', function(line) {
50 | // line is a line of stringified JSON with a newline delimiter at the end
51 | })
52 | serialize.write({"foo": "bar"})
53 | serialize.end()
54 | ```
55 |
56 | ### license
57 |
58 | BSD-3-Clause
59 |
--------------------------------------------------------------------------------
/test.js:
--------------------------------------------------------------------------------
1 | var test = require('tape')
2 | var ndj = require('./')
3 | var os = require('os')
4 | var concat = require('concat-stream')
5 |
6 | test('.parse', function(t) {
7 | var parser = ndj.parse()
8 | parser.on('data', function(obj) {
9 | t.equal(obj.hello, 'world')
10 | t.end()
11 | })
12 |
13 | parser.write('{"hello": "world"}\n')
14 | })
15 |
16 | test('.parse twice', function(t) {
17 | var parser = ndj.parse()
18 | parser.once('data', function(obj) {
19 | t.equal(obj.hello, 'world')
20 | parser.once('data', function(obj) {
21 | t.equal(obj.hola, 'mundo')
22 | t.end()
23 | })
24 | })
25 |
26 | parser.write('{"hello": "world"}\n{"hola": "mundo"}\n')
27 | })
28 |
29 | test('.parse - strict:true error', function (t) {
30 | var parser = ndj.parse({strict: true})
31 | try {
32 | parser.write('{"no":"json"\n')
33 | } catch(e) {
34 | t.pass('error thrown')
35 | t.end()
36 | }
37 | })
38 |
39 | test('.parse - strict:true error event', function (t) {
40 | var parser = ndj.parse({strict: true})
41 | parser.on('error', function (err) {
42 | t.pass('error event called')
43 | t.end()
44 | })
45 | try {
46 | parser.write('{"no":"json"\n')
47 | } catch(e) {
48 | t.fail('should not throw')
49 | }
50 | })
51 |
52 | test('.parse - strict:false error', function (t) {
53 | var parser = ndj.parse({strict: false})
54 | parser.once('data', function (data) {
55 | t.ok(data.json, 'parse second one')
56 | t.end()
57 | })
58 | try {
59 | parser.write('{"json":false\n{"json":true}\n')
60 | } catch(e) {
61 | t.fail('should not call an error')
62 | }
63 | })
64 |
65 | test('.serialize', function(t) {
66 | var serializer = ndj.serialize()
67 | serializer.pipe(concat(function(data) {
68 | t.equal(data, '{"hello":"world"}' + os.EOL)
69 | t.end()
70 | }))
71 | serializer.write({hello: 'world'})
72 | serializer.end()
73 | })
74 |
75 | test('.serialize circular', function(t) {
76 | var serializer = ndj.serialize()
77 | serializer.pipe(concat(function(data) {
78 | t.equal(data, '{"obj":"[Circular ~]"}' + os.EOL)
79 | t.end()
80 | }))
81 | var obj = {}
82 | obj.obj = obj
83 | serializer.write(obj)
84 | serializer.end()
85 | })
86 |
--------------------------------------------------------------------------------