├── .gitignore ├── LICENSE ├── cli.js ├── collaborators.md ├── index.js ├── package.json ├── readme.md ├── test ├── index.js ├── test-transform.js ├── test-transform2.js ├── test-transform3.js └── test.json └── usage.txt /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Max Ogden 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var map = require('./') 4 | var ndjson = require('ndjson') 5 | var args = require('minimist')(process.argv.slice(2)) 6 | var path = require('path') 7 | var fs = require('fs') 8 | 9 | var usage = fs.readFileSync(__dirname + '/usage.txt').toString() 10 | 11 | var transform = args._[0] 12 | if (args.file) transform = require(path.resolve(process.cwd(), args.file)) 13 | 14 | if (!transform) { 15 | console.error(usage) 16 | process.exit(1) 17 | } 18 | 19 | process.stdin 20 | .pipe(map(transform, args)) 21 | .pipe(ndjson.serialize()) 22 | .pipe(process.stdout) 23 | -------------------------------------------------------------------------------- /collaborators.md: -------------------------------------------------------------------------------- 1 | ## Collaborators 2 | 3 | jsonmap is only possible due to the excellent work of the following collaborators: 4 | 5 | 6 | 7 | 8 | 9 |
maxogdenGitHub/maxogden
mafintoshGitHub/mafintosh
groundwaterGitHub/groundwater
ryanramageGitHub/ryanramage
10 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var ndjson = require('ndjson') 2 | var pumpify = require('pumpify') 3 | var through = require('through2') 4 | var split = require('split2') 5 | 6 | module.exports = function(func, opts) { 7 | if (!opts) opts = {} 8 | var transform = createFunctionStream(func, opts) 9 | if (opts.parse === false) return pumpify.obj(split(), transform) 10 | return pumpify.obj(ndjson.parse(), transform) 11 | } 12 | 13 | module.exports.createFunctionStream = createFunctionStream 14 | function createFunctionStream (func, opts) { 15 | if (!opts) opts = {} 16 | var compiled 17 | if (typeof func !== 'function') { 18 | var funcStr = func + ';\n return this;' 19 | if (func[0] === '{' || func[0] === '`') funcStr = 'var t = ' + func + ';\n return t;' 20 | compiled = new Function('require', '_', funcStr) 21 | } else if (opts.through) { 22 | return through.obj(func) 23 | } else { 24 | compiled = function (require, obj) { 25 | return func.call(obj, require, obj) || this // in case the function just mutates `this` w/o returning. 26 | } 27 | } 28 | 29 | function transform (obj, enc, next) { 30 | var out = compiled.call(obj, require, obj) 31 | if (out) this.push(out) 32 | next() 33 | } 34 | 35 | return through.obj(transform) 36 | } 37 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jsonmap", 3 | "description": "streaming command line newline-delimited json transformer utility", 4 | "version": "2.0.1", 5 | "bin": { 6 | "jsonmap": "cli.js" 7 | }, 8 | "scripts": { 9 | "test": "node ./test" 10 | }, 11 | "main": "index.js", 12 | "author": "max ogden", 13 | "license": "BSD-2-Clause", 14 | "dependencies": { 15 | "minimist": "^1.2.0", 16 | "ndjson": "^1.5.0", 17 | "pumpify": "^1.3.5", 18 | "split2": "^2.1.1", 19 | "through2": "^2.0.3" 20 | }, 21 | "devDependencies": {}, 22 | "repository": { 23 | "type": "git", 24 | "url": "https://github.com/maxogden/jsonmap.git" 25 | }, 26 | "bugs": { 27 | "url": "https://github.com/maxogden/jsonmap/issues" 28 | }, 29 | "homepage": "https://github.com/maxogden/jsonmap" 30 | } 31 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # jsonmap 2 | 3 | [![NPM](https://nodei.co/npm/jsonmap.png?global=true)](https://nodei.co/npm/jsonmap/) 4 | 5 | streaming command line newline-delimited json transformer utility 6 | 7 | you must pipe newline-delimited JSON data in (one JSON stringified object per line). you will receive the same format out 8 | 9 | ## installation 10 | 11 | ```BASH 12 | $ npm install jsonmap -g 13 | ``` 14 | 15 | ## usage 16 | 17 | `this` will be each line of JSON that gets parsed out of the incoming newline-delimited json stream. you can also use `_` as a shorthand for `this`, and you are allowed to `require` things. 18 | 19 | there are two 'modes', the first is where you modify `this`: 20 | 21 | ```BASH 22 | $ echo '{"foo": "bar"}\n{"baz": "taco"}' | jsonmap "this.pizza = 1" 23 | {"foo":"bar","pizza":1} 24 | {"baz":"taco","pizza":1} 25 | ``` 26 | 27 | the second mode is where you return a new object: 28 | 29 | ```BASH 30 | $ echo '{"foo": "bar", "cat": "yes"}\n{"baz": "taco", "cat": "yes"}' | jsonmap "{cat: this.cat}" 31 | {"cat":"yes"} 32 | {"cat":"yes"} 33 | ``` 34 | 35 | if your code gets too complex and you'd rather use an external file you can also just specify a module to get required: 36 | 37 | ```BASH 38 | $ echo '{"foo": "bar"}\n{"baz": "taco"}' | jsonmap --file=transform.js 39 | {"foo":"bar","pizza":1} 40 | {"baz":"taco","pizza":1} 41 | ``` 42 | 43 | the above will work if `transform.js` has the following contents: 44 | 45 | ```js 46 | module.exports = function() { 47 | this.pizza = 1 48 | } 49 | ``` 50 | 51 | if you have es6 template strings enabled on your platform (e.g. iojs), template strings will work as well 52 | 53 | ```BASH 54 | $ echo '{"meal": "pizza"}\n{"meal": "taco"}' | jsonmap '`i love ${this.meal}`' 55 | "i love pizza" 56 | "i love taco" 57 | ``` 58 | 59 | if you want to provide a [through2](https://github.com/rvagg/through2) function in a file for more control, or async, you can 60 | 61 | ```BASH 62 | $ echo '{"foo": "bar"}\n{"baz": "taco"}' | jsonmap --file=transform.js --through 63 | {"foo":"bar","pizza":1} 64 | {"baz":"taco","pizza":1} 65 | ``` 66 | 67 | the above will work if `transform.js` has the following contents: 68 | 69 | ```js 70 | module.exports = function(obj, enc, next) { 71 | var self = this; 72 | if(obj.foo === 'bar') return next() // skip the bar 73 | process.nextTick(function(){ 74 | self.push({ count: obj.pizza }) 75 | next() 76 | }) 77 | } 78 | ``` 79 | 80 | you disable JSON parsing (to e.g. process a file line by line as JS strings) by passing `jsonmap --no-parse` 81 | 82 | 83 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | var exec = require('child_process').exec; 2 | 3 | process.chdir(__dirname) 4 | 5 | exec('cat test.json | node ../cli.js \"{dog: this.dog}\"', 6 | expect('{"dog":5}\n{"dog":6}\n')); 7 | 8 | exec('cat test.json | node ../cli.js --file=test-transform.js', 9 | expect('{"summary":["foo","dog"]}\n{"summary":["baz","dog"]}\n')); 10 | 11 | exec('cat test.json | node ../cli.js \"{dog: this.dog}\" | node ../cli.js --file=test-transform2.js', 12 | expect('{"dog":6}\n{"dog":7}\n')); 13 | 14 | exec('cat test.json | node ../cli.js \"{dog: this.dog}\" | node ../cli.js --file=test-transform3.js --through', 15 | expect('{"dog":5}\n{"dog":6}\n{"dog":7}\n')); 16 | 17 | supportsTemplateStrings(function () { 18 | exec('cat test.json | node ../cli.js \'\`${this.dog} dogs\`\'', 19 | expect('"5 dogs"\n"6 dogs"\n')); 20 | }) 21 | 22 | // test require is exposed 23 | exec('cat test.json | node ../cli.js \"return require(\'url\').parse(\'http://foo\').hostname\"', 24 | expect('"foo"\n"foo"\n')); 25 | 26 | function expect(output) { 27 | function done(err, stdout, stderr) { 28 | if(stdout !== output) { 29 | throw new Error('Expected '+output+'\nActual: '+stdout); 30 | } 31 | console.log(stdout); 32 | if(err) { throw err; } 33 | } 34 | 35 | return done; 36 | } 37 | 38 | function supportsTemplateStrings(cb) { 39 | try { 40 | eval('``'); 41 | cb(); 42 | } catch(e) {} 43 | } 44 | -------------------------------------------------------------------------------- /test/test-transform.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = function x() { 3 | return { summary: Object.keys(this) } 4 | } 5 | -------------------------------------------------------------------------------- /test/test-transform2.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = function x() { 3 | this.dog++; 4 | } 5 | -------------------------------------------------------------------------------- /test/test-transform3.js: -------------------------------------------------------------------------------- 1 | module.exports = function(data, enc, next){ 2 | var self = this; 3 | if (data.dog == 5) { 4 | self.push(data) 5 | } 6 | 7 | process.nextTick(function(){ 8 | data.dog++ 9 | self.push(data) 10 | next() 11 | }) 12 | } -------------------------------------------------------------------------------- /test/test.json: -------------------------------------------------------------------------------- 1 | {"foo": "bar", "dog": 5} 2 | {"baz": "taco", "dog": 6} 3 | -------------------------------------------------------------------------------- /usage.txt: -------------------------------------------------------------------------------- 1 | jsonmap 2 | 3 | streaming command line newline-delimited json transformer utility 4 | 5 | Usage: 6 | 7 | jsonmap "" 8 | 9 | Options 10 | 11 | --no-parse treats incoming data as newline separated strings, skips JSON.parse 12 | --file load a file as a transform instead of using "" 13 | --through boolean, default false, treat file/expression as a through stream 14 | 15 | Example 16 | 17 | cat foo.json | jsonmap "this.foo = bar" --------------------------------------------------------------------------------