├── .travis.yml ├── LICENSE ├── README.md ├── example.js ├── index.js ├── package.json └── test ├── data.json └── index.js /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 0.6 4 | - 0.8 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Dominic Tarr 2 | 3 | Permission is hereby granted, free of charge, 4 | to any person obtaining a copy of this software and 5 | associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including 7 | without limitation the rights to use, copy, modify, 8 | merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom 10 | the Software is furnished to do so, 11 | subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR 20 | ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pull-couch 2 | 3 | Parse the rows out of a couchdb views query without 4 | parsing it as one large json object. instead split based on the format couchdb happens to use (delimited by \r) 5 | and then parse each line with `JSON.parse`. 6 | This is much faster than parsing everything with a streaming parser (if implemented in js), 7 | such as [JSONStream](https://github.com/dominictarr/JSONStream) 8 | 9 | this module is a [pull-stream](https://github.com/dominictarr/pull-stream) 10 | 11 | ## example 12 | 13 | ``` js 14 | var pull = require('pull-stream') 15 | var toPull = require('stream-to-pull-stream') 16 | var Couch = require('pull-couch') 17 | pull( 18 | toPull.source(request('https://skimdb.npmjs.com/registry/_all_docs')), 19 | Couch(function (header) { 20 | //the headers are pased to this cb 21 | console.log('header', header) 22 | }), 23 | 24 | //the rest of the data is streamed out. 25 | pull.drain(console.log) 26 | ) 27 | ``` 28 | 29 | ## License 30 | 31 | MIT 32 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | var pull = require('pull-stream') 2 | var toPull = require('stream-to-pull-stream') 3 | var request = require('request') 4 | var Couch = require('./') 5 | 6 | pull( 7 | toPull.source(request('https://skimdb.npmjs.com/registry/_all_docs')), 8 | Couch(function (header) { 9 | //the headers are pased to this cb 10 | console.log('header', header) 11 | }), 12 | 13 | //the rest of the data is streamed out. 14 | pull.drain(console.log) 15 | ) 16 | 17 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var pull = require('pull-stream') 2 | var utf8 = require('pull-utf8-decoder') 3 | var split = require('pull-split') 4 | 5 | module.exports = function (onHeader) { 6 | var stream 7 | return stream = pull( 8 | utf8(), 9 | //couchdb outputs {rows:[..]} 10 | //and each now ends with ",\r" so just match that 11 | //and dump the extra bits. 12 | split(/\r+/), 13 | //filter first and last lines. 14 | //this is much faster than a valid json parser... 15 | pull.map(function (line) { 16 | line = line.trim() 17 | var last = line[line.length - 1] 18 | if(last == ',') 19 | return line.substring(0, line.length - 1) 20 | else if(line[0] === '{' && last === '}') 21 | return line 22 | else if(last == '[') { 23 | var header = JSON.parse(line + ']}') 24 | delete header.rows 25 | stream.header = header 26 | if(onHeader) onHeader(stream.header) 27 | } 28 | }), 29 | pull.filter(Boolean), 30 | pull.map(JSON.parse) 31 | ) 32 | } 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pull-couch", 3 | "description": "", 4 | "version": "1.0.0", 5 | "homepage": "https://github.com/dominictarr/pull-couch", 6 | "repository": { 7 | "type": "git", 8 | "url": "git://github.com/dominictarr/pull-couch.git" 9 | }, 10 | "dependencies": { 11 | "pull-split": "^0.1.3", 12 | "pull-stream": "^3.1.0", 13 | "pull-utf8-decoder": "^1.0.1" 14 | }, 15 | "devDependencies": { 16 | "request": "^2.69.0", 17 | "stream-to-pull-stream": "^1.6.6" 18 | }, 19 | "scripts": { 20 | "test": "set -e; for t in test/*.js; do node $t; done" 21 | }, 22 | "author": "Dominic Tarr (http://dominictarr.com)", 23 | "license": "MIT" 24 | } 25 | -------------------------------------------------------------------------------- /test/data.json: -------------------------------------------------------------------------------- 1 | {"total_rows":228303,"offset":0,"rows":[ 2 | {"id":"0","key":"0","value":{"rev":"1-7cceb8a9790a5445eebc6342e9d93288"}}, 3 | {"id":"0.0.1","key":"0.0.1","value":{"rev":"2-2db1fbc71753c35964d6cf2de5f0fe3a"}}, 4 | {"id":"0.workspace","key":"0.workspace","value":{"rev":"19-511dee174891b0ce3b37e11972eda5da"}}, 5 | {"id":"001","key":"001","value":{"rev":"2-6216e49615faca29448e369bd0756c21"}}, 6 | {"id":"001_skt","key":"001_skt","value":{"rev":"2-7c9430f9e7c5f5b4840253f9715ff9e1"}}, 7 | {"id":"001_test","key":"001_test","value":{"rev":"2-417ad68c5d0a9a21c69a198f4e7cf9b1"}}, 8 | {"id":"007","key":"007","value":{"rev":"1-7eb402737d5f5e48ab23d11c6a702453"}}, 9 | {"id":"008-somepackage","key":"008-somepackage","value":{"rev":"1-1e704e59d28087d1d0a1b55e2e4528a5"}}, 10 | {"id":"009","key":"009","value":{"rev":"1-266bee51433e60b4079cae406d498feb"}}, 11 | {"id":"zz","key":"zz","value":{"rev":"1-d38b447684e6427da5ac90d2f29c28eb"}}, 12 | {"id":"zzcache","key":"zzcache","value":{"rev":"5-8e372b9411f93b88c7c2c19ccd3261d2"}}, 13 | {"id":"zzishsdk","key":"zzishsdk","value":{"rev":"16-041d67d8a83909f4e0882f603b40597b"}}, 14 | {"id":"zzparser","key":"zzparser","value":{"rev":"1-e03d8ba2b6c6ce6e3d29c85893b28042"}}, 15 | {"id":"zzs","key":"zzs","value":{"rev":"1-978a9fb660fce09cfd800d34eee918cf"}}, 16 | {"id":"zztest","key":"zztest","value":{"rev":"1-20ec3ee757d0f40eb53e93fb4b3e3641"}}, 17 | {"id":"zzz","key":"zzz","value":{"rev":"1-95be06e72a244af952f8009933225728"}}, 18 | {"id":"zzzz","key":"zzzz","value":{"rev":"1-35524f1f9923f3e41226c461f2ae58af"}}, 19 | {"id":"zzzzz","key":"zzzzz","value":{"rev":"1-cafd42a567950fd13e050d803c533c30"}} 20 | ]} 21 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | var pull = require('pull-stream') 2 | var split = require('pull-randomly-split') 3 | var data = require('./data.json') 4 | var path = require('path') 5 | var fs = require('fs') 6 | var b = fs.readFileSync(path.join(__dirname, 'data.json')) 7 | 8 | var Couch = require('../') 9 | 10 | var headers 11 | 12 | require('tape')('parsed data matches couch', function (t) { 13 | 14 | pull( 15 | pull.once(b), 16 | split(), 17 | Couch(function (_headers) { 18 | headers = _headers 19 | console.log(headers) 20 | }), 21 | pull.collect(function (err, rows) { 22 | t.deepEqual(rows, data.rows) 23 | delete data.rows 24 | t.deepEqual(headers, data) 25 | t.end() 26 | }) 27 | ) 28 | 29 | }) 30 | --------------------------------------------------------------------------------