├── bin ├── cdbdump ├── cdbload └── cdbmorph ├── .eslintrc ├── .gitignore ├── README.md ├── package.json └── lib ├── couchdbload.js ├── couchdbmorph.js ├── utils.js └── couchdbdump.js /bin/cdbdump: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var path = require('path'); 4 | var fs = require('fs'); 5 | var lib = path.join(path.dirname(fs.realpathSync(__filename)), '../lib'); 6 | 7 | require(lib + '/couchdbdump.js').couchdbdump(); 8 | -------------------------------------------------------------------------------- /bin/cdbload: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var path = require('path'); 4 | var fs = require('fs'); 5 | var lib = path.join(path.dirname(fs.realpathSync(__filename)), '../lib'); 6 | 7 | require(lib + '/couchdbload.js').couchdbload(); 8 | -------------------------------------------------------------------------------- /bin/cdbmorph: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var path = require('path'); 4 | var fs = require('fs'); 5 | var lib = path.join(path.dirname(fs.realpathSync(__filename)), '../lib'); 6 | 7 | require(lib + '/couchdbmorph.js').couchdbmorph(); 8 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "node": true 4 | }, 5 | "rules":{ 6 | "no-trailing-spaces": 0, 7 | "quotes": 0, 8 | "no-use-before-define": 0, 9 | "strict": 0, 10 | "consistent-return": 0, 11 | "camelcase": 0, 12 | "no-underscore-dangle": 0 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | devdocs/ 3 | 4 | # osx noise 5 | .DS_Store 6 | profile 7 | 8 | # xcode noise 9 | build/* 10 | *.mode1 11 | *.mode1v3 12 | *.mode2v3 13 | *.perspective 14 | *.perspectivev3 15 | *.pbxuser 16 | *.xcworkspace 17 | xcuserdata 18 | 19 | # svn & cvs 20 | .svn 21 | CVS 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | couchdb-dump 2 | ======== 3 | 4 | THIS PACKAGE IS NO LONGER MAINTAINED. DO NOT USE. 5 | 6 | SEE INSTEAD ... http://stedolan.github.io/jq/ 7 | 8 | It is possible to simulate most of the functionality of couchdb-dump using curl to fetch the contents of your db, 9 | and piping the output through jq. 10 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "couchdb-dump", 3 | "version": "2.2.1", 4 | "description": "Tools to dump, modify, and load documents in CouchDB from the command line. (Same basic concept as mysqldump, but much more and for CouchDB)", 5 | "author": "Raffi Minassian (https://twitter.com/RaffiMinassian)", 6 | "license": "Apache-2.0", 7 | "engines": { 8 | "node": ">=4.2.2" 9 | }, 10 | "bin": { 11 | "cdbdump": "./bin/cdbdump", 12 | "cdbload": "./bin/cdbload", 13 | "cdbmorph": "./bin/cdbmorph" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "https://github.com/raffi-minassian/couchdb-dump.git" 18 | }, 19 | "main": "./lib/couchdbdump.js", 20 | "scripts": { 21 | "test": "echo 'Error: no test specified' && exit 1" 22 | }, 23 | "keywords": [ 24 | "couchdb", 25 | "backup", 26 | "database", 27 | "dump", 28 | "mysqldump" 29 | ], 30 | "dependencies": { 31 | "async": "^2.6.0", 32 | "jsonparse": "1.x.x", 33 | "minimist": "1.x.x", 34 | "mkdirp": "^0.5.1", 35 | "request": "2.x.x", 36 | "request-promise": "^4.2.2", 37 | "request-promise-native": "^1.0.5", 38 | "through2": "2.x.x" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /lib/couchdbload.js: -------------------------------------------------------------------------------- 1 | var parseargs = require('minimist'); 2 | var couchreq = require('request'); 3 | var pkg = require('../package.json'); 4 | 5 | exports.couchdbload = function(){ 6 | var args = parseargs(process.argv.slice(2), { 7 | default: { 8 | "P": "5984", 9 | "h": "localhost", 10 | "r": "http" 11 | } 12 | }); 13 | if(args.version){ 14 | console.log('v' + pkg.version); 15 | return; 16 | } 17 | if(args.help || !args.h || !args.P || !args.d){ 18 | console.log("usage: cdbload [-u username] [-p password] [-h host] [-P port] [-r protocol] [-v verbose] -d database"); 19 | return; 20 | } 21 | 22 | var auth = ""; 23 | if(args.u){ 24 | auth = args.u + ":"; 25 | } 26 | if(args.p){ 27 | auth = auth + args.p; 28 | } 29 | if(auth){ 30 | auth = auth + "@"; 31 | } 32 | 33 | process.stdin.pipe(couchreq({ 34 | method: "POST", 35 | url: args.r + "://" + auth + args.h + ":" + args.P + "/" + args.d + "/_bulk_docs", 36 | headers: { 37 | "Accept": "application/json", 38 | "Content-type": "application/json" 39 | } 40 | }, function(err, res, body){ 41 | if(err){ 42 | return console.log(err); 43 | } 44 | console.log("CouchDB response code: " + res.statusCode); 45 | console.log("CouchDB response message: " + res.statusMessage); 46 | if(args.v){ 47 | console.log(JSON.stringify(body, null, 2)); 48 | } 49 | })); 50 | 51 | }; 52 | -------------------------------------------------------------------------------- /lib/couchdbmorph.js: -------------------------------------------------------------------------------- 1 | var parseargs = require('minimist'); 2 | var couchreq = require('request'); 3 | var through2 = require('through2'); 4 | var Parser = require('jsonparse'); 5 | var path = require('path'); 6 | var pkg = require('../package.json'); 7 | 8 | exports.couchdbmorph = function(){ 9 | var args = parseargs(process.argv.slice(2), { 10 | default: { 11 | "s": 0, 12 | } 13 | }); 14 | if(args.version){ 15 | console.log('v' + pkg.version); 16 | return; 17 | } 18 | if(args.help || !args.f){ 19 | console.error("usage: cdbmorph [-s json-stringify-space] -f path-to-morphfunction.js"); 20 | return; 21 | } 22 | 23 | var morphfunction = require(path.resolve(process.cwd(), args.f)); 24 | var skip = true; 25 | var p = new Parser(); 26 | 27 | var ts = through2.obj(function (chunk, enc, callback){ 28 | p.write(chunk); 29 | callback(); 30 | }) 31 | .on('data', function (data) { 32 | morphts.write(JSON.stringify(data)); 33 | }) 34 | .on('end', function () { 35 | morphts.end(); 36 | }); 37 | 38 | var morphts = through2(function (chunk, enc, callback){ 39 | var doc = JSON.parse(chunk); 40 | morphfunction(doc, function(err, morphed){ 41 | if(!morphed){ 42 | return callback(); 43 | } 44 | if(!skip){ 45 | morphts.push(','); 46 | } else { 47 | skip = false; 48 | } 49 | if(err){ 50 | console.error(err); 51 | morphts.push(JSON.stringify(chunk, null, args.s)); 52 | return callback(); 53 | } 54 | morphts.push(JSON.stringify(morphed, null, args.s)); 55 | callback(); 56 | }); 57 | }, function(callback){ 58 | this.push(']}\n'); 59 | callback(); 60 | }); 61 | 62 | p.onValue = function (value) { 63 | if(this.stack.length === 2){ 64 | ts.push(value); 65 | } 66 | }; 67 | 68 | process.stdout.write('{"docs": [ '); 69 | process.stdin.pipe(ts); 70 | morphts.pipe(process.stdout); 71 | }; 72 | -------------------------------------------------------------------------------- /lib/utils.js: -------------------------------------------------------------------------------- 1 | const parseargs = require('minimist'); 2 | 3 | exports.buildConf = function() { 4 | let conf = { 5 | couchdb: {}, 6 | output: null, 7 | allDatabases: false, 8 | createDatabase: false, 9 | database: "", 10 | version: false, 11 | jsonStringifySpace: false, 12 | stripRevs: true, 13 | designDoc: false, 14 | views: false, 15 | help: false, 16 | numberPerPage: 500, 17 | concurrency: 1, 18 | verbose: false, 19 | } 20 | 21 | const args = parseargs(process.argv.slice(2)); 22 | conf.couchdb = buildCouchDBConfig(args); 23 | 24 | if(args.version) { 25 | conf.version = true; 26 | return conf; 27 | } 28 | 29 | if(args.help) { 30 | conf.help = true; 31 | return conf; 32 | } 33 | 34 | if(args.verbose) { 35 | conf.verbose = true; 36 | } 37 | 38 | if(args.concurrency) { 39 | conf.concurrency = args.concurrency; 40 | } 41 | 42 | if(args.numberPerPage) { 43 | conf.numberPerPage = args.numberPerPage; 44 | } 45 | 46 | if(args.k) { 47 | conf.stripRevs = false; 48 | } 49 | 50 | if(args.D) { 51 | conf.designDoc = args.D; 52 | } 53 | 54 | if(args.v) { 55 | conf.views = args.v; 56 | } 57 | 58 | if(args.a) { 59 | conf.allDatabases = true; 60 | } 61 | 62 | if(args.d) { 63 | conf.database = args.d; 64 | } 65 | 66 | if(args["create-database"]) { 67 | conf.createDatabase = true; 68 | } 69 | 70 | if(args.o) { 71 | conf.output = args.o; 72 | } 73 | 74 | return conf; 75 | } 76 | 77 | function buildCouchDBConfig(args) { 78 | let conf = { 79 | url: "", 80 | port: 5984, 81 | host: "localhost", 82 | protocol: "http", 83 | auth: false 84 | }; 85 | 86 | if(args.h) { 87 | conf.host = args.h; 88 | } 89 | 90 | if(args.P) { 91 | conf.port = args.P; 92 | } 93 | 94 | if(args.r) { 95 | conf.protocol = args.r; 96 | } 97 | 98 | conf.auth = buildAuth(args.u, args.p); 99 | conf.url = buildCouchDBURL(conf); 100 | 101 | return conf; 102 | } 103 | 104 | function buildAuth(user, pass) { 105 | let auth = ""; 106 | if(user) { 107 | auth = user + ":"; 108 | } 109 | 110 | if(pass) { 111 | auth += pass; 112 | } 113 | 114 | if(auth === "") { 115 | return false; 116 | } 117 | 118 | return auth; 119 | } 120 | 121 | function buildCouchDBURL(conf) { 122 | let couchDBUrl = conf.protocol + '://'; 123 | 124 | if(conf.auth) { 125 | couchDBUrl += conf.auth + "@"; 126 | } 127 | 128 | couchDBUrl += conf.host + ':' + conf.port + '/'; 129 | 130 | return couchDBUrl; 131 | } 132 | 133 | -------------------------------------------------------------------------------- /lib/couchdbdump.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const queue = require('async/queue'); 3 | const requestPromise = require('request-promise'); 4 | const async = require('async'); 5 | const mkdirp = require('mkdirp'); 6 | 7 | const pkg = require('../package.json'); 8 | const util = require('./utils'); 9 | 10 | exports.couchdbdump = function() { 11 | const conf = util.buildConf(); 12 | 13 | if(conf.version) { 14 | console.log('v' + pkg.version); 15 | return; 16 | } 17 | 18 | if(conf.help || !conf.couchdb.host || !conf.couchdb.port || (!conf.database && !conf.allDatabases)) { 19 | console.error("usage: cdbdump [-u username] [-p password] [-h host] [-P port] [-r protocol] [-s json-stringify-space] [-k dont-strip-revs] [-D design doc] [-v view] [-d database|-a all-databases] [--concurency] [-o output-directory]"); 20 | return; 21 | } 22 | 23 | let output = process.stdout; 24 | if(conf.output) { 25 | mkdirp.sync(conf.output); 26 | } 27 | 28 | if(conf.allDatabases) { 29 | const q = async.queue(function(task, cb) { 30 | dump(task.conf, task.db, task.output, 0, () => { 31 | output.close(); 32 | cb(); 33 | }); 34 | }, conf.concurrency); 35 | 36 | requestPromise(conf.couchdb.url + '_all_dbs') 37 | .then(function(data) { 38 | dbs = JSON.parse(data); 39 | dbs.map((db) => { 40 | if(conf.output) { 41 | mkdirp.sync(conf.output); 42 | output = fs.createWriteStream(conf.output + '/' + db + '.json'); 43 | } 44 | q.push({db: db, conf: conf, output: output}); 45 | }); 46 | }) 47 | .catch(function(err) { 48 | console.error(err); 49 | }); 50 | return; 51 | } 52 | 53 | if(conf.output) { 54 | output = fs.createWriteStream(conf.output + '/' + conf.database + '.json'); 55 | } 56 | 57 | dump(conf, conf.database, output); 58 | }; 59 | 60 | function dump(options, database, output, offset = 0, cb = () => {}) { 61 | let request = { 62 | qs: { 63 | include_docs: "true", 64 | attachments: "true", 65 | limit: options.numberPerPage, 66 | skip: offset 67 | }, 68 | headers: { 69 | "Accept": "application/json" 70 | } 71 | } 72 | request.url = options.couchdb.url + database; 73 | if(options.designDoc && options.views) { 74 | request.url += '/_design/' + options.designDoc + '/_view/' + options.views; 75 | } else { 76 | request.url += '/_all_docs'; 77 | } 78 | 79 | if(offset == 0) { 80 | output.write('{"docs": [ '); 81 | } 82 | 83 | requestPromise 84 | .get(request) 85 | .then(function(data) { 86 | console.error('then') 87 | const docs = JSON.parse(data); 88 | if(docs.rows.length > 0) { 89 | let dataArray = []; 90 | docs.rows.map((row) => { 91 | dataArray.push(JSON.stringify(row)); 92 | }); 93 | output.write(dataArray.join(',')); 94 | dataArray = []; 95 | 96 | if(docs.rows.length == options.numberPerPage) { 97 | return dump(options, database, output, (offset+options.numberPerPage), cb) 98 | } 99 | } 100 | output.write(']}\n'); 101 | cb(); 102 | }) 103 | .catch(function(err) { 104 | console.error(err); 105 | }); 106 | } 107 | 108 | --------------------------------------------------------------------------------