├── .gitignore ├── config.sample.json ├── package.json ├── bin └── mongodb_s3_backup.js ├── README.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | *.sw[a-z] 3 | *.orig 4 | .DS_Store 5 | *.working 6 | config.json 7 | npm-debug.log 8 | *.project 9 | -------------------------------------------------------------------------------- /config.sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "mongodb": { 3 | "host": "localhost", 4 | "port": 27017, 5 | "username": false, 6 | "password": false, 7 | "db": "database_to_backup" 8 | }, 9 | "s3": { 10 | "key": "your_s3_key", 11 | "secret": "your_s3_secret", 12 | "bucket": "s3_bucket_to_upload_to", 13 | "destination": "/", 14 | "encrypt": true, 15 | "region": "s3_region_to_use" 16 | }, 17 | "cron": { 18 | "time": "00:00" 19 | } 20 | } 21 | 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongodb_s3_backup", 3 | "version": "0.0.8", 4 | "description": "A tool to help backup your mongodb databases to s3.", 5 | "main": "index.js", 6 | "bin": { 7 | "mongodb_s3_backup": "./bin/mongodb_s3_backup.js" 8 | }, 9 | "preferGlobal": "true", 10 | "scripts": { 11 | "start": "node ./bin/mongodb_s3_backup.js ./config.json", 12 | "test": "echo \"Error: no test specified\" && exit 1" 13 | }, 14 | "repository": "https://github.com/theycallmeswift/node-mongodb-s3-backup.git", 15 | "keywords": [ 16 | "Mongodb", 17 | "S3", 18 | "backup" 19 | ], 20 | "author": "Swift ", 21 | "license": "Beerware", 22 | "dependencies": { 23 | "async": "~0.1.22", 24 | "cli": "~0.4.4-2", 25 | "cli-color": "~0.2.1", 26 | "cron": "~1.0.1", 27 | "knox": "~0.8.0", 28 | "time": "^0.11.2" 29 | }, 30 | "engines": { 31 | "node": ">=0.8" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /bin/mongodb_s3_backup.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /* Dependencies */ 4 | 5 | var cli = require('cli') 6 | , path = require('path') 7 | , util = require('util') 8 | , backup = require('../') 9 | , cronJob = require('cron').CronJob 10 | , pkg = require('../package.json') 11 | , crontab = "0 0 * * *" 12 | , timezone = "UTC" 13 | , time = [0, 0] 14 | , options, configPath, config; 15 | 16 | cli 17 | .enable('version') 18 | .setApp(pkg.name, pkg.version) 19 | .setUsage(cli.app + ' [OPTIONS] '); 20 | 21 | options = cli.parse({ 22 | now: ['n', 'Run sync on start'] 23 | }); 24 | 25 | if(cli.args.length !== 1) { 26 | return cli.getUsage(); 27 | } 28 | 29 | /* Configuration */ 30 | 31 | configPath = path.resolve(process.cwd(), cli.args[0]); 32 | backup.log('Loading config file (' + configPath + ')'); 33 | config = require(configPath); 34 | 35 | if(options.now) { 36 | backup.sync(config.mongodb, config.s3, function(err) { 37 | process.exit(err ? 1 : 0); 38 | }); 39 | } else { 40 | // If the user overrides the default cron behavior 41 | if(config.cron) { 42 | if(config.cron.crontab) { 43 | crontab = config.cron.crontab 44 | } else if(config.cron.time) { 45 | time = config.cron.time.split(':') 46 | crontab = util.format('%d %d * * *', time[1], time[0]); 47 | } 48 | 49 | if(config.cron.timezone) { 50 | try { 51 | require('time'); // Make sure the user has time installed 52 | } catch(e) { 53 | backup.log(e, "error"); 54 | backup.log("Module 'time' is not installed by default, install it with `npm install time`", "error"); 55 | process.exit(1); 56 | } 57 | 58 | timezone = config.cron.timezone; 59 | backup.log('Overriding default timezone with "' + timezone + '"'); 60 | } 61 | } 62 | 63 | new cronJob(crontab, function(){ 64 | backup.sync(config.mongodb, config.s3); 65 | }, null, true, timezone); 66 | backup.log('MongoDB S3 Backup Successfully scheduled (' + crontab + ')'); 67 | } 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Node MongoDB / S3 Backup 2 | 3 | This is a package that makes backing up your mongo databases to S3 simple. 4 | The binary file is a node cronjob that runs at midnight every day and backs up 5 | the database specified in the config file. 6 | 7 | ## Installation 8 | 9 | npm install mongodb_s3_backup -g 10 | 11 | ## Configuration 12 | 13 | To configure the backup, you need to pass the binary a JSON configuration file. 14 | There is a sample configuration file supplied in the package (`config.sample.json`). 15 | The file should have the following format: 16 | 17 | { 18 | "mongodb": { 19 | "host": "localhost", 20 | "port": 27017, 21 | "username": false, 22 | "password": false, 23 | "db": "database_to_backup" 24 | }, 25 | "s3": { 26 | "key": "your_s3_key", 27 | "secret": "your_s3_secret", 28 | "bucket": "s3_bucket_to_upload_to", 29 | "destination": "/", 30 | "encrypt": true, 31 | "region": "s3_region_to_use" 32 | }, 33 | "cron": { 34 | "time": "11:59", 35 | } 36 | } 37 | 38 | All options in the "s3" object, except for desination, will be directly passed to knox, therefore, you can include any of the options listed [in the knox documentation](https://github.com/LearnBoost/knox#client-creation-options "Knox README"). 39 | 40 | ### Crontabs 41 | 42 | You may optionally substitute the cron "time" field with an explicit "crontab" 43 | of the standard format `0 0 * * *`. 44 | 45 | "cron": { 46 | "crontab": "0 0 * * *" 47 | } 48 | 49 | *Note*: The version of cron that we run supports a sixth digit (which is in seconds) if 50 | you need it. 51 | 52 | ### Timezones 53 | 54 | The optional "timezone" allows you to specify timezone-relative time regardless 55 | of local timezone on the host machine. 56 | 57 | "cron": { 58 | "time": "00:00", 59 | "timezone": "America/New_York" 60 | } 61 | 62 | You must first `npm install time` to use "timezone" specification. 63 | 64 | ## Running 65 | 66 | To start a long-running process with scheduled cron job: 67 | 68 | mongodb_s3_backup 69 | 70 | To execute a backup immediately and exit: 71 | 72 | mongodb_s3_backup -n 73 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var exec = require('child_process').exec 4 | , spawn = require('child_process').spawn 5 | , path = require('path') 6 | , domain = require('domain') 7 | , d = domain.create(); 8 | 9 | /** 10 | * log 11 | * 12 | * Logs a message to the console with a tag. 13 | * 14 | * @param message the message to log 15 | * @param tag (optional) the tag to log with. 16 | */ 17 | function log(message, tag) { 18 | var util = require('util') 19 | , color = require('cli-color') 20 | , tags, currentTag; 21 | 22 | tag = tag || 'info'; 23 | 24 | tags = { 25 | error: color.red.bold, 26 | warn: color.yellow, 27 | info: color.cyanBright 28 | }; 29 | 30 | currentTag = tags[tag] || function(str) { return str; }; 31 | util.log((currentTag("[" + tag + "] ") + message).replace(/(\n|\r|\r\n)$/, '')); 32 | } 33 | 34 | /** 35 | * getArchiveName 36 | * 37 | * Returns the archive name in database_YYYY_MM_DD.tar.gz format. 38 | * 39 | * @param databaseName The name of the database 40 | */ 41 | function getArchiveName(databaseName) { 42 | var date = new Date() 43 | , datestring; 44 | 45 | datestring = [ 46 | databaseName, 47 | date.getFullYear(), 48 | date.getMonth() + 1, 49 | date.getDate(), 50 | date.getTime() 51 | ]; 52 | 53 | return datestring.join('_') + '.tar.gz'; 54 | } 55 | 56 | /* removeRF 57 | * 58 | * Remove a file or directory. (Recursive, forced) 59 | * 60 | * @param target path to the file or directory 61 | * @param callback callback(error) 62 | */ 63 | function removeRF(target, callback) { 64 | var fs = require('fs'); 65 | 66 | callback = callback || function() { }; 67 | 68 | fs.exists(target, function(exists) { 69 | if (!exists) { 70 | return callback(null); 71 | } 72 | log("Removing " + target, 'info'); 73 | exec( 'rm -rf ' + target, callback); 74 | }); 75 | } 76 | 77 | /** 78 | * mongoDump 79 | * 80 | * Calls mongodump on a specified database. 81 | * 82 | * @param options MongoDB connection options [host, port, username, password, db] 83 | * @param directory Directory to dump the database to 84 | * @param callback callback(err) 85 | */ 86 | function mongoDump(options, directory, callback) { 87 | var mongodump 88 | , mongoOptions; 89 | 90 | callback = callback || function() { }; 91 | 92 | mongoOptions= [ 93 | '-h', options.host + ':' + options.port, 94 | '-d', options.db, 95 | '-o', directory 96 | ]; 97 | 98 | if(options.username && options.password) { 99 | mongoOptions.push('-u'); 100 | mongoOptions.push(options.username); 101 | 102 | mongoOptions.push('-p'); 103 | mongoOptions.push(options.password); 104 | } 105 | 106 | log('Starting mongodump of ' + options.db, 'info'); 107 | mongodump = spawn('mongodump', mongoOptions); 108 | 109 | mongodump.stdout.on('data', function (data) { 110 | log(data); 111 | }); 112 | 113 | mongodump.stderr.on('data', function (data) { 114 | log(data, 'error'); 115 | }); 116 | 117 | mongodump.on('exit', function (code) { 118 | if(code === 0) { 119 | log('mongodump executed successfully', 'info'); 120 | callback(null); 121 | } else { 122 | callback(new Error("Mongodump exited with code " + code)); 123 | } 124 | }); 125 | } 126 | 127 | /** 128 | * compressDirectory 129 | * 130 | * Compressed the directory so we can upload it to S3. 131 | * 132 | * @param directory current working directory 133 | * @param input path to input file or directory 134 | * @param output path to output archive 135 | * @param callback callback(err) 136 | */ 137 | function compressDirectory(directory, input, output, callback) { 138 | var tar 139 | , tarOptions; 140 | 141 | callback = callback || function() { }; 142 | 143 | tarOptions = [ 144 | '-zcf', 145 | output, 146 | input 147 | ]; 148 | 149 | log('Starting compression of ' + input + ' into ' + output, 'info'); 150 | tar = spawn('tar', tarOptions, { cwd: directory }); 151 | 152 | tar.stderr.on('data', function (data) { 153 | log(data, 'error'); 154 | }); 155 | 156 | tar.on('exit', function (code) { 157 | if(code === 0) { 158 | log('successfully compress directory', 'info'); 159 | callback(null); 160 | } else { 161 | callback(new Error("Tar exited with code " + code)); 162 | } 163 | }); 164 | } 165 | 166 | /** 167 | * sendToS3 168 | * 169 | * Sends a file or directory to S3. 170 | * 171 | * @param options s3 options [key, secret, bucket] 172 | * @param directory directory containing file or directory to upload 173 | * @param target file or directory to upload 174 | * @param callback callback(err) 175 | */ 176 | function sendToS3(options, directory, target, callback) { 177 | var knox = require('knox') 178 | , sourceFile = path.join(directory, target) 179 | , s3client 180 | , destination = options.destination || '/' 181 | , headers = {}; 182 | 183 | callback = callback || function() { }; 184 | 185 | // Deleting destination because it's not an explicitly named knox option 186 | delete options.destination; 187 | s3client = knox.createClient(options); 188 | 189 | if (options.encrypt) 190 | headers = {"x-amz-server-side-encryption": "AES256"} 191 | 192 | log('Attemping to upload ' + target + ' to the ' + options.bucket + ' s3 bucket'); 193 | s3client.putFile(sourceFile, path.join(destination, target), headers, function(err, res){ 194 | if(err) { 195 | return callback(err); 196 | } 197 | 198 | res.setEncoding('utf8'); 199 | 200 | res.on('data', function(chunk){ 201 | if(res.statusCode !== 200) { 202 | log(chunk, 'error'); 203 | } else { 204 | log(chunk); 205 | } 206 | }); 207 | 208 | res.on('end', function(chunk) { 209 | if (res.statusCode !== 200) { 210 | return callback(new Error('Expected a 200 response from S3, got ' + res.statusCode)); 211 | } 212 | log('Successfully uploaded to s3'); 213 | return callback(); 214 | }); 215 | }); 216 | } 217 | 218 | /** 219 | * sync 220 | * 221 | * Performs a mongodump on a specified database, gzips the data, 222 | * and uploads it to s3. 223 | * 224 | * @param mongodbConfig mongodb config [host, port, username, password, db] 225 | * @param s3Config s3 config [key, secret, bucket] 226 | * @param callback callback(err) 227 | */ 228 | function sync(mongodbConfig, s3Config, callback) { 229 | var tmpDir = path.join(require('os').tmpDir(), 'mongodb_s3_backup') 230 | , backupDir = path.join(tmpDir, mongodbConfig.db) 231 | , archiveName = getArchiveName(mongodbConfig.db) 232 | , async = require('async') 233 | , tmpDirCleanupFns; 234 | 235 | callback = callback || function() { }; 236 | 237 | tmpDirCleanupFns = [ 238 | async.apply(removeRF, backupDir), 239 | async.apply(removeRF, path.join(tmpDir, archiveName)) 240 | ]; 241 | 242 | async.series(tmpDirCleanupFns.concat([ 243 | async.apply(mongoDump, mongodbConfig, tmpDir), 244 | async.apply(compressDirectory, tmpDir, mongodbConfig.db, archiveName), 245 | d.bind(async.apply(sendToS3, s3Config, tmpDir, archiveName)) // this function sometimes throws EPIPE errors 246 | ]), function(err) { 247 | if(err) { 248 | log(err, 'error'); 249 | } else { 250 | log('Successfully backed up ' + mongodbConfig.db); 251 | } 252 | // cleanup folders 253 | async.series(tmpDirCleanupFns, function() { 254 | return callback(err); 255 | }); 256 | }); 257 | 258 | // this cleans up folders in case of EPIPE error from AWS connection 259 | d.on('error', function(err) { 260 | d.exit() 261 | async.series(tmpDirCleanupFns, function() { 262 | throw(err); 263 | }); 264 | }); 265 | 266 | } 267 | 268 | module.exports = { sync: sync, log: log }; 269 | --------------------------------------------------------------------------------