├── .gitignore ├── LICENSE ├── README.md ├── cli.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Andrew Kelley 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # s3 cli 2 | 3 | Command line utility frontend to [node-s3-client](https://github.com/andrewrk/node-s3-client). 4 | Inspired by [s3cmd](https://github.com/s3tools/s3cmd) and attempts to be a 5 | drop-in replacement. 6 | 7 | ## Features 8 | 9 | * Compatible with [s3cmd](https://github.com/s3tools/s3cmd)'s config file 10 | * Supports a subset of s3cmd's commands and parameters 11 | - including `put`, `get`, `del`, `ls`, `sync`, `cp`, `mv` 12 | * When syncing directories, instead of uploading one file at a time, it 13 | uploads many files in parallel resulting in more bandwidth. 14 | * Uses multipart uploads for large files and uploads each part in parallel. 15 | * Retries on failure. 16 | 17 | ## Install 18 | 19 | `sudo npm install -g s3-cli` 20 | 21 | ## Configuration 22 | 23 | s3-cli is compatible with s3cmd's config file, so if you already have that 24 | configured, you're all set. Otherwise you can put this in `~/.s3cfg`: 25 | 26 | ```ini 27 | [default] 28 | access_key = foo 29 | secret_key = bar 30 | ``` 31 | 32 | You can also point it to another config file with e.g. `$ s3-cli --config /path/to/s3cmd.conf`. 33 | 34 | ## Documentation 35 | 36 | ### put 37 | 38 | Uploads a file to S3. Assumes the target filename to be the same as the source filename (if none specified) 39 | 40 | Example: 41 | 42 | ``` 43 | s3-cli put /path/to/file s3://bucket/key/on/s3 44 | s3-cli put /path/to/source-file s3://bucket/target-file 45 | ``` 46 | 47 | Options: 48 | 49 | * `--acl-public` or `-P` - Store objects with ACL allowing read for anyone. 50 | * `--default-mime-type` - Default MIME-type for stored objects. Application 51 | default is `binary/octet-stream`. 52 | * `--no-guess-mime-type` - Don't guess MIME-type and use the default type 53 | instead. 54 | * `--add-header=NAME:VALUE` - Add a given HTTP header to the upload request. Can be 55 | used multiple times. For instance set 'Expires' or 'Cache-Control' headers 56 | (or both) using this options if you like. 57 | * `--region=REGION-NAME` - Specify the region (defaults to us-east-1) 58 | 59 | ### get 60 | 61 | Downloads a file from S3. 62 | 63 | Example: 64 | 65 | ``` 66 | s3-cli get s3://bucket/key/on/s3 /path/to/file 67 | ``` 68 | 69 | ### del 70 | 71 | Deletes an object or a directory on S3. 72 | 73 | Example: 74 | 75 | ``` 76 | s3-cli del [--recursive] s3://bucket/key/on/s3/ 77 | ``` 78 | 79 | ### ls 80 | 81 | Lists S3 objects. 82 | 83 | Example: 84 | 85 | ``` 86 | s3-cli ls [--recursive] s3://mybucketname/this/is/the/key/ 87 | ``` 88 | 89 | ### sync 90 | 91 | #### Sync a local directory to S3 92 | 93 | Example: 94 | 95 | ``` 96 | s3-cli sync [--delete-removed] /path/to/folder/ s3://bucket/key/on/s3/ 97 | ``` 98 | 99 | Supports the same options as `put`. 100 | 101 | #### Sync a directory on S3 to disk 102 | 103 | Example: 104 | 105 | ``` 106 | s3-cli sync [--delete-removed] s3://bucket/key/on/s3/ /path/to/folder/ 107 | ``` 108 | 109 | ### cp 110 | 111 | Copy an object which is already on S3. 112 | 113 | Example: 114 | 115 | ``` 116 | s3-cli cp s3://sourcebucket/source/key s3://destbucket/dest/key 117 | ``` 118 | 119 | ### mv 120 | 121 | Move an object which is already on S3. 122 | 123 | Example: 124 | 125 | ``` 126 | s3-cli mv s3://sourcebucket/source/key s3://destbucket/dest/key 127 | ``` 128 | -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var minimist = require('minimist'); 4 | var osenv = require('osenv'); 5 | var humanSize = require('human-size'); 6 | var ini = require('ini'); 7 | var fs = require('fs'); 8 | var path = require('path'); 9 | var s3 = require('s3'); 10 | var url = require('url'); 11 | var http = require('http'); 12 | var https = require('https'); 13 | var argOptions = { 14 | 'default': { 15 | 'config': path.join(osenv.home(), '.s3cfg'), 16 | 'delete-removed': false, 17 | 'max-sockets': 20, 18 | 'region': 'us-east-1', 19 | 'default-mime-type': null, 20 | 'add-header': null, 21 | }, 22 | 'boolean': [ 23 | 'recursive', 24 | 'delete-removed', 25 | 'insecure', 26 | 'acl-public', 27 | 'acl-private', 28 | 'no-guess-mime-type', 29 | ], 30 | 'alias': { 31 | 'P': 'acl-public', 32 | }, 33 | }; 34 | var args = minimist(process.argv.slice(2), argOptions); 35 | 36 | var fns = { 37 | 'sync': cmdSync, 38 | 'ls': cmdList, 39 | 'help': cmdHelp, 40 | 'del': cmdDelete, 41 | 'put': cmdPut, 42 | 'get': cmdGet, 43 | 'cp': cmdCp, 44 | 'mv': cmdMv, 45 | }; 46 | var USAGE_TEXT = 47 | "Usage: s3-cli (command) (command arguments)\n" + 48 | "Commands: " + Object.keys(fns).join(" "); 49 | var splitPathRe = /^(\/?|)([\s\S]*?)((?:\.{1,2}|[^\/]+?|)(\.[^.\/]*|))(?:[\/]*)$/; 50 | 51 | var s3UrlRe = /^[sS]3:\/\/(.*?)\/(.*)/; 52 | barfOnUnexpectedArgs(); 53 | 54 | var client; 55 | 56 | fs.readFile(args.config, {encoding: 'utf8'}, function(err, contents) { 57 | if (err) { 58 | if (process.env.AWS_SECRET_KEY && process.env.AWS_ACCESS_KEY) { 59 | setup(process.env.AWS_SECRET_KEY, process.env.AWS_ACCESS_KEY); 60 | } else { 61 | console.error("This utility needs a config file formatted the same as for s3cmd"); 62 | console.error("or AWS_SECRET_KEY and AWS_ACCESS_KEY environment variables."); 63 | process.exit(1); 64 | } 65 | return; 66 | } 67 | var config = ini.parse(contents); 68 | var accessKeyId, secretAccessKey; 69 | if (config && config.default) { 70 | accessKeyId = config.default.access_key; 71 | secretAccessKey = config.default.secret_key; 72 | } 73 | if (!secretAccessKey || !accessKeyId) { 74 | console.error("Config file missing access_key or secret_key"); 75 | process.exit(1); 76 | return; 77 | } 78 | setup(secretAccessKey, accessKeyId); 79 | }); 80 | 81 | function setup(secretAccessKey, accessKeyId) { 82 | var maxSockets = parseInt(args['max-sockets'], 10); 83 | http.globalAgent.maxSockets = maxSockets; 84 | https.globalAgent.maxSockets = maxSockets; 85 | client = s3.createClient({ 86 | s3Options: { 87 | accessKeyId: accessKeyId, 88 | secretAccessKey: secretAccessKey, 89 | sslEnabled: !args.insecure, 90 | region: args.region, 91 | }, 92 | }); 93 | var cmd = args._.shift(); 94 | var fn = fns[cmd]; 95 | if (!fn) fn = cmdHelp; 96 | fn(); 97 | } 98 | 99 | function cmdSync() { 100 | expectArgCount(2); 101 | var source = args._[0]; 102 | var dest = args._[1]; 103 | 104 | var sourceS3 = isS3Url(source); 105 | var destS3 = isS3Url(dest); 106 | 107 | var localDir, s3Url, method; 108 | var getS3Params; 109 | var s3Params = {}; 110 | if (sourceS3 && !destS3) { 111 | localDir = dest; 112 | s3Url = source; 113 | method = client.downloadDir; 114 | getS3Params = downloadGetS3Params; 115 | } else if (!sourceS3 && destS3) { 116 | localDir = source; 117 | s3Url = dest; 118 | method = client.uploadDir; 119 | s3Params.ACL = getAcl(); 120 | getS3Params = uploadGetS3Params; 121 | } else { 122 | console.error("one target must be from S3, the other must be from local file system."); 123 | process.exit(1); 124 | } 125 | var parts = parseS3Url(s3Url); 126 | s3Params.Prefix = parts.key; 127 | s3Params.Bucket = parts.bucket; 128 | 129 | parseAddHeaders(s3Params); 130 | 131 | var params = { 132 | deleteRemoved: args['delete-removed'], 133 | getS3Params: getS3Params, 134 | localDir: localDir, 135 | s3Params: s3Params, 136 | defaultContentType: getDefaultContentType(), 137 | }; 138 | var syncer = method.call(client, params); 139 | setUpProgress(syncer); 140 | } 141 | 142 | function uploadGetS3Params(filePath, stat, callback) { 143 | //console.error("Uploading", filePath); 144 | callback(null, { 145 | ContentType: getContentType(), 146 | }); 147 | } 148 | 149 | function downloadGetS3Params(filePath, s3Object, callback) { 150 | //console.error("Downloading", filePath); 151 | callback(null, {}); 152 | } 153 | 154 | function cmdList() { 155 | expectArgCount(1); 156 | var recursive = args.recursive; 157 | var s3Url = args._[0]; 158 | var parts = parseS3Url(s3Url); 159 | var params = { 160 | recursive: recursive, 161 | s3Params: { 162 | Bucket: parts.bucket, 163 | Prefix: parts.key, 164 | Delimiter: recursive ? null : '/', 165 | }, 166 | }; 167 | var finder = client.listObjects(params); 168 | finder.on('data', function(data) { 169 | data.CommonPrefixes.forEach(function(dirObject) { 170 | console.log("DIR " + dirObject.Prefix); 171 | }); 172 | data.Contents.forEach(function(object) { 173 | console.log(object.LastModified + " " + object.Size + " " + object.Key); 174 | }); 175 | }); 176 | finder.on('error', function(err) { 177 | console.error("Error: " + err.message); 178 | process.exit(1); 179 | }); 180 | } 181 | 182 | function cmdDelete() { 183 | expectArgCount(1); 184 | var parts = parseS3Url(args._[0]); 185 | if (args.recursive) { 186 | doDeleteDir(); 187 | } else { 188 | doDeleteObject(); 189 | } 190 | 191 | function doDeleteDir() { 192 | var params = { 193 | Bucket: parts.bucket, 194 | Prefix: parts.key, 195 | }; 196 | var deleter = client.deleteDir(params); 197 | setUpProgress(deleter, true); 198 | } 199 | 200 | function doDeleteObject() { 201 | var params = { 202 | Bucket: parts.bucket, 203 | Delete: { 204 | Objects: [ 205 | { 206 | Key: parts.key, 207 | }, 208 | ], 209 | } 210 | }; 211 | var deleter = client.deleteObjects(params); 212 | deleter.on('error', function(err) { 213 | console.error("Error: " + err.message); 214 | process.exit(1); 215 | }); 216 | } 217 | } 218 | 219 | function cmdPut() { 220 | expectArgCount(2); 221 | var source = args._[0]; 222 | var dest = args._[1]; 223 | var parts = parseS3Url(dest); 224 | if (/\/$/.test(parts.key) || parts.key == '') { 225 | parts.key += path.basename(source); 226 | } 227 | var acl = getAcl(); 228 | var s3Params = { 229 | Bucket: parts.bucket, 230 | Key: parts.key, 231 | ACL: acl, 232 | ContentType: getContentType(), 233 | }; 234 | parseAddHeaders(s3Params); 235 | var params = { 236 | localFile: source, 237 | s3Params: s3Params, 238 | defaultContentType: getDefaultContentType(), 239 | }; 240 | var uploader = client.uploadFile(params); 241 | var doneText; 242 | if (acl === 'public-read') { 243 | var publicUrl = args.insecure ? 244 | s3.getPublicUrlHttp(parts.bucket, parts.key) : 245 | s3.getPublicUrl(parts.bucket, parts.key, args.region); 246 | doneText = "Public URL: " + publicUrl; 247 | } else { 248 | doneText = "done"; 249 | } 250 | setUpProgress(uploader, false, doneText); 251 | } 252 | 253 | function cmdGet() { 254 | expectArgCount(1, 2); 255 | var source = args._[0]; 256 | var dest = args._[1]; 257 | var parts = parseS3Url(source); 258 | if (!dest) { 259 | dest = unixBasename(source); 260 | } else if (dest[dest.length - 1] === path.sep) { 261 | dest = path.join(dest, unixBasename(source)); 262 | } 263 | 264 | var params = { 265 | localFile: dest, 266 | s3Params: { 267 | Bucket: parts.bucket, 268 | Key: parts.key, 269 | }, 270 | }; 271 | var downloader = client.downloadFile(params); 272 | setUpProgress(downloader); 273 | } 274 | 275 | function cmdCp() { 276 | expectArgCount(2); 277 | var source = args._[0]; 278 | var dest = args._[1]; 279 | var sourceParts = parseS3Url(source); 280 | var destParts = parseS3Url(dest); 281 | 282 | var s3Params = { 283 | CopySource: sourceParts.bucket + '/' + sourceParts.key, 284 | Bucket: destParts.bucket, 285 | Key: destParts.key, 286 | }; 287 | 288 | var copier = client.copyObject(s3Params); 289 | copier.on('error', function(err) { 290 | console.error("Error: " + err.message); 291 | process.exit(1); 292 | }); 293 | } 294 | 295 | function cmdMv() { 296 | expectArgCount(2); 297 | var source = args._[0]; 298 | var dest = args._[1]; 299 | var sourceParts = parseS3Url(source); 300 | var destParts = parseS3Url(dest); 301 | 302 | var s3Params = { 303 | CopySource: sourceParts.bucket + '/' + sourceParts.key, 304 | Bucket: destParts.bucket, 305 | Key: destParts.key, 306 | }; 307 | 308 | var mover = client.moveObject(s3Params); 309 | mover.on('error', function(err) { 310 | console.error("Error: " + err.message); 311 | process.exit(1); 312 | }); 313 | } 314 | 315 | function cmdHelp() { 316 | console.log(USAGE_TEXT); 317 | } 318 | 319 | function parseS3Url(s3Url) { 320 | if (!s3Url) { 321 | console.error("Expected S3 URL argument"); 322 | process.exit(1); 323 | } 324 | var match = s3Url.match(s3UrlRe); 325 | if (!match) { 326 | console.error("Not a valid S3 URL:", s3Url); 327 | process.exit(1); 328 | } 329 | return { 330 | bucket: match[1], 331 | key: match[2], 332 | }; 333 | } 334 | 335 | function isS3Url(str) { 336 | return s3UrlRe.test(str); 337 | } 338 | 339 | function getContentType() { 340 | return args['no-guess-mime-type'] ? null : undefined; 341 | } 342 | 343 | function getDefaultContentType() { 344 | return args['default-mime-type'] || null; 345 | } 346 | 347 | function getAcl() { 348 | var acl = null; 349 | if (args['acl-public']) { 350 | acl = 'public-read'; 351 | } else if (args['acl-private']) { 352 | acl = 'private'; 353 | } 354 | return acl; 355 | } 356 | 357 | function setUpProgress(o, notBytes, doneText) { 358 | var start = null; 359 | doneText = doneText || "done"; 360 | var printFn = process.stderr.isTTY ? printProgress : noop; 361 | printFn(); 362 | var progressInterval = setInterval(printFn, 100); 363 | o.on('end', function() { 364 | clearInterval(progressInterval); 365 | process.stderr.write("\n" + doneText + "\n"); 366 | }); 367 | o.on('error', function(err) { 368 | clearInterval(progressInterval); 369 | process.stderr.write("\nError: " + err.message + "\n"); 370 | process.exit(1); 371 | }); 372 | 373 | function printProgress() { 374 | var percent = Math.floor(o.progressAmount / o.progressTotal * 100); 375 | var amt = notBytes ? String(o.progressAmount) : fmtBytes(o.progressAmount); 376 | var total = notBytes ? String(o.progressTotal) : fmtBytes(o.progressTotal); 377 | var parts = []; 378 | if (o.filesFound > 0 && !o.doneFindingFiles) { 379 | parts.push(o.filesFound + " files"); 380 | } 381 | if (o.objectsFound > 0 && !o.doneFindingObjects) { 382 | parts.push(o.objectsFound + " objects"); 383 | } 384 | if (o.deleteTotal > 0) { 385 | parts.push(o.deleteAmount + "/" + o.deleteTotal + " deleted"); 386 | } 387 | if (o.progressMd5Amount > 0 && !o.doneMd5) { 388 | parts.push(fmtBytes(o.progressMd5Amount) + "/" + fmtBytes(o.progressMd5Total) + " MD5"); 389 | } 390 | if (o.progressTotal > 0) { 391 | if (!start) start = new Date(); 392 | var part = amt + "/" + total; 393 | if (!isNaN(percent)) part += " " + percent + "% done"; 394 | parts.push(part); 395 | if (!notBytes) { 396 | var now = new Date(); 397 | var seconds = (now - start) / 1000; 398 | var bytesPerSec = o.progressAmount / seconds; 399 | var humanSpeed = fmtBytes(bytesPerSec) + '/s'; 400 | parts.push(humanSpeed); 401 | } 402 | } 403 | var line = parts.join(", "); 404 | process.stderr.clearLine(); 405 | process.stderr.cursorTo(0); 406 | process.stderr.write(line); 407 | } 408 | } 409 | 410 | function parseAddHeaders(s3Params) { 411 | var addHeaders = args['add-header']; 412 | if (addHeaders) { 413 | if (Array.isArray(addHeaders)) { 414 | addHeaders.forEach(handleAddHeader); 415 | } else { 416 | handleAddHeader(addHeaders); 417 | } 418 | } 419 | function handleAddHeader(header) { 420 | var match = header.match(/^(.*):\s*(.*)$/); 421 | if (!match) { 422 | console.error("Improperly formatted header:", header); 423 | process.exit(1); 424 | } 425 | var headerName = match[1]; 426 | var paramName = headerName.replace(/-/g, ''); 427 | var paramValue = match[2]; 428 | s3Params[paramName] = paramValue; 429 | } 430 | } 431 | 432 | function fmtBytes(byteCount) { 433 | if (byteCount <= 0) { 434 | return "0 B"; 435 | } else { 436 | return humanSize(byteCount, 1); 437 | } 438 | } 439 | 440 | function noop() {} 441 | 442 | function barfOnUnexpectedArgs() { 443 | var validArgs = {'_': true}; 444 | addValid(Object.keys(argOptions.default)); 445 | addValid(Object.keys(argOptions.alias)); 446 | addValid(argOptions.boolean); 447 | 448 | var invalidArgs = []; 449 | for (var argName in args) { 450 | if (!validArgs[argName]) { 451 | invalidArgs.push(argName); 452 | } 453 | } 454 | 455 | if (invalidArgs.length) { 456 | console.error(USAGE_TEXT); 457 | console.error("Unrecognized option(s): " + invalidArgs.join(", ")); 458 | process.exit(1); 459 | } 460 | 461 | function addValid(array) { 462 | array.forEach(function(name) { 463 | validArgs[name] = true; 464 | }); 465 | } 466 | } 467 | 468 | function expectArgCount(min, max) { 469 | if (max == null) max = min; 470 | if (args._.length < min) { 471 | console.error("Expected at least " + min + " arguments, got " + args._.length); 472 | process.exit(1); 473 | } 474 | if (args._.length > max) { 475 | console.error("Expected at most " + max + " arguments, got " + args._.length); 476 | process.exit(1); 477 | } 478 | } 479 | 480 | // copied from Node.js path module for unix only 481 | function unixSplitPath(filename) { 482 | return splitPathRe.exec(filename).slice(1); 483 | } 484 | function unixBasename(path) { 485 | return unixSplitPath(path)[2]; 486 | } 487 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "s3-cli", 3 | "version": "0.13.0", 4 | "description": "command line utility to go along with s3 module", 5 | "main": "index.js", 6 | "bin": { 7 | "s3-cli": "./cli.js" 8 | }, 9 | "author": "Andrew Kelley ", 10 | "license": "MIT", 11 | "dependencies": { 12 | "ini": "~1.2.1", 13 | "minimist": "~1.1.0", 14 | "osenv": "~0.1.0", 15 | "s3": "~4.2.0", 16 | "human-size": "~1.1.0" 17 | }, 18 | "devDependencies": {}, 19 | "scripts": { 20 | "test": "echo \"Error: no test specified\" && exit 1" 21 | }, 22 | "repository": { 23 | "type": "git", 24 | "url": "git://github.com/andrewrk/node-s3-cli.git" 25 | }, 26 | "bugs": { 27 | "url": "https://github.com/andrewrk/node-s3-cli/issues" 28 | }, 29 | "homepage": "https://github.com/andrewrk/node-s3-cli" 30 | } 31 | --------------------------------------------------------------------------------