├── .editorconfig ├── encrypt-test-credentials.js ├── .gitignore ├── .travis.yml ├── .travis.yml.tpl ├── standalone ├── async-reduce.js ├── recompile-travis-yml-file.js ├── encrypt-and-recompile-for-travis.js └── encrypt-for-travis.js ├── LICENSE ├── package.json ├── .jshintrc ├── README.md ├── ROADMAP.md └── index.js /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true -------------------------------------------------------------------------------- /encrypt-test-credentials.js: -------------------------------------------------------------------------------- 1 | // To use this file, run: 2 | // 3 | // key=YOUR_AWS_KEY_HERE secret=YOUR_AWS_SECRET_HERE node encrypt-test-credentials 4 | // 5 | // This will recompile the .travis.yml file using 6 | 7 | var buildYml = require('./standalone/encrypt-and-recompile-for-travis'); 8 | buildYml({ 9 | repo: 'balderdashy/skipper-s3', 10 | envVars: { 11 | key: process.env.KEY, 12 | secret: process.env.SECRET 13 | } 14 | }); 15 | 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # Compiled binary addons (http://nodejs.org/api/addons.html) 20 | build/Release 21 | 22 | # Dependency directory 23 | # Deployed apps should consider commenting this line out: 24 | # see https://npmjs.org/doc/faq.html#Should-I-check-my-node_modules-folder-into-git 25 | node_modules 26 | 27 | .tmp 28 | .DS_Store 29 | 30 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" 4 | 5 | env: 6 | global: 7 | - secure: "Ju77ouEvmRzOi68q6Qws6OMaVdGlVO6QJt/1Z3UoJbEbnaBQmkaY0LRK9xyl6i8HK67pNErxMtosE+jrZU8Jd0RfHHQxSyVhLGVRnWvX9KS+DVV/s+4buZGAp1IoZDkzE88M0jLVsYuWTADe6XWPnI+V9YAoLptQ/e4vdZR7aQc=" 8 | 9 | 10 | # ~*** -------------------------------------------------> 11 | # 12 | # IMPORTANT NOTE: 13 | # 14 | # Please don't change this file manually! 15 | # It is automatically compiled from `.travis.yml.tpl` 16 | # in order to encrypt sensitive information using the 17 | # repo public key. 18 | # 19 | # Change `.travis.yml.tpl` instead. 20 | # 21 | # <--------------------------------------------------- ***~ 22 | 23 | 24 | -------------------------------------------------------------------------------- /.travis.yml.tpl: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" 4 | 5 | env: 6 | global: 7 | - secure: "<%-secure%>" 8 | 9 | 10 | # ~*** -------------------------------------------------> 11 | # 12 | # IMPORTANT NOTE: 13 | # 14 | # Please don't change this file manually! 15 | # It is automatically compiled from `.travis.yml.tpl` 16 | # in order to encrypt sensitive information using the 17 | # repo public key. 18 | # 19 | # Change `.travis.yml.tpl` instead. 20 | # 21 | # <--------------------------------------------------- ***~ 22 | 23 | <% 24 | // Just kidding about all that ^^^^ 25 | // Actually this is the template file so you SHOULD change it. 26 | // The warning below is only here because we need to copy it over every time 27 | // .travis.yml is recompiled. 28 | %> 29 | -------------------------------------------------------------------------------- /standalone/async-reduce.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module dependencies 3 | */ 4 | 5 | var async = require('async'); 6 | var _ = require('lodash'); 7 | 8 | 9 | 10 | 11 | 12 | /** 13 | * [asyncReduce description] 14 | * @param {Object|Array} collection 15 | * @param {Function(memo,value,key,cb)} iteratorFn 16 | * @param {*} initialMemo 17 | * @param {Function(err,result)} afterwards 18 | */ 19 | 20 | module.exports = function asyncReduce(collection, iteratorFn, initialMemo, afterwards){ 21 | var memo = initialMemo; 22 | async.eachSeries(_.keys(collection), function (key, next){ 23 | iteratorFn(memo, collection[key], key, function (err, _updatedMemo) { 24 | if (err) return next(err); 25 | memo = _updatedMemo; 26 | next(); 27 | }); 28 | }, function (err) { 29 | return afterwards(err, memo); 30 | }); 31 | } 32 | 33 | -------------------------------------------------------------------------------- /standalone/recompile-travis-yml-file.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module dependencies 3 | */ 4 | 5 | var util = require('util'); 6 | var fsx = require('fs-extra'); 7 | var _ = require('lodash'); 8 | 9 | 10 | 11 | 12 | /** 13 | * Recompile the travis yml file. 14 | */ 15 | 16 | module.exports = function recompileTravisYmlFile(options, cb){ 17 | options = options || {}; 18 | 19 | options.outputPath = options.outputPath || '.travis.yml'; 20 | options.tplPath = options.tplPath || '.travis.yml.tpl'; 21 | options.locals = options.locals || {}; 22 | 23 | try { 24 | 25 | // Now if `.travis.yml.tpl` exists, write the encrypted data 26 | // to the travis file. 27 | if (!fsx.existsSync(options.tplPath)) { 28 | return cb(new Error(util('Could not find template file for .travis.yml at %s', options.tplPath))); 29 | } 30 | 31 | fsx.writeFileSync(options.outputPath, 32 | _.template( 33 | fsx.readFileSync(options.tplPath), 34 | options.locals 35 | ) 36 | ); 37 | } 38 | catch (e) { return cb(e); } 39 | 40 | return cb(); 41 | }; 42 | 43 | -------------------------------------------------------------------------------- /standalone/encrypt-and-recompile-for-travis.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module dependencies 3 | */ 4 | 5 | var encryptForTravis = require('./encrypt-for-travis'); 6 | var recompileTravisYmlFile = require('./recompile-travis-yml-file'); 7 | 8 | 9 | /** 10 | * [encryptAndRecompileForTravis description] 11 | * @param {[type]} options [description] 12 | * @param {Function} cb [description] 13 | * @return {[type]} [description] 14 | */ 15 | 16 | module.exports = function encryptAndRecompileForTravis (options, cb) { 17 | options = options || {}; 18 | cb = cb || function noOp(err){ if (err) throw err; }; 19 | 20 | var repo = options.repo || 'balderdashy/sails'; 21 | var envVars = options.envVars || {}; 22 | 23 | encryptForTravis({ 24 | repo: repo, 25 | envVars: envVars 26 | }, function (err, encryptedEnvVars) { 27 | if (err) return cb(err); 28 | recompileTravisYmlFile({ 29 | locals: encryptedEnvVars 30 | }, function (err) { 31 | if (err) return cb(err); 32 | return cb(null, encryptedEnvVars); 33 | }); 34 | }); 35 | }; 36 | 37 | 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Mike McNeil 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "skipper-s3", 3 | "version": "0.5.5", 4 | "description": "Streaming file uploads to S3", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "node -e \"require('skipper-adapter-tests')({mocha: { timeout: 360000 }, module: require('./'), bucket:process.env.BUCKET || 'skipper-adapter-tests', key: process.env.KEY, secret: process.env.SECRET});\"" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git://github.com/balderdashy/skipper-s3.git" 12 | }, 13 | "keywords": [ 14 | "s3", 15 | "streaming-file-upload", 16 | "streaming", 17 | "file", 18 | "upload", 19 | "sails", 20 | "express" 21 | ], 22 | "author": "Mike McNeil", 23 | "license": "MIT", 24 | "bugs": { 25 | "url": "https://github.com/balderdashy/skipper-s3/issues" 26 | }, 27 | "homepage": "https://github.com/balderdashy/skipper-s3", 28 | "devDependencies": { 29 | "skipper-adapter-tests": "~1.5.2", 30 | "travis-encrypt": "~1.0.1" 31 | }, 32 | "dependencies": { 33 | "lodash": "~2.4.1", 34 | "fs-extra": "~0.17.0", 35 | "knox-mpu-alt": "https://github.com/alexyoung/knox-mpu/tarball/master", 36 | "knox": "~0.9.2", 37 | "merge-defaults": "~0.1.0", 38 | "concat-stream": "~1.4.5", 39 | "s3-lister": "~0.1.0", 40 | "mime": "~1.3.4" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | // To fix column positions for JSHint errors you may want to add `"indent": 1` to your 3 | // **User** "jshint_options". This issue affects users with tabs for indentation. 4 | // This fix was reverted due to a conflict with using the `"white": true` option. 5 | // "indent": 1, 6 | "evil": true, 7 | "regexdash": true, 8 | "browser": true, 9 | "wsh": true, 10 | "sub": true, 11 | 12 | // Suppress warnings about mixed tabs and spaces 13 | "smarttabs": true, 14 | 15 | // Suppress warnings about trailing whitespace 16 | "trailing": false, 17 | 18 | // Suppress warnings about the use of expressions where fn calls or assignments are expected 19 | "expr": true, 20 | 21 | // Suppress warnings about using functions inside loops (useful for inifinityCounters) 22 | "loopfunc": true, 23 | 24 | // Suppress warnings about using assignments where conditionals are expected 25 | "boss": true, 26 | 27 | // Suppress warnings about "weird constructions" 28 | // i.e. allow code like: 29 | // (new (function OneTimeUsePrototype () { } )) 30 | "supernew": true, 31 | 32 | // Allow backwards, node-dependency-style commas 33 | "laxcomma": true 34 | 35 | // "bitwise": true, 36 | // "camelcase": true, 37 | // "node": true, 38 | // "undef": true, 39 | // "unused": true, 40 | // "curly": true, 41 | // "immed": true, 42 | // "latedef": true, 43 | // "noarg": true, 44 | // "noempty": true, 45 | // "plusplus": true, 46 | // "quotmark": "single", 47 | // "trailing": true, 48 | // "asi": false, 49 | // "eqnull": true, 50 | // "eval": true, 51 | // "sub": true, 52 | // "supernew": true, 53 | // "eqeqeq": true, 54 | // "eqnull": true 55 | } 56 | -------------------------------------------------------------------------------- /standalone/encrypt-for-travis.js: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * Module dependencies 4 | */ 5 | 6 | var encrypt = require('travis-encrypt'); 7 | var _ = require('lodash'); 8 | 9 | 10 | 11 | /** 12 | * [encryptForTravis description] 13 | * @param {[type]} options [description] 14 | * @param {Function} cb [description] 15 | * @return {[type]} [description] 16 | */ 17 | 18 | module.exports = function encryptForTravis (options, cb) { 19 | options = options || {}; 20 | var repo = options.repo || 'balderdashy/sails'; 21 | var envVars = options.envVars || {}; 22 | 23 | // Build string like "FOO=bar BAZ=boo" 24 | var envVarString = _.reduce(envVars, function (memo, value, envVarName){ 25 | return envVarName.toUpperCase() + '=' + value +' ' + memo; 26 | }, ''); 27 | // Trim trailing whitespace 28 | envVarString = envVarString.replace(/\s*$/, ''); 29 | 30 | encrypt(repo, envVarString, undefined, undefined, function (err, encryptedEnvVars) { 31 | if (err) { 32 | var e = new Error('Failed to encrypt environment variable values'); 33 | e.code = 'E_FAILED_TO_ENCRYPT'; 34 | e.message = typeof err == 'object' ? (err.message || err) : err; 35 | e.stack = typeof err == 'object' ? (err.stack || err) : err; 36 | return cb(e); 37 | } 38 | 39 | return cb(undefined, {secure: encryptedEnvVars}); 40 | }); 41 | }; 42 | 43 | 44 | 45 | 46 | // Based on the suggestions here: 47 | // • http://docs.travis-ci.com/user/travis-pro/ 48 | // 49 | // and using the module here: 50 | // • https://www.npmjs.org/package/travis-encrypt 51 | 52 | // asyncReduce(envVars, function (memo, value, envVarName, next) { 53 | 54 | // encrypt(REPO, envVarName.toUpperCase()+'='+value, undefined, undefined, function (err, blob) { 55 | // if (err) return next(err); 56 | 57 | // memo[envVarName] = blob; 58 | // return next(null, memo); 59 | // }); 60 | // }, {}, function (err, encryptedEnvVars) { 61 | // if (err) { 62 | // var e = new Error('Failed to encrypt environment variable values'); 63 | // e.code = 'E_FAILED_TO_ENCRYPT'; 64 | // e.message = typeof err == 'object' ? (err.message || err) : err; 65 | // e.stack = typeof err == 'object' ? (err.stack || err) : err; 66 | // return cb(e); 67 | // } 68 | 69 | // return cb(undefined, encryptedEnvVars); 70 | // }); 71 | 72 | 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [skipper emblem - face of a ship's captain](https://github.com/balderdashy/skipper-s3) S3 Blob Adapter 2 | 3 | [![NPM version](https://badge.fury.io/js/skipper-s3.png)](http://badge.fury.io/js/skipper-s3)     4 | [![Build Status](https://travis-ci.org/balderdashy/skipper-s3.svg?branch=master)](https://travis-ci.org/balderdashy/skipper-s3) 5 | 6 | S3 adapter for receiving [upstreams](https://github.com/balderdashy/skipper#what-are-upstreams). Particularly useful for handling streaming multipart file uploads from the [Skipper](https://github.com/balderdashy/skipper) body parser. 7 | 8 | 9 | ## Installation 10 | 11 | ``` 12 | $ npm install skipper-s3 --save 13 | ``` 14 | 15 | Also make sure you have skipper itself [installed as your body parser](http://beta.sailsjs.org/#/documentation/concepts/Middleware?q=adding-or-overriding-http-middleware). This is the default configuration in [Sails](https://github.com/balderdashy/sails) as of v0.10. 16 | 17 | 18 | ## Usage 19 | 20 | ```javascript 21 | req.file('avatar') 22 | .upload({ 23 | adapter: require('skipper-s3'), 24 | key: 'thekyehthethaeiaghadkthtekey' 25 | secret: 'AB2g1939eaGAdesoccertournament' 26 | bucket: 'my_stuff' 27 | }, function whenDone(err, uploadedFiles) { 28 | if (err) return res.negotiate(err); 29 | else return res.ok({ 30 | files: uploadedFiles, 31 | textParams: req.params.all() 32 | }); 33 | }); 34 | ``` 35 | 36 | 37 | For more detailed usage information and a full list of available options, see the Skipper docs, especially the section on "[https://github.com/balderdashy/skipper#uploading-files-to-s3](Uploading to S3)". 38 | 39 | 40 | ## Contribute 41 | 42 | See [ROADMAP.md](https://github.com/balderdashy/skipper-s3/blob/master/ROADMAP.md). 43 | 44 | Also be sure to check out [ROADMAP.md in the Skipper repo](https://github.com/balderdashy/skipper/blob/master/ROADMAP.md). 45 | 46 | To run the tests: 47 | 48 | ```sh 49 | git clone git@github.com:balderdashy/skipper-s3.git 50 | cd skipper-s3 51 | npm install 52 | KEY= your_aws_access_key SECRET=your_aws_access_secret BUCKET=your_s3_bucket npm test 53 | ``` 54 | 55 | Please don't check in your aws credentials :) 56 | 57 | 58 | ## License 59 | 60 | **[MIT](./LICENSE)** 61 | © 2013, 2014- 62 | 63 | [Mike McNeil](http://michaelmcneil.com), [Balderdash](http://balderdash.co) & contributors 64 | 65 | See `LICENSE.md`. 66 | 67 | This module is part of the [Sails framework](http://sailsjs.org), and is free and open-source under the [MIT License](http://sails.mit-license.org/). 68 | 69 | 70 | ![image_squidhome@2x.png](http://i.imgur.com/RIvu9.png) 71 | 72 | 73 | [![githalytics.com alpha](https://cruel-carlota.pagodabox.com/a22d3919de208c90c898986619efaa85 "githalytics.com")](http://githalytics.com/balderdashy/skipper-s3) 74 | -------------------------------------------------------------------------------- /ROADMAP.md: -------------------------------------------------------------------------------- 1 | # Roadmap 2 | 3 | The build status, immediate-term plans, and future goals of this repository. 4 | 5 | > ###### Feature Requests 6 | > 7 | > We welcome feature requests as edits to the "Backlog" section below. 8 | > 9 | > Before editing this file, please check out [How To Contribute to ROADMAP.md](https://gist.github.com/mikermcneil/bdad2108f3d9a9a5c5ed)- it's a quick read :) 10 | 11 | 12 | ## Current Build Status 13 | 14 | The current Travis test output for this repository. 15 | 16 | | Release | Install Command | Build Status 17 | |------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------- | ----------------- 18 | | [![NPM version](https://badge.fury.io/js/skipper-s3.png)](https://github.com/balderdashy/skipper-s3/tree/stable) _(stable)_ | `npm install skipper-s3` | [![Build Status](https://travis-ci.org/balderdashy/skipper-s3.png?branch=stable)](https://travis-ci.org/balderdashy/skipper-s3) | 19 | | [edge](https://github.com/balderdashy/skipper-s3/tree/master) | `npm install skipper-s3@git://github.com/balderdashy/skipper-s3.git` | [![Build Status](https://travis-ci.org/balderdashy/skipper-s3.png?branch=master)](https://travis-ci.org/balderdashy/skipper-s3) | 20 | 21 | 22 | ## Roadmap 23 | 24 | Our short-to-medium-term roadmap items, in order of descending priority: 25 | 26 | _(feel free to suggest things)_ 27 | 28 | Feature | Owner | Details 29 | :------------------------------------------------------- | :------------------------------------------------------------------------------- | :------ 30 | explore alternatives to knox | [@mikermcneil](https://github.com/mikermcneil) | explore a migration from knox-mpu+knox to one of the other S3 node modules which is more actively maintained. 31 | 32 | 33 | #### Backlog 34 | 35 | The backlog consists of features which are not currently in the immediate-term roadmap above, but are useful. We would exuberantly accept a pull request implementing any of the items below, so long as it was accompanied with reasonable tests that prove it, and it doesn't break other core functionality. 36 | 37 | Feature | Owner | Details 38 | :---------------------------------------------- | :------------------------------------------------------------------------------- | :------ 39 | support customizable ACL parameters in options | [@abrantes01](https://github.com/abrantes01) | ability to attach ACL parameters in order to allow public access to files (See example [here](https://github.com/balderdashy/skipper-s3/issues/1) 40 | 41 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module dependencies 3 | */ 4 | 5 | var path = require('path'); 6 | var Writable = require('stream').Writable; 7 | var Transform = require('stream').Transform; 8 | var concat = require('concat-stream'); 9 | var _ = require('lodash'); 10 | _.defaultsDeep = require('merge-defaults'); 11 | var knox = require('knox'); 12 | var S3MultipartUpload = require('knox-mpu-alt'); 13 | var S3Lister = require('s3-lister'); 14 | var mime = require('mime'); 15 | 16 | /** 17 | * skipper-s3 18 | * 19 | * @param {Object} globalOpts 20 | * @return {Object} 21 | */ 22 | 23 | module.exports = function SkipperS3 (globalOpts) { 24 | globalOpts = globalOpts || {}; 25 | 26 | console.log('S3 adapter was instantiated...'); 27 | 28 | 29 | var adapter = { 30 | 31 | read: function (fd, cb) { 32 | 33 | var prefix = fd; 34 | 35 | var client = knox.createClient({ 36 | key: globalOpts.key, 37 | secret: globalOpts.secret, 38 | bucket: globalOpts.bucket, 39 | region: globalOpts.region||undefined, 40 | endpoint: globalOpts.endpoint||undefined 41 | }); 42 | 43 | // Build a noop transform stream that will pump the S3 output through 44 | var __transform__ = new Transform(); 45 | __transform__._transform = function (chunk, encoding, callback) { 46 | return callback(null, chunk); 47 | }; 48 | 49 | client.get(prefix).on('response', function(s3res){ 50 | // Handle explicit s3res errors 51 | s3res.once('error', function (err) { 52 | __transform__.emit('error', err); 53 | }); 54 | 55 | // check whether we got an actual file stream: 56 | if (s3res.statusCode < 300) { 57 | s3res.pipe(__transform__); 58 | } 59 | // or an error: 60 | else { 61 | // Wait for the body of the error message to stream in: 62 | var body = ''; 63 | s3res.setEncoding('utf8'); 64 | s3res.on('readable', function (){ 65 | var chunk = s3res.read(); 66 | if (typeof chunk === 'string') body += chunk; 67 | }); 68 | // Then build the error and emit it 69 | s3res.once('end', function () { 70 | var err = new Error(); 71 | err.status = s3res.statusCode; 72 | err.headers = s3res.headers; 73 | err.message = 'Non-200 status code returned from S3 for requested file.'; 74 | if (body) err.message += ('\n'+body); 75 | __transform__.emit('error', err); 76 | }); 77 | } 78 | }) 79 | .end(); 80 | 81 | if (cb) { 82 | var firedCb; 83 | __transform__.once('error', function (err) { 84 | if (firedCb) return; 85 | firedCb = true; 86 | cb(err); 87 | }); 88 | __transform__.pipe(concat(function (data) { 89 | if (firedCb) return; 90 | firedCb = true; 91 | cb(null, data); 92 | })); 93 | } 94 | 95 | return __transform__; 96 | }, 97 | 98 | rm: function (fd, cb){ 99 | return cb(new Error('TODO')); 100 | }, 101 | ls: function (dirname, cb) { 102 | var client = knox.createClient({ 103 | key: globalOpts.key, 104 | secret: globalOpts.secret, 105 | bucket: globalOpts.bucket, 106 | region: globalOpts.region, 107 | endpoint: globalOpts.endpoint 108 | }); 109 | 110 | // TODO: take a look at maxKeys 111 | // https://www.npmjs.org/package/s3-lister 112 | 113 | // Allow empty dirname (defaults to `/`) 114 | if (!dirname) { 115 | prefix='/'; 116 | } 117 | else prefix = dirname; 118 | 119 | // Strip leading slash from dirname to form prefix 120 | var prefix = dirname.replace(/^\//, ''); 121 | 122 | var lister = new S3Lister(client, { 123 | prefix : prefix 124 | }); 125 | 126 | if (!cb) { 127 | return lister; 128 | } 129 | else { 130 | var firedCb; 131 | lister.once('error', function (err) { 132 | if(firedCb)return; 133 | firedCb=true; 134 | cb(err); 135 | }); 136 | lister.pipe(concat(function (data) { 137 | if(firedCb)return; 138 | firedCb=true; 139 | 140 | // Pluck just the "Key" (i.e. file path) 141 | // and return only the filename (i.e. snip 142 | // off the path prefix) 143 | data = _.pluck(data, 'Key'); 144 | data = _.map(data, function snipPathPrefixes (thisPath) { 145 | thisPath = thisPath.replace(/^.*[\/]([^\/]*)$/, '$1'); 146 | 147 | // Join the dirname with the filename 148 | thisPath = path.join(dirname, path.basename(thisPath)); 149 | 150 | return thisPath; 151 | }); 152 | 153 | 154 | 155 | console.log('______ files _______\n', data); 156 | cb(null, data); 157 | })); 158 | 159 | // TODO: marshal each matched file in the stream 160 | // (using a Transform- take a look at all the 161 | // "plucking" and stuff I have going on above ^) 162 | return lister; 163 | } 164 | }, 165 | 166 | receive: S3Receiver 167 | }; 168 | 169 | return adapter; 170 | 171 | /** 172 | * A simple receiver for Skipper that writes Upstreams to 173 | * S3 to the configured bucket at the configured path. 174 | * 175 | * Includes a garbage-collection mechanism for failed 176 | * uploads. 177 | * 178 | * @param {Object} options 179 | * @return {Stream.Writable} 180 | */ 181 | function S3Receiver (options) { 182 | console.log('`.receive()` was called...'); 183 | options = options || {}; 184 | options = _.defaults(options, globalOpts); 185 | 186 | var receiver__ = Writable({ 187 | objectMode: true 188 | }); 189 | 190 | receiver__.once('error', function (err) { 191 | console.log('ERROR ON RECEIVER__ ::',err); 192 | }); 193 | 194 | // This `_write` method is invoked each time a new file is received 195 | // from the Readable stream (Upstream) which is pumping filestreams 196 | // into this receiver. (filename === `__newFile.filename`). 197 | receiver__._write = function onFile(__newFile, encoding, next) { 198 | 199 | var startedAt = new Date(); 200 | 201 | __newFile.once('error', function (err) { 202 | console.log('ERROR ON file read stream in receiver (%s) ::', __newFile.filename, err); 203 | // TODO: the upload has been cancelled, so we need to stop writing 204 | // all buffered bytes, then call gc() to remove the parts of the file that WERE written. 205 | // (caveat: may not need to actually call gc()-- need to see how this is implemented 206 | // in the underlying knox-mpu module) 207 | // 208 | // Skipper core should gc() for us. 209 | }); 210 | 211 | // Allow `tmpdir` for knox-mpu to be passed in, or default 212 | // to `.tmp/s3-upload-part-queue` 213 | options.tmpdir = options.tmpdir || path.resolve(process.cwd(), '.tmp/s3-upload-part-queue'); 214 | 215 | var headers = options.headers || {}; 216 | 217 | // Lookup content type with mime if not set 218 | if ('undefined' === typeof headers['content-type']) { 219 | headers['content-type'] = mime.lookup(__newFile.fd); 220 | console.log('[skipper-s3] Looked up mime:', headers['content-type']); 221 | } 222 | 223 | console.log('[skipper-s3] options.headers', options.headers); 224 | 225 | var useNoDisk = true; /* if options.headers.size > whatever */ 226 | 227 | var mpu = new S3MultipartUpload({ 228 | objectName: __newFile.fd, 229 | stream: __newFile, 230 | maxUploadSize: options.maxBytes, 231 | tmpDir: options.tmpdir, 232 | headers: headers, 233 | noDisk: useNoDisk, 234 | client: knox.createClient({ 235 | key: options.key, 236 | secret: options.secret, 237 | bucket: options.bucket, 238 | region: globalOpts.region||undefined, 239 | endpoint: globalOpts.endpoint||undefined 240 | }) 241 | }, function (err, body) { 242 | if (err) { 243 | console.log(('Receiver: Error writing `' + __newFile.filename + '`:: ' + require('util').inspect(err) + ' :: Cancelling upload and cleaning up already-written bytes...').red); 244 | receiver__.emit('error', err); 245 | return; 246 | } 247 | 248 | // Package extra metadata about the S3 response on each file stream 249 | // in case we decide we want to use it for something later 250 | __newFile.extra = body; 251 | 252 | console.log(('Receiver: Finished writing `' + __newFile.filename + '`').grey); 253 | 254 | 255 | var endedAt = new Date(); 256 | var duration = ((endedAt - startedAt) / 1000); 257 | console.log('**** S3 upload took '+duration+' seconds...'); 258 | 259 | next(); 260 | }); 261 | 262 | 263 | mpu.on('progress', function(data) { 264 | var snapshot = new Date(); 265 | var secondsElapsed = ((snapshot - startedAt) / 1000); 266 | var estUploadRate = (data.written/1000) / secondsElapsed; 267 | console.log('Uploading at %dkB/s', estUploadRate); 268 | console.log('Elapsed:',secondsElapsed+'s'); 269 | 270 | console.log('Uploading (%s)..',__newFile.filename, data); 271 | receiver__.emit('progress', { 272 | name: __newFile.filename, 273 | written: data.written, 274 | total: data.total, 275 | percent: data.percent 276 | }); 277 | }); 278 | }; 279 | 280 | return receiver__; 281 | } 282 | 283 | 284 | 285 | }; 286 | 287 | 288 | --------------------------------------------------------------------------------