├── .gitignore ├── .npmignore ├── History.md ├── LICENSE.txt ├── Makefile ├── Readme.md ├── index.html ├── lib ├── auth.js ├── client.js ├── index.js └── utils.js ├── package.json └── test ├── auth.test.js ├── createClient.test.js ├── fixtures └── user.json ├── initClients.js ├── knox.test.js ├── mocha.opts ├── signedUrl.test.js └── utils.test.js /.gitignore: -------------------------------------------------------------------------------- 1 | /test/auth.json 2 | /node_modules/ 3 | /npm-debug.log 4 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | # From .gitignore 2 | /npm-debug.log 3 | 4 | # Additions for .npmignore only 5 | /test/ 6 | /.npmignore 7 | /History.md 8 | /index.html 9 | /Makefile 10 | -------------------------------------------------------------------------------- /History.md: -------------------------------------------------------------------------------- 1 | 0.9.3 / 2020-11-13 2 | ================== 3 | 4 | * Updated dependency for `debug` 5 | 6 | 0.9.2 / 2015-01-05 7 | ================== 8 | 9 | * Fix encoding of filenames with `+` or `?` characters in them. (hurrymaplelad) 10 | 11 | 0.9.1 / 2014-08-24 12 | ================== 13 | 14 | * Remove `Expect: 100-continue` headers from PUT and copy commands. We weren't using them correctly anyway. 15 | * Add `extraHeaders` option to `signedUrl`. (dweinstein) 16 | 17 | 0.9.0 / 2014-06-11 18 | =================== 19 | 20 | * Update dependencies: Knox will now no longer work on Node 0.8. 21 | * Fix files with `#` in their filename not working. (kristokaiv) 22 | * Fix a variety of intermittent double-callback bugs, e.g. both response and error, or two errors. If there are two errors, or an error on the request after the response is delivered, those are now swallowed. (willisblackburn, domenic) 23 | * Fix missing return value of `client.deleteFile`. 24 | 25 | 0.8.10 / 2014-05-11 26 | =================== 27 | 28 | * Fix mapping of `us-east-1` region to be `s3.amazonaws.com`, instead of `s3-us-east-1.amazonaws.com`. (coen-hyde) 29 | 30 | 0.8.9 / 2014-02-08 31 | ================== 32 | 33 | * Fix reported sporadic error with `client.list` getting null data by reporting it instead of crashing. (pauliusuza) 34 | 35 | 0.8.8 / 2013-11-27 36 | ================== 37 | 38 | * Fix double-encoding bug introduced in 0.8.7, where using `client.list` with a prefix containing special characters would fail. (colinmutter) 39 | 40 | 0.8.7 / 2013-11-21 41 | ================== 42 | 43 | * Fix handling of non-ASCII characters. (jbuck) 44 | 45 | 0.8.6 / 2013-07-31 46 | ================== 47 | 48 | * Fix normalization of `CommonPrefixes` to an array when doing a `client.list` operation. (mackyi) 49 | * Fix doing operations with spaces in filenames. 50 | * Throw when an invalid port is passed to the constructor. 51 | 52 | 0.8.5 / 2013-07-29 53 | ================== 54 | 55 | * Fix bucket name validation to allow short segments, e.g. in `buck.e.t`. 56 | 57 | 0.8.4 / 2013-07-13 58 | ================== 59 | 60 | * Add the ability to pass arbitrary destination options to `copyTo`. (kof) 61 | * Fix a regression where custom ports were not being used properly in the actual HTTP requests. (aslakhellesoy) 62 | * Re-emit errors from the underlying HTTP request when using `putFile`. 63 | 64 | 0.8.3 / 2013-06-09 65 | ================== 66 | 67 | * No longer modifies `options` objects passed to `knox.createClient`. 68 | 69 | 0.8.2 / 2013-05-20 70 | ================== 71 | 72 | * Fixed a potential issue where request listeners were not cleaned up properly if a callback threw an error. (spollack) 73 | 74 | 0.8.1 / 2013-05-19 75 | ================== 76 | 77 | * Fixed a regression introduced in 0.8.0 that, in certain cases that only some people were able to reproduce, caused 307 responses to every request. 78 | 79 | 0.8.0 / 2013-05-06 80 | ================== 81 | 82 | * Now allows path-style bucket access using `style` option, and automatically chooses it in a few cases: 83 | - DNS-uncompliant bucket names (in the US Standard region, where they are allowed) 84 | - When `secure` is not set to `false`, but the bucket name contains a period 85 | * More extensive validation of bucket names, with good error messages, as per [the Amazon documentation](http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html). 86 | 87 | 0.7.1 / 2013-05-01 88 | ================== 89 | 90 | * If using a custom port, reflect it in the `endpoint` property and in any URLs created using the client. (#168, @jbuck) 91 | * Fix requests using certain Amazon headers, like the conditional copy headers. (#174, @rrjamie) 92 | 93 | 0.7.0 / 2013-04-08 94 | ================== 95 | 96 | * Added real streams2 compatibility: Knox does not kick incoming streams into "old mode," nor does it return streams already kicked into "old mode." (#156, @superjoe30). 97 | * Fixed a rare bug where sometimes callbacks would be called twice, once with an error then with a failed response. (#159) 98 | * Made Node.js 0.8 a requirement in `package.json`; it seems like Knox did not work with Node.js 0.6 anyway. 99 | 100 | 0.6.0 / 2013-03-24 101 | ================== 102 | 103 | * Added a stopgap fix for Knox in Node.js 0.10 with streams2, although we do not yet expose a fully streams2-compatible interface. (#146, @pifantastic) 104 | * Fixed "socket hang up" errors (hopefully!) by disabling the default HTTPS agent. (#116, fix discovered by @kof) 105 | * Added the `domain` configuration option for easy use of other S3-compatible services. (#154, @clee) 106 | * Changed and enhanced `signedUrl`: its third parameter is now `options`, which can contain a `verb` string, a `contentType` string, and a `qs` object. In particular, the new `contentType` capability allows creating pre-signed URLs for PUTs. (#152) 107 | 108 | 0.5.5 / 2013-03-18 109 | ================== 110 | 111 | * Fixed `signedUrl` query-string extra-param support for parameters that contained Unicode characters. (#149) 112 | * Automatically include STS tokens, when a client is created with the `token` option, in URLs generated from `client.signedUrl`. (#147, @willwhite) 113 | 114 | 0.5.4 / 2013-02-27 115 | ================== 116 | 117 | * Fixed `signedUrl` query-string extra-param support for parameters that contained URL-encodable characters. 118 | * Added support for arbitrary verbs (not just `GET`) to `signedUrl`. (#144, @markdaws) 119 | 120 | 0.5.3 / 2013-02-15 121 | ================== 122 | 123 | * The `x-amz-security-token` header is no longer sent when the `token` option is undefined. (#143, @ianshward) 124 | 125 | 0.5.2 / 2013-02-05 126 | ================== 127 | 128 | * Fixed `signedUrl` query-string param support, as introduced in 0.4.7. 129 | * Added [debug](https://npmjs.org/package/debug) support. 130 | 131 | 0.5.0 / 2013-01-25 132 | ================== 133 | 134 | * Added `copyTo` and `copyFileTo` for copying files between buckets. (#16, @kof) 135 | 136 | 0.4.7 / 2013-01-17 137 | ================== 138 | 139 | * Fixed 403s when sending requests for files with any of `!'()*` in their name. (#135, @jeremycondon) 140 | * Added support for arbitrary extra parameters to `signedUrl`, e.g. for use in generating download URLs. (#133) 141 | 142 | 0.4.6 / 2012-12-22 143 | ================== 144 | 145 | * Fixed `signedUrl` to work without a leading slash in the filename, like all other Knox methods. (#129, @relistan) 146 | 147 | 0.4.5 / 2012-12-16 148 | ================== 149 | 150 | * Bucket names with periods are now allowed again, even with SSL. (#128) 151 | 152 | 0.4.4 / 2012-12-08 153 | ================== 154 | 155 | * Added an informative error when using bucket names with periods in them without first turning off SSL. (#125) 156 | * Fixed all requests when passing in `'Content-Type'` or `'Content-MD5'` headers using any casing other than those exact ones, e.g. `'content-type'`. (#126) 157 | 158 | 0.4.3 / 2012-12-05 159 | ================== 160 | 161 | * Fixed `list` always giving `IsTruncated` as `true`. (#124, @simonwex) 162 | 163 | 0.4.2 / 2012-11-24 164 | ================== 165 | 166 | * Fixed `deleteMultiple` when passed keys that start with leading slashes (like they do in the README example). (#121) 167 | * Fixed `list` not always returning an array for the `Contents` property. 168 | 169 | 0.4.1 / 2012-11-02 170 | ================== 171 | 172 | * Added `token` configuration option for temporary security tokens. (@corp186, #110) 173 | 174 | 0.4.0 / 2012-10-27 175 | ================== 176 | 177 | * Added `list` to list all the objects in a bucket. (@kof, #101) 178 | * Fixed tests in Node 0.6.x and in non-ET timezones. (@ianshward, #102) 179 | * Fixed `putStream`'s early-error logic to accept lowercase versions of `'Content-Length'` as well. (#96) 180 | * Added `agent` configuration option for configurable HTTP agents. (@ianshward, #111) 181 | 182 | 0.3.1 / 2012-09-22 183 | ================== 184 | 185 | * No longer specifying `'x-amz-acl'` header as `'public-read'` by default. (@shlevy, #91) 186 | * Made the port configurable with the new `port` option, and defaulting to insecure if the port is customized. (@pifantastic, #86) 187 | * Made `putStream` give an early and user-intelligible error when no `'Content-Length'` header is set, instead of letting Amazon return a cryptic 501 about `'Transfer-Encoding'`. 188 | 189 | 0.3.0 / 2012-08-17 190 | ================== 191 | 192 | * Added `putStream` "progress" event to go along with `putFile`'s. `putStream` now also returns a request object, just like `put`. 193 | * Added new `putBuffer` method as a higher-level way to PUT `Buffer`s. 194 | * When uploading text files using `putFile`, `charset=UTF-8` is now added to the `'Content-Type'` header. (@pifantastic, #83) 195 | * Fixed `signedUrl` method, which was last working in Knox 0.0.9. (@shawnburke, #81) 196 | 197 | 0.2.0 / 2012-08-16 198 | ================== 199 | 200 | * Added `putFile` "progress" event. 201 | 202 | 0.1.0 / 2012-08-02 203 | ================== 204 | 205 | * `putStream` now works with every type of stream, not just file streams, and actually streams the data using `pipe`, instead of buffering chunks into memory. Note that a `'Content-Length'` header is now required, if you weren't using one already. (#14 #32 #48 #57 #72) 206 | * `putFile` is now based on `putStream`, and thus no longer buffers the entire file into memory. 207 | * Added `copyFile` method as a higher-level version of existing `copy`. 208 | * Fixed signing logic for URLs with query parameters outside the Amazon whitelist. (Seth Purcell, #78) 209 | * Leading slashes are now optional again, after becoming mandatory in 0.0.10. (#77) 210 | * Lots of README updates for a more pleasant documentation experience. 211 | 212 | 0.0.11 / 2012-07-18 213 | =================== 214 | 215 | * Now using HTTPS by default, instead of HTTP. This can be disabled with the option `secure: false`. 216 | * Now using the [mime](https://github.com/broofa/node-mime) package as a dependency instead of bundling an outdated version of it. This should result in a much more complete registry of MIME types for auto-detection when using `putFile`. 217 | * Trying to use bucket names that are not all lowercase will give an early error instead of failing with `SignatureDoesNotMatch` upon attempting any operation. [See #44](https://github.com/LearnBoost/knox/issues/44#issuecomment-7074177) for more information. 218 | * Fixed capturing of HTTP request errors to forward to the callback function for all "higher-level API" methods (i.e. those accepting callbacks). (@shuzhang, #71) 219 | * Fixed README example to use `"image/jpeg"` instead of `"image/jpg"`. (@jedwood, #74) 220 | 221 | 0.0.10 / 2012-07-16 222 | =================== 223 | 224 | * Added `client.copy(sourceFilename, destFilename, headers)` method for copying files within a bucket. 225 | * Added `client.deleteMultiple(filenames, headers, cb)` method for [multi-object delete](http://docs.amazonwebservices.com/AmazonS3/latest/API/multiobjectdeleteapi.html). 226 | * Knox now passes through any Content-MD5 headers supplied to any of its methods, and automatically generates one for `putFile`. (@staer, #36) 227 | * Fixed a bug with error propagation in `putStream`. (@xmilliard, #48) 228 | * Fixed requests to querystring resources. (@richtera, #70) 229 | * Updated tests to use [Mocha](http://visionmedia.github.com/mocha/) instead of Expresso; now they can be run on Windows. 230 | 231 | 0.0.9 / 2011-06-20 232 | ================== 233 | 234 | * Fixed signedUrl signature, needs encodeURIComponent() not escape() to prevent SignatureDoesNotMatch errors on signatures containing plus signs. 235 | 236 | 0.0.8 / 2011-06-15 237 | ================== 238 | 239 | * Fixed bug introduced in refactor 240 | 241 | 0.0.7 / 2011-06-14 242 | ================== 243 | 244 | * Fixed resource canonicalization 245 | 246 | 0.0.6 / 2011-06-07 247 | ================== 248 | 249 | * Fixed; ignoring certain query params when preparing stringToSign. [Rajiv Navada] 250 | 251 | 0.0.4 / 2011-05-20 252 | ================== 253 | 254 | * Added `Client#https?(filename)` 255 | 256 | 0.0.3 / 2011-04-12 257 | ================== 258 | 259 | * 0.4.x support 260 | 261 | 0.0.2 / 2011-01-10 262 | ================== 263 | 264 | * Removed `util` require 265 | * Support for S3 presigned URLs 266 | 267 | 0.0.1 / 2010-12-12 268 | ================== 269 | 270 | * Initial release 271 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | (The MIT License) 2 | 3 | Copyright (c) 2010–2012 LearnBoost 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | 'Software'), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | test: 3 | npm test 4 | 5 | docs: index.html 6 | 7 | index.html: 8 | dox \ 9 | --title "Knox" \ 10 | --desc "Light-weight Amazon S3 client for [NodeJS](http://nodejs.org)." \ 11 | --ribbon "http://github.com/LearnBoost/knox" \ 12 | --private \ 13 | lib/knox/*.js > $@ 14 | 15 | docclean: 16 | rm -f index.html 17 | 18 | .PHONY: test docs docclean -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # knox 2 | 3 | Node Amazon S3 Client. 4 | 5 | ## Features 6 | 7 | - Familiar API (`client.get()`, `client.put()`, etc.) 8 | - Very Node-like low-level request capabilities via `http.Client` 9 | - Higher-level API with `client.putStream()`, `client.getFile()`, etc. 10 | - Copying and multi-file delete support 11 | - Streaming file upload and direct stream-piping support 12 | 13 | ## Examples 14 | 15 | The following examples demonstrate some capabilities of knox and the S3 REST 16 | API. First things first, create an S3 client: 17 | 18 | ```js 19 | var client = knox.createClient({ 20 | key: '' 21 | , secret: '' 22 | , bucket: 'learnboost' 23 | }); 24 | ``` 25 | 26 | More options are documented below for features like other endpoints or regions. 27 | 28 | ### PUT 29 | 30 | If you want to directly upload some strings to S3, you can use the `Client#put` 31 | method with a string or buffer, just like you would for any `http.Client` 32 | request. You pass in the filename as the first parameter, some headers for the 33 | second, and then listen for a `'response'` event on the request. Then send the 34 | request using `req.end()`. If we get a 200 response, great! 35 | 36 | > If you send a string, set `Content-Length` to the length of the buffer of your string, rather than of the string itself. 37 | 38 | ```js 39 | var object = { foo: "bar" }; 40 | var string = JSON.stringify(object); 41 | var req = client.put('/test/obj.json', { 42 | 'Content-Length': Buffer.byteLength(string) 43 | , 'Content-Type': 'application/json' 44 | }); 45 | req.on('response', function(res){ 46 | if (200 == res.statusCode) { 47 | console.log('saved to %s', req.url); 48 | } 49 | }); 50 | req.end(string); 51 | ``` 52 | 53 | By default the _x-amz-acl_ header is _private_. To alter this simply pass this 54 | header to the client request method. 55 | 56 | ```js 57 | client.put('/test/obj.json', { 'x-amz-acl': 'public-read' }); 58 | ``` 59 | 60 | Each HTTP verb has an alternate method with the "File" suffix, for example 61 | `put()` also has a higher level method named `putFile()`, accepting a source 62 | filename and performing the dirty work shown above for you. Here is an example 63 | usage: 64 | 65 | ```js 66 | client.putFile('my.json', '/user.json', function(err, res){ 67 | // Always either do something with `res` or at least call `res.resume()`. 68 | }); 69 | ``` 70 | 71 | Another alternative is to stream via `Client#putStream()`, for example: 72 | 73 | ```js 74 | http.get('http://google.com/doodle.png', function(res){ 75 | var headers = { 76 | 'Content-Length': res.headers['content-length'] 77 | , 'Content-Type': res.headers['content-type'] 78 | }; 79 | client.putStream(res, '/doodle.png', headers, function(err, res){ 80 | // check `err`, then do `res.pipe(..)` or `res.resume()` or whatever. 81 | }); 82 | }); 83 | ``` 84 | 85 | You can also use your stream's `pipe` method to pipe to the PUT request, but 86 | you'll still have to set the `'Content-Length'` header. For example: 87 | 88 | ```js 89 | fs.stat('./Readme.md', function(err, stat){ 90 | // Be sure to handle `err`. 91 | 92 | var req = client.put('/Readme.md', { 93 | 'Content-Length': stat.size 94 | , 'Content-Type': 'text/plain' 95 | }); 96 | 97 | fs.createReadStream('./Readme.md').pipe(req); 98 | 99 | req.on('response', function(res){ 100 | // ... 101 | }); 102 | }); 103 | ``` 104 | 105 | Finally, if you want a nice interface for putting a buffer or a string of data, 106 | use `Client#putBuffer()`: 107 | 108 | ```js 109 | var buffer = new Buffer('a string of data'); 110 | var headers = { 111 | 'Content-Type': 'text/plain' 112 | }; 113 | client.putBuffer(buffer, '/string.txt', headers, function(err, res){ 114 | // ... 115 | }); 116 | ``` 117 | 118 | Note that both `putFile` and `putStream` will stream to S3 instead of reading 119 | into memory, which is great. And they return objects that emit `'progress'` 120 | events too, so you can monitor how the streaming goes! The progress events have 121 | fields `written`, `total`, and `percent`. 122 | 123 | ### GET 124 | 125 | Below is an example __GET__ request on the file we just shoved at S3. It simply 126 | outputs the response status code, headers, and body. 127 | 128 | ```js 129 | client.get('/test/Readme.md').on('response', function(res){ 130 | console.log(res.statusCode); 131 | console.log(res.headers); 132 | res.setEncoding('utf8'); 133 | res.on('data', function(chunk){ 134 | console.log(chunk); 135 | }); 136 | }).end(); 137 | ``` 138 | 139 | There is also `Client#getFile()` which uses a callback pattern instead of giving 140 | you the raw request: 141 | 142 | ```js 143 | client.getFile('/test/Readme.md', function(err, res){ 144 | // check `err`, then do `res.pipe(..)` or `res.resume()` or whatever. 145 | }); 146 | ``` 147 | 148 | ### DELETE 149 | 150 | Delete our file: 151 | 152 | ```js 153 | client.del('/test/Readme.md').on('response', function(res){ 154 | console.log(res.statusCode); 155 | console.log(res.headers); 156 | }).end(); 157 | ``` 158 | 159 | Likewise we also have `Client#deleteFile()` as a more concise (yet less 160 | flexible) solution: 161 | 162 | ```js 163 | client.deleteFile('/test/Readme.md', function(err, res){ 164 | // check `err`, then do `res.pipe(..)` or `res.resume()` or whatever. 165 | }); 166 | ``` 167 | 168 | ### HEAD 169 | 170 | As you might expect we have `Client#head` and `Client#headFile`, following the 171 | same pattern as above. 172 | 173 | ### Advanced Operations 174 | 175 | Knox supports a few advanced operations. Like [copying files][copy]: 176 | 177 | ```js 178 | client.copy('/test/source.txt', '/test/dest.txt').on('response', function(res){ 179 | console.log(res.statusCode); 180 | console.log(res.headers); 181 | }).end(); 182 | 183 | // or 184 | 185 | client.copyFile('/source.txt', '/dest.txt', function(err, res){ 186 | // ... 187 | }); 188 | ``` 189 | 190 | even between buckets: 191 | 192 | ```js 193 | client.copyTo('/source.txt', 'dest-bucket', '/dest.txt').on('response', function(res){ 194 | // ... 195 | }).end(); 196 | ``` 197 | 198 | and even between buckets in different regions: 199 | 200 | ```js 201 | var destOptions = { region: 'us-west-2', bucket: 'dest-bucket' }; 202 | client.copyTo('/source.txt', destOptions, '/dest.txt', function(res){ 203 | // ... 204 | }).end(); 205 | ``` 206 | 207 | or [deleting multiple files at once][multi-delete]: 208 | 209 | ```js 210 | client.deleteMultiple(['/test/Readme.md', '/test/Readme.markdown'], function(err, res){ 211 | // ... 212 | }); 213 | ``` 214 | 215 | or [listing all the files in your bucket][list]: 216 | 217 | ```js 218 | client.list({ prefix: 'my-prefix' }, function(err, data){ 219 | /* `data` will look roughly like: 220 | 221 | { 222 | Prefix: 'my-prefix', 223 | IsTruncated: true, 224 | MaxKeys: 1000, 225 | Contents: [ 226 | { 227 | Key: 'whatever' 228 | LastModified: new Date(2012, 11, 25, 0, 0, 0), 229 | ETag: 'whatever', 230 | Size: 123, 231 | Owner: 'you', 232 | StorageClass: 'whatever' 233 | }, 234 | ⋮ 235 | ] 236 | } 237 | 238 | */ 239 | }); 240 | ``` 241 | 242 | And you can always issue ad-hoc requests, e.g. the following to 243 | [get an object's ACL][acl]: 244 | 245 | ```js 246 | client.request('GET', '/test/Readme.md?acl').on('response', function(res){ 247 | // Read and parse the XML response. 248 | // Everyone loves XML parsing. 249 | }).end(); 250 | ``` 251 | 252 | Finally, you can construct HTTP or HTTPS URLs for a file like so: 253 | 254 | ```js 255 | var readmeUrl = client.http('/test/Readme.md'); 256 | var userDataUrl = client.https('/user.json'); 257 | ``` 258 | 259 | [copy]: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html 260 | [multi-delete]: http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html 261 | [list]: http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTBucketGET.html 262 | [acl]: http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectGETacl.html 263 | 264 | ## Client Creation Options 265 | 266 | Besides the required `key`, `secret`, and `bucket` options, you can supply any 267 | of the following: 268 | 269 | ### `endpoint` 270 | 271 | By default knox will send all requests to the global endpoint 272 | (s3.amazonaws.com). This works regardless of the region where the bucket 273 | is. But if you want to manually set the endpoint, e.g. for performance or 274 | testing reasons, or because you are using a S3-compatible service that isn't 275 | hosted by Amazon, you can do it with the `endpoint` option. 276 | 277 | ### `region` 278 | 279 | For your convenience when using buckets not in the US Standard region, you can 280 | specify the `region` option. When you do so, the `endpoint` is automatically 281 | assembled. 282 | 283 | As of this writing, valid values for the `region` option are: 284 | 285 | * US Standard (default): `us-standard` 286 | * US West (Oregon): `us-west-2` 287 | * US West (Northern California): `us-west-1` 288 | * EU (Ireland): `eu-west-1` 289 | * Asia Pacific (Singapore): `ap-southeast-1` 290 | * Asia Pacific (Tokyo): `ap-northeast-1` 291 | * South America (Sao Paulo): `sa-east-1` 292 | 293 | If new regions are added later, their subdomain names will also work when passed 294 | as the `region` option. See the [AWS endpoint documentation][endpoint-docs] for 295 | the latest list. 296 | 297 | **Convenience APIs such as `putFile` and `putStream` currently do not work as 298 | expected with buckets in regions other than US Standard without explicitly 299 | specify the region option.** This will eventually be addressed by resolving 300 | [issue #66][]; however, for performance reasons, it is always best to specify 301 | the region option anyway. 302 | 303 | [endpoint-docs]: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region 304 | [issue #66]: https://github.com/LearnBoost/knox/issues/66 305 | 306 | ### `secure` and `port` 307 | 308 | By default, knox uses HTTPS to connect to S3 on port 443. You can override 309 | either of these with the `secure` and `port` options. Note that if you specify a 310 | custom `port` option, the default for `secure` switches to `false`, although 311 | you can override it manually if you want to run HTTPS against a specific port. 312 | 313 | ### `token` 314 | 315 | If you are using the [AWS Security Token Service][sts] APIs, you can construct 316 | the client with a `token` parameter containing the temporary security 317 | credentials token. This simply sets the _x-amz-security-token_ header on every 318 | request made by the client. 319 | 320 | [sts]: http://docs.amazonwebservices.com/STS/latest/UsingSTS/Welcome.html 321 | 322 | ### `style` 323 | 324 | By default, knox tries to use the "virtual hosted style" URLs for accessing S3, 325 | e.g. `bucket.s3.amazonaws.com`. If you pass in `"path"` as the `style` option, 326 | or pass in a `bucket` value that cannot be used with virtual hosted style URLs, 327 | knox will use "path style" URLs, e.g. `s3.amazonaws.com/bucket`. There are 328 | tradeoffs you should be aware of: 329 | 330 | - Virtual hosted style URLs can work with any region, without requiring it to be 331 | explicitly specified; path style URLs cannot. 332 | - You can access programmatically-created buckets only by using virtual hosted 333 | style URLs; path style URLs will not work. 334 | - You can access buckets with periods in their names over SSL using path style 335 | URLs; virtual host style URLs will not work unless you turn off certificate 336 | validation. 337 | - You can access buckets with mixed-case names only using path style URLs; 338 | virtual host style URLs will not work. 339 | 340 | For more information on the differences between these two types of URLs, and 341 | limitations related to them, see the following S3 documentation pages: 342 | 343 | - [Virtual Hosting of Buckets][virtual] 344 | - [Bucket Configuration Options][config] 345 | - [Bucket Restrictions and Limitations][limits] 346 | 347 | [virtual]: http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html 348 | [config]: http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketConfiguration.html 349 | [limits]: http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html 350 | 351 | ### `agent` 352 | 353 | Knox disables the default [HTTP agent][], because it leads to lots of "socket 354 | hang up" errors when doing more than 5 requests at once. See [#116][] for 355 | details. If you want to get the default agent back, you can specify 356 | `agent: require("https").globalAgent`, or use your own. 357 | 358 | [#116]: https://github.com/LearnBoost/knox/issues/116#issuecomment-15045187 359 | [HTTP agent]: http://nodejs.org/docs/latest/api/http.html#http_class_http_agent 360 | 361 | 362 | ## Beyond Knox 363 | 364 | ### Multipart Upload 365 | 366 | S3's [multipart upload][] is their [rather-complicated][] way of uploading large 367 | files. In particular, it is the only way of streaming files without knowing 368 | their Content-Length ahead of time. 369 | 370 | Adding the complexity of multipart upload directly to knox is not a great idea. 371 | For example, it requires buffering at least 5 MiB of data at a time in memory, 372 | which you want to avoid if possible. Fortunately, [@nathanoehlman][] has created 373 | the excellent [knox-mpu][] package to let you use multipart upload with knox if 374 | you need it! 375 | 376 | [multipart upload]: http://aws.typepad.com/aws/2010/11/amazon-s3-multipart-upload.html 377 | [rather-complicated]: http://stackoverflow.com/q/8653146/3191 378 | [@nathanoehlman]: https://github.com/nathanoehlman 379 | [knox-mpu]: https://npmjs.org/package/knox-mpu 380 | 381 | ### Easy Download/Upload 382 | 383 | [@superjoe30][] has created a nice library, called simply [s3][], that makes it 384 | very easy to upload local files directly to S3, and download them back to your 385 | filesystem. For simple cases this is often exactly what you want! 386 | 387 | [@superjoe30]: https://github.com/superjoe30 388 | [s3]: https://npmjs.org/package/s3 389 | 390 | ### Uploading With Retries and Exponential Backoff 391 | 392 | [@jergason][] created [intimidate][], a library wrapping Knox to automatically retry 393 | failed uploads with exponential backoff. This helps your app deal with intermittent 394 | connectivity to S3 without bringing it to a ginding halt. 395 | 396 | [@jergason]: https://github.com/jergason 397 | [intimidate]: https://npmjs.org/package/intimidate 398 | 399 | ### Listing and Copying Large Buckets 400 | 401 | [@goodeggs][] created [knox-copy][] to easily copy and stream keys of buckets beyond Amazon's 1000 key page size limit. 402 | 403 | [@goodeggs]: https://github.com/goodeggs 404 | [knox-copy]: https://npmjs.org/package/knox-copy 405 | 406 | 407 | [@segmentio][] created [s3-lister][] to stream a list of bucket keys using the new streams2 interface. 408 | 409 | [@segmentio]: https://github.com/segmentio 410 | [s3-lister]: https://npmjs.org/package/s3-lister 411 | 412 | [@drob][] created [s3-deleter][], a writable stream that batch-deletes bucket keys. 413 | 414 | [@drob]: https://github.com/drob 415 | [s3-deleter]: https://npmjs.org/package/s3-deleter 416 | 417 | ## Running Tests 418 | 419 | To run the test suite you must first have an S3 account. Then create a file named 420 | _./test/auth.json_, which contains your credentials as JSON, for example: 421 | 422 | ```json 423 | { 424 | "key": "", 425 | "secret": "", 426 | "bucket": "", 427 | "bucket2": "", 428 | "bucketUsWest2": "" 429 | } 430 | ``` 431 | 432 | Then install the dev dependencies and execute the test suite: 433 | 434 | ``` 435 | $ npm install 436 | $ npm test 437 | ``` 438 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | Fork me on GitHub 2 | 3 | Knox 4 | 5 | 99 | 109 | 110 | 111 | 112 | 116 | 119 | 120 | 121 | 129 | 134 | 135 | 136 | 143 | 149 | 150 | 151 | 167 | 180 | 181 | 182 | 191 | 205 | 206 | 210 | 216 | 217 | 218 | 229 | 238 | 239 | 240 | 247 | 276 | 277 | 278 | 308 | 317 | 318 | 319 | 326 | 331 | 332 | 333 | 340 | 345 | 346 | 347 | 354 | 359 | 360 | 361 | 368 | 373 | 374 | 375 | 383 | 388 | 389 | 393 | 396 | 397 | 398 | 403 | 406 | 407 | 408 | 413 | 416 | 417 | 418 | 423 | 426 | 427 | 434 | 444 | 445 | 446 | 450 | 453 | 454 | 455 | 462 | 467 | 468 | 469 | 476 | 482 | 483 |

Knox

Light-weight Amazon S3 client for NodeJS.

auth

lib/knox/auth.js
113 |

Module dependencies. 114 |

115 | 		117 | 
var crypto = require('crypto');
118 |
122 |

Return an "Authorization" header value with the given options 123 | in the form of "AWS <key>:<signature>"

124 | 125 |

126 | 127 |

  • param: Object options

  • return: String

  • api: private

128 | 		130 | 
exports.authorization = function(options){
131 |   return 'AWS ' + options.key + ':' + exports.sign(options);
132 | };
133 |
137 |

Create a base64 sha1 HMAC for options.

138 | 139 |

140 | 141 |

  • param: Object options

  • return: String

  • api: private

142 | 		144 | 
exports.sign = function(options){
145 |   var str = exports.stringToSign(options);
146 |   return crypto.createHmac('sha1', options.secret).update(str).digest('base64');
147 | };
148 |
152 |

Return a string for sign() with the given options.

153 | 154 |

Spec

155 | 156 |

<verb>\n 157 | <md5>\n 158 | <content-type>\n 159 | <date>\n 160 | [headers\n] 161 | <resource>

162 | 163 |

164 | 165 |

  • param: Object options

  • return: String

  • api: private

166 | 		168 | 
exports.stringToSign = function(options){
169 |   var headers = options.amazonHeaders || '';
170 |   if (headers) headers += '\n';
171 |   return [
172 |       options.verb
173 |     , options.md5
174 |     , options.contentType
175 |     , options.date.toUTCString()
176 |     , headers + options.resource
177 |   ].join('\n');
178 | };
179 |
183 |

Perform the following:

184 | 185 |
  • ignore non-amazon headers
  • lowercase fields
  • sort lexicographically
  • trim whitespace between ":"
  • join with newline
186 | 187 |

188 | 189 |

  • param: Object headers

  • return: String

  • api: private

190 | 		192 | 
exports.canonicalizeHeaders = function(headers){
193 |   var buf = []
194 |     , fields = Object.keys(headers);
195 |   for (var i = 0, len = fields.length; i &lt; len; ++i) {
196 |     var field = fields[i]
197 |       , val = headers[field]
198 |       , field = field.toLowerCase();
199 |     if (0 !== field.indexOf('x-amz')) continue;
200 |     buf.push(field + ':' + val);
201 |   }
202 |   return buf.sort().join('\n');
203 | };
204 |

client

lib/knox/client.js
207 |

Module dependencies. 208 |

209 | 		211 | 
var utils = require('./utils')
212 |   , auth = require('./auth')
213 |   , http = require('http')
214 |   , join = require('path').join;
215 |
219 |

Initialize a Client with the given options.

220 | 221 |

Required

222 | 223 |
  • key amazon api key
  • secret amazon secret
  • bucket bucket name string, ex: "learnboost"
224 | 225 |

226 | 227 |

  • param: Object options

  • api: public

228 | 		230 | 
var Client = module.exports = exports = function Client(options) {
231 |   this.host = 's3.amazonaws.com';
232 |   utils.merge(this, options);
233 |   if (!this.key) throw new Error('aws "key" required');
234 |   if (!this.secret) throw new Error('aws "secret" required');
235 |   if (!this.bucket) throw new Error('aws "bucket" required');
236 | };
237 |
241 |

Request with filename the given method, and optional headers.

242 | 243 |

244 | 245 |

  • param: String method

  • param: String filename

  • param: Object headers

  • return: ClientRequest

  • api: private

246 | 		248 | 
Client.prototype.request = function(method, filename, headers){
249 |   var client = http.createClient(80, this.host)
250 |     , path = join('/', this.bucket, filename)
251 |     , date = new Date
252 |     , headers = headers || {};
253 | 
254 |   // Default headers
255 |   utils.merge(headers, {
256 |       Date: date.toUTCString()
257 |     , Host: this.host
258 |   });
259 | 
260 |   // Authorization header
261 |   headers.Authorization = auth.authorization({
262 |       key: this.key
263 |     , secret: this.secret
264 |     , verb: method
265 |     , date: date
266 |     , resource: path
267 |     , contentType: headers['Content-Type']
268 |     , amazonHeaders: auth.canonicalizeHeaders(headers)
269 |   });
270 | 
271 |   var req = client.request(method, path, headers);
272 |   req.url = this.url(filename);
273 |   return req;
274 | };
275 |
279 |

PUT data to filename with optional headers.

280 | 281 |

Example

282 | 283 | 
// Fetch the size
284 | fs.stat('Readme.md', function(err, stat){
285 |  // Create our request
286 |  var req = client.put('/test/Readme.md', {
287 |      'Content-Length': stat.size
288 |    , 'Content-Type': 'text/plain'
289 |  });
290 |  fs.readFile('Readme.md', function(err, buf){
291 |    // Output response
292 |    req.on('response', function(res){
293 |      console.log(res.statusCode);
294 |      console.log(res.headers);
295 |      res.on('data', function(chunk){
296 |        console.log(chunk.toString());
297 |      });
298 |    }); 
299 |    // Send the request with the file's Buffer obj
300 |    req.end(buf);
301 |  });
302 | });
303 | 304 |

305 | 306 |

  • param: String filename

  • param: Object headers

  • return: ClientRequest

  • api: public

307 | 		309 | 
Client.prototype.put = function(filename, headers){
310 |   headers = utils.merge({
311 |       Expect: '100-continue'
312 |     , 'x-amz-acl': 'public-read'
313 |   }, headers || {});
314 |   return this.request('PUT', filename, headers);
315 | };
316 |
320 |

GET filename with optional headers.

321 | 322 |

323 | 324 |

  • param: String filename

  • param: Object headers

  • return: ClientRequest

  • api: public

325 | 		327 | 
Client.prototype.get = function(filename, headers){
328 |   return this.request('GET', filename, headers);
329 | };
330 |
334 |

Issue a HEAD request on filename with optional `headers.

335 | 336 |

337 | 338 |

  • param: String filename

  • param: Object headers

  • return: ClientRequest

  • api: public

339 | 		341 | 
Client.prototype.head = function(filename, headers){
342 |   return this.request('HEAD', filename, headers);
343 | };
344 |
348 |

DELETE filename with optional `headers.

349 | 350 |

351 | 352 |

  • param: String filename

  • param: Object headers

  • return: ClientRequest

  • api: public

353 | 		355 | 
Client.prototype.del = function(filename, headers){
356 |   return this.request('DELETE', filename, headers);
357 | };
358 |
362 |

Return a url to the given filename.

363 | 364 |

365 | 366 |

  • param: String filename

  • return: String

  • api: public

367 | 		369 | 
Client.prototype.url = function(filename){
370 |   return 'http://' + this.bucket + '.' + this.host + join('/', filename);
371 | };
372 |
376 |

Shortcut for new Client().

377 | 378 |

379 | 380 |

  • param: Object options

  • see: Client 381 | ()

  • api: public

382 | 		384 | 
exports.createClient = function(options){
385 |   return new Client(options);
386 | };
387 |

index

lib/knox/index.js
390 |

Client is the main export. 391 |

392 | 		394 | 
exports = module.exports = require('./client');
395 |
399 |

Library version.

400 | 401 |

  • type: String

402 | 		404 | 
exports.version = '0.0.1';
405 |
409 |

Expose utilities.

410 | 411 |

  • type: Object

412 | 		414 | 
exports.utils = require('./utils');
415 |
419 |

Expose auth utils.

420 | 421 |

  • type: Object

422 | 		424 | 
exports.auth = require('./auth');
425 |

utils

lib/knox/utils.js
428 |

Merge object b with object a.

429 | 430 |

431 | 432 |

  • param: Object a

  • param: Object b

  • return: Object a

  • api: private

433 | 		435 | 
exports.merge = function(a, b){
436 |   var keys = Object.keys(b);
437 |   for (var i = 0, len = keys.length; i &lt; len; ++i) {
438 |     var key = keys[i];
439 |     a[key] = b[key]
440 |   }
441 |   return a;
442 | };
443 |
447 |

Base64. 448 |

449 | 		451 | 
exports.base64 = {
452 |
456 |

Base64 encode the given str.

457 | 458 |

459 | 460 |

  • param: String str

  • return: String

  • api: private

461 | 		463 | 
encode: function(str){
464 |     return new Buffer(str).toString('base64');
465 |   },
466 |
470 |

Base64 decode the given str.

471 | 472 |

473 | 474 |

  • param: String str

  • return: String

  • api: private

475 | 		477 | 
decode: function(str){
478 |     return new Buffer(str, 'base64').toString();
479 |   }
480 | };
481 |
-------------------------------------------------------------------------------- /lib/auth.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /*! 4 | * knox - auth 5 | * Copyright(c) 2010 LearnBoost 6 | * MIT Licensed 7 | */ 8 | 9 | /** 10 | * Module dependencies. 11 | */ 12 | 13 | var crypto = require('crypto') 14 | , parse = require('url').parse; 15 | 16 | /** 17 | * Query string params permitted in the canonicalized resource. 18 | * @see http://docs.amazonwebservices.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheCanonicalizedResourceElement 19 | */ 20 | 21 | var whitelist = [ 22 | 'acl' 23 | , 'delete' 24 | , 'lifecycle' 25 | , 'location' 26 | , 'logging' 27 | , 'notification' 28 | , 'partNumber' 29 | , 'policy' 30 | , 'requestPayment' 31 | , 'torrent' 32 | , 'uploadId' 33 | , 'uploads' 34 | , 'versionId' 35 | , 'versioning' 36 | , 'versions' 37 | , 'website' 38 | ]; 39 | 40 | /** 41 | * Return an "Authorization" header value with the given `options` 42 | * in the form of "AWS :" 43 | * 44 | * @param {Object} options 45 | * @return {String} 46 | * @api private 47 | */ 48 | 49 | exports.authorization = function(options){ 50 | return 'AWS ' + options.key + ':' + exports.sign(options); 51 | }; 52 | 53 | /** 54 | * Simple HMAC-SHA1 Wrapper 55 | * 56 | * @param {Object} options 57 | * @return {String} 58 | * @api private 59 | */ 60 | 61 | exports.hmacSha1 = function(options){ 62 | return crypto.createHmac('sha1', options.secret).update(new Buffer(options.message, 'utf-8')).digest('base64'); 63 | }; 64 | 65 | /** 66 | * Create a base64 sha1 HMAC for `options`. 67 | * 68 | * @param {Object} options 69 | * @return {String} 70 | * @api private 71 | */ 72 | 73 | exports.sign = function(options){ 74 | options.message = exports.stringToSign(options); 75 | return exports.hmacSha1(options); 76 | }; 77 | 78 | /** 79 | * Create a base64 sha1 HMAC for `options`. 80 | * 81 | * Specifically to be used with S3 presigned URLs 82 | * 83 | * @param {Object} options 84 | * @return {String} 85 | * @api private 86 | */ 87 | 88 | exports.signQuery = function(options){ 89 | options.message = exports.queryStringToSign(options); 90 | return exports.hmacSha1(options); 91 | }; 92 | 93 | /** 94 | * Return a string for sign() with the given `options`. 95 | * 96 | * Spec: 97 | * 98 | * \n 99 | * \n 100 | * \n 101 | * \n 102 | * [headers\n] 103 | * 104 | * 105 | * @param {Object} options 106 | * @return {String} 107 | * @api private 108 | */ 109 | 110 | exports.stringToSign = function(options){ 111 | var headers = options.amazonHeaders || ''; 112 | if (headers) headers += '\n'; 113 | return [ 114 | options.verb 115 | , options.md5 116 | , options.contentType 117 | , options.date instanceof Date ? options.date.toUTCString() : options.date 118 | , headers + options.resource 119 | ].join('\n'); 120 | }; 121 | 122 | /** 123 | * Return a string for sign() with the given `options`, but is meant exclusively 124 | * for S3 presigned URLs 125 | * 126 | * Spec: 127 | * 128 | * \n\n 129 | * \n 130 | * \n 131 | * \n --- optional 132 | * 133 | * 134 | * @param {Object} options 135 | * @return {String} 136 | * @api private 137 | */ 138 | 139 | exports.queryStringToSign = function(options){ 140 | return (options.verb || 'GET') + '\n\n' + 141 | (typeof options.contentType !== 'undefined' ? 142 | options.contentType : '') + '\n' + 143 | options.date + '\n' + 144 | (typeof options.extraHeaders !== 'undefined' ? 145 | exports.canonicalizeHeaders(options.extraHeaders) + '\n' : '') + 146 | (typeof options.token !== 'undefined' ? 147 | 'x-amz-security-token:' + options.token + '\n' : '') + 148 | options.resource; 149 | }; 150 | 151 | /** 152 | * Perform the following: 153 | * 154 | * - ignore non-amazon headers 155 | * - lowercase fields 156 | * - sort lexicographically 157 | * - trim whitespace between ":" 158 | * - join with newline 159 | * 160 | * @param {Object} headers 161 | * @return {String} 162 | * @api private 163 | */ 164 | 165 | exports.canonicalizeHeaders = function(headers){ 166 | var buf = [] 167 | , fields = Object.keys(headers); 168 | for (var i = 0, len = fields.length; i < len; ++i) { 169 | var field = fields[i] 170 | , val = headers[field]; 171 | 172 | field = field.toLowerCase(); 173 | 174 | if (field.indexOf('x-amz') !== 0 || field === 'x-amz-date') { 175 | continue; 176 | } 177 | 178 | buf.push(field + ':' + val); 179 | } 180 | 181 | var headerSort = function(a, b) { 182 | // Headers are sorted lexigraphically based on the header name only. 183 | a = a.split(":")[0] 184 | b = b.split(":")[0] 185 | 186 | return a > b ? 1 : -1; 187 | } 188 | return buf.sort(headerSort).join('\n'); 189 | }; 190 | 191 | /** 192 | * Perform the following: 193 | * 194 | * - ignore non sub-resources 195 | * - sort lexicographically 196 | * 197 | * @param {String} a URI-encoded resource (path + query string) 198 | * @return {String} 199 | * @api private 200 | */ 201 | 202 | exports.canonicalizeResource = function(resource){ 203 | var url = parse(resource, true) 204 | , path = url.pathname 205 | , buf = []; 206 | 207 | // apply the query string whitelist 208 | Object.keys(url.query).forEach(function (key) { 209 | if (whitelist.indexOf(key) != -1) { 210 | buf.push(key + (url.query[key] ? "=" + url.query[key] : '')); 211 | } 212 | }); 213 | 214 | return path + (buf.length 215 | ? '?' + buf.sort().join('&') 216 | : ''); 217 | }; 218 | -------------------------------------------------------------------------------- /lib/client.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /*! 4 | * knox - Client 5 | * Copyright(c) 2010 LearnBoost 6 | * MIT Licensed 7 | */ 8 | 9 | /** 10 | * Module dependencies. 11 | */ 12 | 13 | var Emitter = require('events').EventEmitter 14 | , debug = require('debug')('knox') 15 | , utils = require('./utils') 16 | , auth = require('./auth') 17 | , http = require('http') 18 | , https = require('https') 19 | , url = require('url') 20 | , mime = require('mime') 21 | , fs = require('fs') 22 | , crypto = require('crypto') 23 | , once = require('once') 24 | , xml2js = require('xml2js') 25 | , StreamCounter = require('stream-counter') 26 | , qs = require('querystring'); 27 | 28 | // The max for multi-object delete, bucket listings, etc. 29 | var BUCKET_OPS_MAX = 1000; 30 | 31 | // http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html 32 | var MIN_BUCKET_LENGTH = 3; 33 | var MAX_NON_US_STANDARD_BUCKET_LENGTH = 63; 34 | var MAX_US_STANDARD_BUCKET_LENGTH = 255; 35 | var US_STANDARD_BUCKET = /^[A-Za-z0-9\._-]*$/; 36 | var BUCKET_LABEL = /^(?:[a-z0-9][a-z0-9-]*[a-z0-9]|[a-z0-9])$/; 37 | var IPV4_ADDRESS = /^(\d{1,3}\.){3}(\d{1,3})$/; 38 | 39 | /** 40 | * Register event listeners on a request object to convert standard http 41 | * request events into appropriate call backs. 42 | * @param {Request} req The http request 43 | * @param {Function} fn(err, res) The callback function. 44 | * err - The exception if an exception occurred while sending the http 45 | * request (for example if internet connection was lost). 46 | * res - The http response if no exception occurred. 47 | * @api private 48 | */ 49 | function registerReqListeners(req, fn){ 50 | req.on('response', function (res) { 51 | fn(null, res); 52 | }); 53 | req.on('error', fn); 54 | } 55 | 56 | function ensureLeadingSlash(filename) { 57 | return filename[0] !== '/' ? '/' + filename : filename; 58 | } 59 | 60 | function removeLeadingSlash(filename) { 61 | return filename[0] === '/' ? filename.substring(1) : filename; 62 | } 63 | 64 | function encodeSpecialCharacters(filename) { 65 | // Note: these characters are valid in URIs, but S3 does not like them for 66 | // some reason. 67 | return encodeURI(filename).replace(/[!'()#*+? ]/g, function (char) { 68 | return '%' + char.charCodeAt(0).toString(16); 69 | }); 70 | } 71 | 72 | function getHeader(headers, headerNameLowerCase) { 73 | for (var header in headers) { 74 | if (header.toLowerCase() === headerNameLowerCase) { 75 | return headers[header]; 76 | } 77 | } 78 | return null; 79 | } 80 | 81 | function isNotDnsCompliant(bucket) { 82 | if (bucket.length > MAX_NON_US_STANDARD_BUCKET_LENGTH) { 83 | return 'is more than ' + MAX_NON_US_STANDARD_BUCKET_LENGTH + ' characters'; 84 | } 85 | 86 | if (IPV4_ADDRESS.test(bucket)) { 87 | return 'is formatted as an IPv4 address'; 88 | } 89 | 90 | var bucketLabels = bucket.split('.'); 91 | var bucketLabelsAreValid = bucketLabels.every(function (label) { 92 | return BUCKET_LABEL.test(label); 93 | }); 94 | 95 | if (!bucketLabelsAreValid) { 96 | return 'does not consist of valid period-separated labels'; 97 | } 98 | 99 | return false; 100 | } 101 | 102 | function isInvalid(bucket) { 103 | if (bucket.length < MIN_BUCKET_LENGTH) { 104 | return 'is less than ' + MIN_BUCKET_LENGTH + ' characters'; 105 | } 106 | if (bucket.length > MAX_US_STANDARD_BUCKET_LENGTH) { 107 | return 'is more than ' + MAX_US_STANDARD_BUCKET_LENGTH + ' characters'; 108 | } 109 | 110 | if (!US_STANDARD_BUCKET.test(bucket)) { 111 | return 'contains invalid characters'; 112 | } 113 | 114 | return false; 115 | } 116 | 117 | function containsPeriod(bucket) { 118 | return bucket.indexOf('.') !== -1; 119 | } 120 | 121 | function autoDetermineStyle(options) { 122 | if (!options.style && options.secure !== false && 123 | containsPeriod(options.bucket)) { 124 | options.style = 'path'; 125 | return; 126 | } 127 | 128 | var dnsUncompliance = isNotDnsCompliant(options.bucket); 129 | if (dnsUncompliance) { 130 | if (options.style === 'virtualHosted') { 131 | throw new Error('Cannot use "virtualHosted" style with a ' + 132 | 'DNS-uncompliant bucket name: "' + options.bucket + 133 | '" is ' + dnsUncompliance + '.'); 134 | } 135 | 136 | options.style = 'path'; 137 | return; 138 | } 139 | 140 | if (!options.style) { 141 | options.style = 'virtualHosted'; 142 | } 143 | } 144 | 145 | /** 146 | * Get headers needed for Client#copy and Client#copyTo. 147 | * 148 | * @param {String} sourceFilename 149 | * @param {Object} headers 150 | * @api private 151 | */ 152 | 153 | function getCopyHeaders(sourceBucket, sourceFilename, headers) { 154 | sourceFilename = encodeSpecialCharacters(ensureLeadingSlash(sourceFilename)); 155 | headers = utils.merge({}, headers || {}); 156 | headers['x-amz-copy-source'] = '/' + sourceBucket + sourceFilename; 157 | headers['Content-Length'] = 0; // to avoid Node's automatic chunking if omitted 158 | return headers; 159 | } 160 | 161 | 162 | /** 163 | * Initialize a `Client` with the given `options`. 164 | * 165 | * Required: 166 | * 167 | * - `key` amazon api key 168 | * - `secret` amazon secret 169 | * - `bucket` bucket name string, ex: "learnboost" 170 | * 171 | * @param {Object} options 172 | * @api public 173 | */ 174 | 175 | var Client = module.exports = exports = function Client(options) { 176 | if (!options.key) throw new Error('aws "key" required'); 177 | if (!options.secret) throw new Error('aws "secret" required'); 178 | if (!options.bucket) throw new Error('aws "bucket" required'); 179 | 180 | if (options.style && options.style !== 'virtualHosted' && 181 | options.style !== 'path') { 182 | throw new Error('style must be "virtualHosted" or "path"'); 183 | } 184 | 185 | if (options.port !== undefined && isNaN(parseInt(options.port))) { 186 | throw new Error('port must be a number.'); 187 | } 188 | 189 | var invalidness = isInvalid(options.bucket); 190 | var dnsUncompliance = isNotDnsCompliant(options.bucket); 191 | 192 | if (invalidness) { 193 | throw new Error('Bucket name "' + options.bucket + '" ' + invalidness + '.'); 194 | } 195 | 196 | // Save original options, we will need them for Client#copyTo 197 | this.options = utils.merge({}, options); 198 | 199 | // Make sure we don't override options the user passes in. 200 | options = utils.merge({}, options); 201 | autoDetermineStyle(options); 202 | 203 | if (!options.endpoint) { 204 | if (!options.region || options.region === 'us-standard' || options.region === 'us-east-1') { 205 | options.endpoint = 's3.amazonaws.com'; 206 | options.region = 'us-standard'; 207 | } else { 208 | options.endpoint = 's3-' + options.region + '.amazonaws.com'; 209 | } 210 | 211 | if (options.region !== 'us-standard') { 212 | if (dnsUncompliance) { 213 | throw new Error('Outside of the us-standard region, bucket names must' + 214 | ' be DNS-compliant. The name "' + options.bucket + 215 | '" ' + dnsUncompliance + '.'); 216 | } 217 | } 218 | } else { 219 | options.region = undefined; 220 | } 221 | 222 | var portSuffix = 'undefined' == typeof options.port ? "" : ":" + options.port; 223 | this.secure = 'undefined' == typeof options.port; 224 | 225 | if (options.style === 'virtualHosted') { 226 | this.host = options.bucket + '.' + options.endpoint; 227 | this.urlBase = options.bucket + '.' + options.endpoint + portSuffix; 228 | } else { 229 | this.host = options.endpoint; 230 | this.urlBase = options.endpoint + portSuffix + '/' + options.bucket; 231 | } 232 | 233 | // HTTP in Node.js < 0.12 is horribly broken, and leads to lots of "socket 234 | // hang up" errors: https://github.com/LearnBoost/knox/issues/116. See 235 | // https://github.com/LearnBoost/knox/issues/116#issuecomment-15045187 and 236 | // https://github.com/substack/hyperquest#rant 237 | this.agent = false; 238 | 239 | utils.merge(this, options); 240 | 241 | this.url = this.secure ? this.https : this.http; 242 | }; 243 | 244 | /** 245 | * Request with `filename` the given `method`, and optional `headers`. 246 | * 247 | * @param {String} method 248 | * @param {String} filename 249 | * @param {Object} headers 250 | * @return {ClientRequest} 251 | * @api private 252 | */ 253 | 254 | Client.prototype.request = function(method, filename, headers){ 255 | var options = { hostname: this.host, agent: this.agent, port: this.port } 256 | , date = new Date 257 | , headers = headers || {} 258 | , fixedFilename = ensureLeadingSlash(filename); 259 | 260 | // Default headers 261 | headers.Date = date.toUTCString() 262 | if (this.style === 'virtualHosted') { 263 | headers.Host = this.host; 264 | } 265 | 266 | if ('undefined' != typeof this.token) 267 | headers['x-amz-security-token'] = this.token; 268 | 269 | // Authorization header 270 | headers.Authorization = auth.authorization({ 271 | key: this.key 272 | , secret: this.secret 273 | , verb: method 274 | , date: date 275 | , resource: auth.canonicalizeResource('/' + this.bucket + fixedFilename) 276 | , contentType: getHeader(headers, 'content-type') 277 | , md5: getHeader(headers, 'content-md5') || '' 278 | , amazonHeaders: auth.canonicalizeHeaders(headers) 279 | }); 280 | 281 | var pathPrefix = this.style === 'path' ? '/' + this.bucket : ''; 282 | 283 | // Issue request 284 | options.method = method; 285 | options.path = pathPrefix + fixedFilename; 286 | options.headers = headers; 287 | var req = (this.secure ? https : http).request(options); 288 | req.url = this.url(filename); 289 | debug('%s %s', method, req.url); 290 | 291 | return req; 292 | }; 293 | 294 | /** 295 | * PUT data to `filename` with optional `headers`. 296 | * 297 | * Example: 298 | * 299 | * // Fetch the size 300 | * fs.stat('Readme.md', function(err, stat){ 301 | * // Create our request 302 | * var req = client.put('/test/Readme.md', { 303 | * 'Content-Length': stat.size 304 | * , 'Content-Type': 'text/plain' 305 | * }); 306 | * fs.readFile('Readme.md', function(err, buf){ 307 | * // Output response 308 | * req.on('response', function(res){ 309 | * console.log(res.statusCode); 310 | * console.log(res.headers); 311 | * res.pipe(fs.createWriteStream('Readme.md')); 312 | * }); 313 | * // Send the request with the file's Buffer obj 314 | * req.end(buf); 315 | * }); 316 | * }); 317 | * 318 | * @param {String} filename 319 | * @param {Object} headers 320 | * @return {ClientRequest} 321 | * @api public 322 | */ 323 | 324 | Client.prototype.put = function(filename, headers){ 325 | headers = utils.merge({}, headers || {}); 326 | return this.request('PUT', encodeSpecialCharacters(filename), headers); 327 | }; 328 | 329 | /** 330 | * PUT the file at `src` to `filename`, with callback `fn` 331 | * receiving a possible exception, and the response object. 332 | * 333 | * Example: 334 | * 335 | * client 336 | * .putFile('package.json', '/test/package.json', function(err, res){ 337 | * if (err) throw err; 338 | * console.log(res.statusCode); 339 | * console.log(res.headers); 340 | * }); 341 | * 342 | * @param {String} src 343 | * @param {String} filename 344 | * @param {Object|Function} headers 345 | * @param {Function} fn 346 | * @return {EventEmitter} 347 | * @api public 348 | */ 349 | 350 | Client.prototype.putFile = function(src, filename, headers, fn){ 351 | var self = this; 352 | var emitter = new Emitter; 353 | 354 | if ('function' == typeof headers) { 355 | fn = headers; 356 | headers = {}; 357 | } 358 | 359 | debug('put %s', src); 360 | fs.stat(src, function (err, stat) { 361 | if (err) return fn(err); 362 | 363 | var contentType = mime.lookup(src); 364 | 365 | // Add charset if it's known. 366 | var charset = mime.charsets.lookup(contentType); 367 | if (charset) { 368 | contentType += '; charset=' + charset; 369 | } 370 | 371 | headers = utils.merge({ 372 | 'Content-Length': stat.size 373 | , 'Content-Type': contentType 374 | }, headers); 375 | 376 | var stream = fs.createReadStream(src); 377 | 378 | var req = self.putStream(stream, filename, headers, fn); 379 | 380 | req.on('progress', emitter.emit.bind(emitter, 'progress')); 381 | }); 382 | 383 | return emitter; 384 | }; 385 | 386 | /** 387 | * PUT the given `stream` as `filename` with `headers`. 388 | * `headers` must contain `'Content-Length'` at least. 389 | * 390 | * @param {Stream} stream 391 | * @param {String} filename 392 | * @param {Object} headers 393 | * @param {Function} fn 394 | * @return {ClientRequest} 395 | * @api public 396 | */ 397 | 398 | Client.prototype.putStream = function(stream, filename, headers, fn){ 399 | var contentLength = getHeader(headers, 'content-length'); 400 | if (contentLength === null) { 401 | process.nextTick(function () { 402 | fn(new Error('You must specify a Content-Length header.')); 403 | }); 404 | return; 405 | } 406 | 407 | var self = this; 408 | var req = self.put(filename, headers); 409 | 410 | fn = once(fn); 411 | registerReqListeners(req, fn); 412 | stream.on('error', fn); 413 | 414 | var counter = new StreamCounter(); 415 | counter.on('progress', function(){ 416 | req.emit('progress', { 417 | percent: counter.bytes / contentLength * 100 | 0 418 | , written: counter.bytes 419 | , total: contentLength 420 | }); 421 | }); 422 | 423 | stream.pipe(counter); 424 | stream.pipe(req); 425 | return req; 426 | }; 427 | 428 | /** 429 | * PUT the given `buffer` as `filename` with optional `headers`. 430 | * Callback `fn` receives a possible exception and the response object. 431 | * 432 | * @param {Buffer} buffer 433 | * @param {String} filename 434 | * @param {Object|Function} headers 435 | * @param {Function} fn 436 | * @return {ClientRequest} 437 | * @api public 438 | */ 439 | 440 | Client.prototype.putBuffer = function(buffer, filename, headers, fn){ 441 | if ('function' == typeof headers) { 442 | fn = headers; 443 | headers = {}; 444 | } 445 | 446 | headers['Content-Length'] = buffer.length; 447 | 448 | var req = this.put(filename, headers); 449 | fn = once(fn); 450 | registerReqListeners(req, fn); 451 | req.end(buffer); 452 | return req; 453 | }; 454 | 455 | /** 456 | * Copy files from `sourceFilename` to `destFilename` with optional `headers`. 457 | * 458 | * @param {String} sourceFilename 459 | * @param {String} destFilename 460 | * @param {Object} headers 461 | * @return {ClientRequest} 462 | * @api public 463 | */ 464 | 465 | Client.prototype.copy = function(sourceFilename, destFilename, headers){ 466 | return this.put(destFilename, getCopyHeaders(this.bucket, sourceFilename, headers)); 467 | }; 468 | 469 | /** 470 | * Copy files from `sourceFilename` to `destFilename` with optional `headers` 471 | * and callback `fn` with a possible exception and the response. 472 | * 473 | * @param {String} sourceFilename 474 | * @param {String} destFilename 475 | * @param {Object|Function} headers 476 | * @param {Function} fn 477 | * @api public 478 | */ 479 | 480 | Client.prototype.copyFile = function(sourceFilename, destFilename, headers, fn){ 481 | if ('function' == typeof headers) { 482 | fn = headers; 483 | headers = {}; 484 | } 485 | 486 | var req = this.copy(sourceFilename, destFilename, headers); 487 | fn = once(fn); 488 | registerReqListeners(req, fn); 489 | req.end(); 490 | return req; 491 | }; 492 | 493 | /** 494 | * Copy files from `sourceFilename` to `destFilename` of the bucket `destBucket` 495 | * with optional `headers`. 496 | * 497 | * @param {String} sourceFilename 498 | * @param {String|Object} destBucket 499 | * @param {String} destFilename 500 | * @param {Object} headers 501 | * @return {ClientRequest} 502 | * @api public 503 | */ 504 | 505 | Client.prototype.copyTo = function(sourceFilename, destBucket, destFilename, headers){ 506 | var options = utils.merge({}, this.options); 507 | if (typeof destBucket == 'string') { 508 | options.bucket = destBucket; 509 | } else { 510 | utils.merge(options, destBucket); 511 | } 512 | var client = exports.createClient(options); 513 | return client.put(destFilename, getCopyHeaders(this.bucket, sourceFilename, headers)); 514 | }; 515 | 516 | /** 517 | * Copy file from `sourceFilename` to `destFilename` of the bucket `destBucket 518 | * with optional `headers` and callback `fn` with a possible exception and the response. 519 | * 520 | * @param {String} sourceFilename 521 | * @param {String} destBucket 522 | * @param {String} destFilename 523 | * @param {Object|Function} headers 524 | * @param {Function} fn 525 | * @api public 526 | */ 527 | 528 | Client.prototype.copyFileTo = function(sourceFilename, destBucket, destFilename, headers, fn){ 529 | if ('function' == typeof headers) { 530 | fn = headers; 531 | headers = {}; 532 | } 533 | 534 | var req = this.copyTo(sourceFilename, destBucket, destFilename, headers); 535 | fn = once(fn); 536 | registerReqListeners(req, fn); 537 | req.end(); 538 | return req; 539 | }; 540 | 541 | /** 542 | * GET `filename` with optional `headers`. 543 | * 544 | * @param {String} filename 545 | * @param {Object} headers 546 | * @return {ClientRequest} 547 | * @api public 548 | */ 549 | 550 | Client.prototype.get = function(filename, headers){ 551 | return this.request('GET', encodeSpecialCharacters(filename), headers); 552 | }; 553 | 554 | /** 555 | * GET `filename` with optional `headers` and callback `fn` 556 | * with a possible exception and the response. 557 | * 558 | * @param {String} filename 559 | * @param {Object|Function} headers 560 | * @param {Function} fn 561 | * @api public 562 | */ 563 | 564 | Client.prototype.getFile = function(filename, headers, fn){ 565 | if ('function' == typeof headers) { 566 | fn = headers; 567 | headers = {}; 568 | } 569 | 570 | var req = this.get(filename, headers); 571 | registerReqListeners(req, fn); 572 | req.end(); 573 | return req; 574 | }; 575 | 576 | /** 577 | * Issue a HEAD request on `filename` with optional `headers. 578 | * 579 | * @param {String} filename 580 | * @param {Object} headers 581 | * @return {ClientRequest} 582 | * @api public 583 | */ 584 | 585 | Client.prototype.head = function(filename, headers){ 586 | return this.request('HEAD', encodeSpecialCharacters(filename), headers); 587 | }; 588 | 589 | /** 590 | * Issue a HEAD request on `filename` with optional `headers` 591 | * and callback `fn` with a possible exception and the response. 592 | * 593 | * @param {String} filename 594 | * @param {Object|Function} headers 595 | * @param {Function} fn 596 | * @api public 597 | */ 598 | 599 | Client.prototype.headFile = function(filename, headers, fn){ 600 | if ('function' == typeof headers) { 601 | fn = headers; 602 | headers = {}; 603 | } 604 | var req = this.head(filename, headers); 605 | fn = once(fn); 606 | registerReqListeners(req, fn); 607 | req.end(); 608 | return req; 609 | }; 610 | 611 | /** 612 | * DELETE `filename` with optional `headers. 613 | * 614 | * @param {String} filename 615 | * @param {Object} headers 616 | * @return {ClientRequest} 617 | * @api public 618 | */ 619 | 620 | Client.prototype.del = function(filename, headers){ 621 | return this.request('DELETE', encodeSpecialCharacters(filename), headers); 622 | }; 623 | 624 | /** 625 | * DELETE `filename` with optional `headers` 626 | * and callback `fn` with a possible exception and the response. 627 | * 628 | * @param {String} filename 629 | * @param {Object|Function} headers 630 | * @param {Function} fn 631 | * @api public 632 | */ 633 | 634 | Client.prototype.deleteFile = function(filename, headers, fn){ 635 | if ('function' == typeof headers) { 636 | fn = headers; 637 | headers = {}; 638 | } 639 | var req = this.del(filename, headers); 640 | fn = once(fn); 641 | registerReqListeners(req, fn); 642 | req.end(); 643 | return req; 644 | }; 645 | 646 | function xmlEscape(string) { 647 | return string 648 | .replace(/&/g, '&') 649 | .replace(//g, '>') 651 | .replace(/"/g, '"'); 652 | } 653 | 654 | function makeDeleteXmlBuffer(keys) { 655 | var tags = keys.map(function(key){ 656 | return '' + 657 | xmlEscape(removeLeadingSlash(key)) + 658 | ''; 659 | }); 660 | return new Buffer('' + 661 | '' + tags.join('') + '', 'utf8'); 662 | } 663 | 664 | /** 665 | * Delete up to 1000 files at a time, with optional `headers` 666 | * and callback `fn` with a possible exception and the response. 667 | * 668 | * @param {Array[String]} filenames 669 | * @param {Object|Function} headers 670 | * @param {Function} fn 671 | * @api public 672 | */ 673 | 674 | Client.prototype.deleteMultiple = function(filenames, headers, fn){ 675 | if (filenames.length > BUCKET_OPS_MAX) { 676 | throw new Error('Can only delete up to ' + BUCKET_OPS_MAX + ' files ' + 677 | 'at a time. You\'ll need to batch them.'); 678 | } 679 | 680 | if ('function' == typeof headers) { 681 | fn = headers; 682 | headers = {}; 683 | } 684 | 685 | var xml = makeDeleteXmlBuffer(filenames); 686 | 687 | headers['Content-Length'] = xml.length; 688 | headers['Content-MD5'] = crypto.createHash('md5').update(xml).digest('base64'); 689 | 690 | var req = this.request('POST', '/?delete', headers); 691 | fn = once(fn); 692 | registerReqListeners(req, fn); 693 | req.end(xml); 694 | return req; 695 | }; 696 | 697 | /** 698 | * Possible params for Client#list. 699 | * 700 | * @type {Object} 701 | */ 702 | 703 | var LIST_PARAMS = { 704 | delimiter: true 705 | , marker: true 706 | ,'max-keys': true 707 | , prefix: true 708 | }; 709 | 710 | /** 711 | * Normalization map for Client#list. 712 | * 713 | * @type {Object} 714 | */ 715 | 716 | var RESPONSE_NORMALIZATION = { 717 | MaxKeys: Number, 718 | IsTruncated: Boolean, 719 | LastModified: Date, 720 | Size: Number, 721 | Contents: Array, 722 | CommonPrefixes: Array 723 | }; 724 | 725 | /** 726 | * Convert data we get from S3 xml in Client#list, since every primitive 727 | * value there is a string. 728 | * 729 | * @type {Object} 730 | */ 731 | 732 | function normalizeResponse(data) { 733 | for (var key in data) { 734 | var Constr = RESPONSE_NORMALIZATION[key]; 735 | 736 | if (Constr) { 737 | if (Constr === Date) { 738 | data[key] = new Date(data[key]); 739 | } else if (Constr === Array) { 740 | // If there's only one element in the array xml2js doesn't know that 741 | // it should be an array; array-ify it. 742 | if (!Array.isArray(data[key])) { 743 | data[key] = [data[key]]; 744 | } 745 | } else if (Constr === Boolean) { 746 | data[key] = data[key] === 'true'; 747 | } else { 748 | data[key] = Constr(data[key]); 749 | } 750 | } 751 | 752 | if (Array.isArray(data[key])) { 753 | data[key].forEach(normalizeResponse); 754 | } 755 | } 756 | } 757 | 758 | /** 759 | * List up to 1000 objects at a time, with optional `headers`, `params` 760 | * and callback `fn` with a possible exception and the response. 761 | * 762 | * @param {Object|Function} params 763 | * @param {Object|Function} headers 764 | * @param {Function} fn 765 | * @api public 766 | */ 767 | 768 | Client.prototype.list = function(params, headers, fn){ 769 | if ('function' == typeof headers) { 770 | fn = headers; 771 | headers = {}; 772 | } 773 | 774 | if ('function' == typeof params) { 775 | fn = params; 776 | params = null; 777 | } 778 | 779 | if (params && !LIST_PARAMS[Object.keys(params)[0]]) { 780 | headers = params; 781 | params = null; 782 | } 783 | 784 | var url = params ? '?' + qs.stringify(params) : ''; 785 | var req = this.request('GET', url, headers); 786 | registerReqListeners(req, function(err, res){ 787 | if (err) return fn(err); 788 | 789 | var xmlStr = ''; 790 | 791 | res.on('data', function(chunk){ 792 | xmlStr += chunk; 793 | }); 794 | 795 | res.on('end', function(){ 796 | new xml2js.Parser({explicitArray: false, explicitRoot: false}) 797 | .parseString(xmlStr, function(err, data){ 798 | if (err) return fn(err); 799 | if (data == null) return fn(new Error('null response received')); 800 | 801 | delete data.$; 802 | normalizeResponse(data); 803 | 804 | if (!('Contents' in data)) { 805 | data.Contents = []; 806 | } 807 | 808 | fn(null, data); 809 | }); 810 | }); 811 | }); 812 | req.on('error', fn); 813 | req.end(); 814 | return req; 815 | }; 816 | 817 | /** 818 | * Return a url to the given `filename`. 819 | * 820 | * @param {String} filename 821 | * @return {String} 822 | * @api public 823 | */ 824 | 825 | Client.prototype.http = function(filename){ 826 | filename = encodeSpecialCharacters(ensureLeadingSlash(filename)); 827 | 828 | return 'http://' + this.urlBase + filename; 829 | }; 830 | 831 | /** 832 | * Return an HTTPS url to the given `filename`. 833 | * 834 | * @param {String} filename 835 | * @return {String} 836 | * @api public 837 | */ 838 | 839 | Client.prototype.https = function(filename){ 840 | filename = encodeSpecialCharacters(ensureLeadingSlash(filename)); 841 | 842 | return 'https://' + this.urlBase + filename; 843 | }; 844 | 845 | /** 846 | * Return an S3 presigned url to the given `filename`. 847 | * 848 | * @param {String} filename 849 | * @param {Date} expiration 850 | * @param {Object} options: can take verb, contentType, and qs object 851 | * @return {String} 852 | * @api public 853 | */ 854 | 855 | Client.prototype.signedUrl = function(filename, expiration, options){ 856 | var epoch = Math.floor(expiration.getTime()/1000) 857 | , pathname = url.parse(filename).pathname 858 | , resource = '/' + this.bucket + ensureLeadingSlash(pathname); 859 | 860 | if (options && options.qs) { 861 | resource += '?' + decodeURIComponent(qs.stringify(options.qs)); 862 | } 863 | 864 | var signature = auth.signQuery({ 865 | secret: this.secret 866 | , date: epoch 867 | , resource: resource 868 | , verb: (options && options.verb) || 'GET' 869 | , contentType: options && options.contentType 870 | , extraHeaders : options && options.extraHeaders 871 | , token: this.token 872 | }); 873 | 874 | var queryString = qs.stringify(utils.merge({ 875 | Expires: epoch, 876 | AWSAccessKeyId: this.key, 877 | Signature: signature 878 | }, (options && options.qs) || {})); 879 | 880 | if (typeof this.token !== 'undefined') 881 | queryString += '&x-amz-security-token=' + encodeURIComponent(this.token); 882 | 883 | return this.url(filename) + '?' + queryString; 884 | }; 885 | 886 | /** 887 | * Shortcut for `new Client()`. 888 | * 889 | * @param {Object} options 890 | * @see Client() 891 | * @api public 892 | */ 893 | 894 | exports.createClient = function(options){ 895 | return new Client(options); 896 | }; 897 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /*! 4 | * knox 5 | * Copyright(c) 2010–2012 LearnBoost 6 | * MIT Licensed 7 | */ 8 | 9 | /** 10 | * Client is the main export. 11 | */ 12 | 13 | exports = module.exports = require('./client'); 14 | 15 | /** 16 | * Expose utilities. 17 | * 18 | * @type Object 19 | */ 20 | 21 | exports.utils = require('./utils'); 22 | 23 | /** 24 | * Expose auth utils. 25 | * 26 | * @type Object 27 | */ 28 | 29 | exports.auth = require('./auth'); 30 | -------------------------------------------------------------------------------- /lib/utils.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /*! 4 | * knox - utils 5 | * Copyright(c) 2010 LearnBoost 6 | * MIT Licensed 7 | */ 8 | 9 | /** 10 | * Merge object `b` with object `a`. 11 | * 12 | * @param {Object} a 13 | * @param {Object} b 14 | * @return {Object} a 15 | * @api private 16 | */ 17 | 18 | exports.merge = function(a, b){ 19 | var keys = Object.keys(b); 20 | for (var i = 0, len = keys.length; i < len; ++i) { 21 | var key = keys[i]; 22 | a[key] = b[key] 23 | } 24 | return a; 25 | }; 26 | 27 | /** 28 | * Base64. 29 | */ 30 | 31 | exports.base64 = { 32 | 33 | /** 34 | * Base64 encode the given `str`. 35 | * 36 | * @param {String} str 37 | * @return {String} 38 | * @api private 39 | */ 40 | 41 | encode: function(str){ 42 | return new Buffer(str).toString('base64'); 43 | }, 44 | 45 | /** 46 | * Base64 decode the given `str`. 47 | * 48 | * @param {String} str 49 | * @return {String} 50 | * @api private 51 | */ 52 | 53 | decode: function(str){ 54 | return new Buffer(str, 'base64').toString(); 55 | } 56 | }; 57 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "knox", 3 | "description": "Amazon S3 client", 4 | "keywords": [ 5 | "aws", 6 | "amazon", 7 | "s3" 8 | ], 9 | "version": "0.9.3", 10 | "author": "TJ Holowaychuk ", 11 | "contributors": [ 12 | "TJ Holowaychuk ", 13 | "Domenic Denicola ", 14 | "Oleg Slobodskoi " 15 | ], 16 | "license": "MIT", 17 | "main": "./lib/index.js", 18 | "repository": { 19 | "type": "git", 20 | "url": "git://github.com/LearnBoost/knox.git" 21 | }, 22 | "bugs": "https://github.com/LearnBoost/knox/issues", 23 | "dependencies": { 24 | "mime": "*", 25 | "xml2js": "^0.4.4", 26 | "debug": "^2.2.0", 27 | "stream-counter": "^1.0.0", 28 | "once": "^1.3.0" 29 | }, 30 | "devDependencies": { 31 | "mocha": "*" 32 | }, 33 | "scripts": { 34 | "test": "mocha" 35 | }, 36 | "directories": { 37 | "lib": "./lib" 38 | }, 39 | "engines": { 40 | "node": ">= 0.8" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /test/auth.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var knox = require('..') 4 | , auth = knox.auth 5 | , assert = require('assert'); 6 | 7 | describe('knox.auth', function () { 8 | describe('.stringToSign()', function () { 9 | specify('for a basic PUT', function () { 10 | var actual = auth.stringToSign({ 11 | verb: 'PUT' 12 | , md5: '09c68b914d66457508f6ad727d860d5b' 13 | , contentType: 'text/plain' 14 | , resource: '/learnboost' 15 | , date: new Date('Mon, May 25 1987 00:00:00 GMT') 16 | }); 17 | 18 | var expected = [ 19 | 'PUT' 20 | , '09c68b914d66457508f6ad727d860d5b' 21 | , 'text/plain' 22 | , new Date('Mon, May 25 1987 00:00:00 GMT').toUTCString() 23 | , '/learnboost' 24 | ].join('\n'); 25 | 26 | assert.equal(actual, expected); 27 | }); 28 | }); 29 | 30 | describe('.sign()', function () { 31 | specify('for a basic PUT', function () { 32 | var actual = auth.sign({ 33 | verb: 'PUT' 34 | , secret: 'test' 35 | , md5: '09c68b914d66457508f6ad727d860d5b' 36 | , contentType: 'text/plain' 37 | , resource: '/learnboost' 38 | , date: new Date('Mon, May 25 1987 00:00:00 GMT') 39 | }); 40 | 41 | assert.equal(actual, '7xIdjyy+W17/k0le5kwBnfrZTiM='); 42 | }); 43 | }); 44 | 45 | describe('.authorization() [from the Amazon docs]', function () { 46 | // http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationExamples 47 | 48 | specify('Example Object GET', function () { 49 | var actual = auth.authorization({ 50 | verb: 'GET' 51 | , key: 'AKIAIOSFODNN7EXAMPLE' 52 | , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' 53 | , resource: auth.canonicalizeResource('/johnsmith/photos/puppy.jpg') 54 | , date: 'Tue, 27 Mar 2007 19:36:42 +0000' 55 | }); 56 | 57 | assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:bWq2s1WEIj+Ydj0vQ697zp+IXMU='); 58 | }); 59 | 60 | specify('Example Object PUT', function () { 61 | var actual = auth.authorization({ 62 | verb: 'PUT' 63 | , key: 'AKIAIOSFODNN7EXAMPLE' 64 | , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' 65 | , resource: auth.canonicalizeResource('/johnsmith/photos/puppy.jpg') 66 | , contentType: 'image/jpeg' 67 | , date: 'Tue, 27 Mar 2007 21:15:45 +0000' 68 | }); 69 | 70 | assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:MyyxeRY7whkBe+bq8fHCL/2kKUg='); 71 | }); 72 | 73 | specify('Example List', function () { 74 | var actual = auth.authorization({ 75 | verb: 'GET' 76 | , key: 'AKIAIOSFODNN7EXAMPLE' 77 | , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' 78 | , resource: auth.canonicalizeResource('/johnsmith/') 79 | , date: 'Tue, 27 Mar 2007 19:42:41 +0000' 80 | }); 81 | 82 | assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:htDYFYduRNen8P9ZfE/s9SuKy0U='); 83 | }); 84 | 85 | specify('Example Fetch', function () { 86 | var actual = auth.authorization({ 87 | verb: 'GET' 88 | , key: 'AKIAIOSFODNN7EXAMPLE' 89 | , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' 90 | , resource: auth.canonicalizeResource('/johnsmith/?acl') 91 | , date: 'Tue, 27 Mar 2007 19:44:46 +0000' 92 | }); 93 | 94 | assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:c2WLPFtWHVgbEmeEG93a4cG37dM='); 95 | }); 96 | 97 | specify('Example Delete', function () { 98 | // This is modified from the docs: knox does not allow setting the date 99 | // through x-amz-date, so we test that the x-amz-date is ignored and the 100 | // date is instead used. 101 | var actual = auth.authorization({ 102 | verb: 'DELETE' 103 | , key: 'AKIAIOSFODNN7EXAMPLE' 104 | , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' 105 | , resource: auth.canonicalizeResource('/johnsmith/photos/puppy.jpg') 106 | , amazonHeaders: auth.canonicalizeHeaders({ 107 | 'x-amz-date': 'Tue, 27 Mar 2007 21:20:27 +0000' 108 | }) 109 | , date: 'Tue, 27 Mar 2007 21:20:26 +0000' 110 | }); 111 | 112 | assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:lx3byBScXR6KzyMaifNkardMwNk='); 113 | }); 114 | 115 | specify.skip('Example Upload', function () { 116 | // Knox doesn't support multiple values for a single header; see 117 | // discussion at https://github.com/LearnBoost/knox/pull/6. Elegant pull 118 | // requests to implement this feature welcome! 119 | }); 120 | 121 | specify('Example List All My Buckets', function () { 122 | var actual = auth.authorization({ 123 | verb: 'GET' 124 | , key: 'AKIAIOSFODNN7EXAMPLE' 125 | , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' 126 | , resource: auth.canonicalizeResource('/') 127 | , date: 'Wed, 28 Mar 2007 01:29:59 +0000' 128 | }); 129 | 130 | assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:qGdzdERIC03wnaRNKh6OqZehG9s='); 131 | }); 132 | 133 | specify('Example Unicode Keys', function () { 134 | var actual = auth.authorization({ 135 | verb: 'GET' 136 | , key: 'AKIAIOSFODNN7EXAMPLE' 137 | , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' 138 | , resource: auth.canonicalizeResource('/dictionary/fran%C3%A7ais/pr%c3%a9f%c3%a8re') 139 | , date: 'Wed, 28 Mar 2007 01:49:49 +0000' 140 | }); 141 | 142 | assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:DNEZGsoieTZ92F3bUfSPQcbGmlM='); 143 | }); 144 | }); 145 | 146 | describe('.canonicalizeHeaders()', function () { 147 | specify('with no Amazon headers', function () { 148 | assert.equal(auth.canonicalizeHeaders({}), ''); 149 | }); 150 | 151 | specify('with several Amazon headers', function () { 152 | var actual = auth.canonicalizeHeaders({ 153 | 'X-Amz-Copy-Source-If-Match': 'etagvalue' 154 | , 'X-Amz-Copy-Source': '/bucket/object' 155 | , 'X-Amz-Date': 'some date' 156 | , 'X-Amz-Acl': 'private' 157 | , 'X-Foo': 'bar' 158 | }); 159 | 160 | var expected = [ 161 | 'x-amz-acl:private' 162 | , 'x-amz-copy-source:/bucket/object' 163 | , 'x-amz-copy-source-if-match:etagvalue' 164 | ].join('\n'); 165 | 166 | assert.equal(actual, expected); 167 | }); 168 | }); 169 | 170 | describe('.queryStringToSign()', function() { 171 | var date = new Date().toUTCString() 172 | , resource = 'foo.jpg'; 173 | 174 | specify('for a HEAD request', function () { 175 | var actual = auth.queryStringToSign({ 176 | verb: 'HEAD' 177 | , date: date 178 | , resource: resource 179 | }); 180 | 181 | var expected = [ 182 | 'HEAD\n\n' 183 | , date 184 | , resource 185 | ].join('\n'); 186 | 187 | assert.equal(actual, expected); 188 | }); 189 | 190 | specify('for a GET request', function () { 191 | var actual = auth.queryStringToSign({ 192 | verb: 'GET' 193 | , date: date 194 | , resource: resource 195 | }); 196 | 197 | var expected = [ 198 | 'GET\n\n' 199 | , date 200 | , resource 201 | ].join('\n'); 202 | 203 | assert.equal(actual, expected); 204 | }); 205 | 206 | specify('for a GET request with a token', function () { 207 | var actual = auth.queryStringToSign({ 208 | verb: 'GET' 209 | , date: date 210 | , resource: resource 211 | , token: 'foobar' 212 | }); 213 | 214 | var expected = [ 215 | 'GET\n\n' 216 | , date 217 | , 'x-amz-security-token:foobar' 218 | , resource 219 | ].join('\n'); 220 | 221 | assert.equal(actual, expected); 222 | }); 223 | }); 224 | 225 | describe('.canonicalizeResource()', function () { 226 | specify('for a bucket alone', function () { 227 | assert.equal(auth.canonicalizeResource('/bucket/'), '/bucket/'); 228 | }); 229 | 230 | specify('for a bucket, folder, and file', function () { 231 | assert.equal(auth.canonicalizeResource('/bucket/test/user2.json'), '/bucket/test/user2.json'); 232 | }); 233 | 234 | specify('for a bucket\'s ACL list URL', function () { 235 | assert.equal(auth.canonicalizeResource('/bucket/?acl'), '/bucket/?acl'); 236 | }); 237 | 238 | specify('for a bucket\'s delete multiple URL', function () { 239 | assert.equal(auth.canonicalizeResource('/bucket/?delete'), '/bucket/?delete'); 240 | }); 241 | 242 | specify('for a bucket filtered by a simple prefix', function () { 243 | assert.equal(auth.canonicalizeResource('/bucket/?prefix=logs'), '/bucket/'); 244 | }); 245 | 246 | specify('for a bucket filtered by a simple prefix and a delimiter', function () { 247 | assert.equal(auth.canonicalizeResource('/bucket/?prefix=logs/&delimiter=/'), '/bucket/'); 248 | }); 249 | 250 | specify('for a bucket filtered by a complex prefix and a delimiter', function () { 251 | assert.equal(auth.canonicalizeResource('/bucket/?prefix=log%20files/&delimiter=/'), '/bucket/'); 252 | }); 253 | }); 254 | }); 255 | -------------------------------------------------------------------------------- /test/createClient.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var knox = require('..') 4 | , assert = require('assert'); 5 | 6 | describe('knox.createClient()', function () { 7 | describe('invalid options', function () { 8 | it('should ask for a key when nothing is passed', function () { 9 | assert.throws( 10 | function () { knox.createClient({}); }, 11 | /aws "key" required/ 12 | ); 13 | }); 14 | 15 | it('should ask for a secret when only a key is passed', function () { 16 | assert.throws( 17 | function () { knox.createClient({ key: 'foo' }); }, 18 | /aws "secret" required/ 19 | ); 20 | }); 21 | 22 | it('should ask for a bucket when only a key and secret are passed', function () { 23 | assert.throws( 24 | function () { knox.createClient({ key: 'foo', secret: 'bar' }); }, 25 | /aws "bucket" required/ 26 | ); 27 | }); 28 | 29 | it('should throw when an invalid style is given', function () { 30 | assert.throws( 31 | function () { 32 | knox.createClient({ 33 | key: 'foo' 34 | , secret: 'bar' 35 | , bucket: 'bucket' 36 | , style: 'gangnam' 37 | }); 38 | }, 39 | function (err) { 40 | return err instanceof Error && 41 | /style must be "virtualHosted" or "path"/.test(err.message); 42 | } 43 | ); 44 | }); 45 | 46 | it('should throw when an invalid port', function () { 47 | assert.throws( 48 | function () { 49 | knox.createClient({ 50 | key: 'foo' 51 | , secret: 'bar' 52 | , bucket: 'bucket' 53 | , port: '' 54 | }); 55 | }, 56 | function (err) { 57 | return err instanceof Error && 58 | /port must be a number/.test(err.message); 59 | } 60 | ); 61 | }); 62 | 63 | describe('bucket names', function () { 64 | describe('in us-standard region', function () { 65 | it('should throw when bucket names are too short', function () { 66 | assert.throws( 67 | function () { 68 | knox.createClient({ key: 'foo', secret: 'bar', bucket: 'bu' }); 69 | }, 70 | /less than 3 characters/ 71 | ); 72 | }); 73 | 74 | it('should throw when bucket names are too long', function () { 75 | var bucket = new Array(257).join('b'); 76 | assert.throws( 77 | function () { 78 | knox.createClient({ key: 'foo', secret: 'bar', bucket: bucket }); 79 | }, 80 | /more than 255 characters/ 81 | ); 82 | }); 83 | 84 | it('should throw when bucket names contain invalid characters', function () { 85 | assert.throws( 86 | function () { 87 | knox.createClient({ key: 'foo', secret: 'bar', bucket: 'buc!ket' }); 88 | }, 89 | /invalid characters/ 90 | ); 91 | }); 92 | }); 93 | 94 | describe('in us-west-1 region', function () { 95 | it('should throw when bucket names are too long', function () { 96 | var bucket = new Array(65).join('b'); 97 | assert.throws( 98 | function () { 99 | knox.createClient({ 100 | key: 'foo' 101 | , secret: 'bar' 102 | , bucket: bucket 103 | , region: 'us-west-1' 104 | }); 105 | }, 106 | /more than 63 characters/ 107 | ); 108 | }); 109 | 110 | it('should throw when bucket names contain invalid characters', function () { 111 | assert.throws( 112 | function () { 113 | knox.createClient({ 114 | key: 'foo' 115 | , secret: 'bar' 116 | , bucket: 'buck_et' 117 | , region: 'us-west-1' 118 | }); 119 | }, 120 | /valid period-separated labels/ 121 | ); 122 | }); 123 | 124 | it('should throw when bucket names look like IPv4 addresses', function () { 125 | assert.throws( 126 | function () { 127 | knox.createClient({ 128 | key: 'foo' 129 | , secret: 'bar' 130 | , bucket: '192.0.0.12' 131 | , region: 'us-west-1' 132 | }); 133 | }, 134 | /IPv4 address/ 135 | ); 136 | }); 137 | }); 138 | 139 | describe('when forcing virtual hosted style', function () { 140 | it('should throw when bucket names are too long', function () { 141 | var bucket = new Array(65).join('b'); 142 | assert.throws( 143 | function () { 144 | knox.createClient({ 145 | key: 'foo' 146 | , secret: 'bar' 147 | , bucket: bucket 148 | , style: 'virtualHosted' 149 | }); 150 | }, 151 | /more than 63 characters/ 152 | ); 153 | }); 154 | 155 | it('should throw when bucket names contain invalid characters', function () { 156 | assert.throws( 157 | function () { 158 | knox.createClient({ 159 | key: 'foo' 160 | , secret: 'bar' 161 | , bucket: 'buck_et' 162 | , style: 'virtualHosted' 163 | }); 164 | }, 165 | /valid period-separated labels/ 166 | ); 167 | }); 168 | 169 | it('should throw when bucket names look like IPv4 addresses', function () { 170 | assert.throws( 171 | function () { 172 | knox.createClient({ 173 | key: 'foo' 174 | , secret: 'bar' 175 | , bucket: '192.0.0.12' 176 | , style: 'virtualHosted' 177 | }); 178 | }, 179 | /IPv4 address/ 180 | ); 181 | }); 182 | }); 183 | }); 184 | }); 185 | 186 | describe('valid options', function () { 187 | it('should copy over basic properties', function () { 188 | var client = knox.createClient({ 189 | key: 'foobar' 190 | , secret: 'baz' 191 | , bucket: 'misc' 192 | }); 193 | 194 | assert.equal(client.key, 'foobar'); 195 | assert.equal(client.secret, 'baz'); 196 | assert.equal(client.bucket, 'misc'); 197 | }); 198 | 199 | describe('with virtual hosted style', function () { 200 | it('should use a default region and endpoint given a bucket', function () { 201 | var client = knox.createClient({ 202 | key: 'foobar' 203 | , secret: 'baz' 204 | , bucket: 'misc' 205 | , style: 'virtualHosted' 206 | }); 207 | 208 | assert.equal(client.secure, true); 209 | assert.equal(client.style, 'virtualHosted'); 210 | assert.equal(client.region, 'us-standard'); 211 | assert.equal(client.endpoint, 's3.amazonaws.com'); 212 | assert.equal(client.url('file'), 'https://misc.s3.amazonaws.com/file'); 213 | }); 214 | 215 | it('should use a custom endpoint directly', function () { 216 | var client = knox.createClient({ 217 | key: 'foobar' 218 | , secret: 'baz' 219 | , bucket: 'misc' 220 | , endpoint: 'objects.dreamhost.com' 221 | , style: 'virtualHosted' 222 | }); 223 | 224 | assert.equal(client.secure, true); 225 | assert.equal(client.style, 'virtualHosted'); 226 | assert.equal(client.region, undefined); 227 | assert.equal(client.domain, undefined); 228 | assert.equal(client.endpoint, 'objects.dreamhost.com'); 229 | assert.equal(client.url('file'), 'https://misc.objects.dreamhost.com/file'); 230 | }); 231 | 232 | it('should derive endpoint correctly from a region', function () { 233 | var client = knox.createClient({ 234 | key: 'foobar' 235 | , secret: 'baz' 236 | , bucket: 'misc' 237 | , style: 'virtualHosted' 238 | , region: 'us-west-1' 239 | }); 240 | 241 | assert.equal(client.secure, true); 242 | assert.equal(client.style, 'virtualHosted'); 243 | assert.equal(client.region, 'us-west-1'); 244 | assert.equal(client.endpoint, 's3-us-west-1.amazonaws.com'); 245 | assert.equal(client.url('file'), 'https://misc.s3-us-west-1.amazonaws.com/file'); 246 | }); 247 | 248 | it('should derive endpoint correctly from explicit us-standard region', function () { 249 | var client = knox.createClient({ 250 | key: 'foobar' 251 | , secret: 'baz' 252 | , bucket: 'misc' 253 | , style: 'virtualHosted' 254 | , region: 'us-standard' 255 | }); 256 | 257 | assert.equal(client.secure, true); 258 | assert.equal(client.style, 'virtualHosted'); 259 | assert.equal(client.region, 'us-standard'); 260 | assert.equal(client.endpoint, 's3.amazonaws.com'); 261 | assert.equal(client.url('file'), 'https://misc.s3.amazonaws.com/file'); 262 | }); 263 | 264 | it('should derive endpoint correctly from explicit us-east-1 region', function () { 265 | var client = knox.createClient({ 266 | key: 'foobar' 267 | , secret: 'baz' 268 | , bucket: 'misc' 269 | , style: 'virtualHosted' 270 | , region: 'us-east-1' 271 | }); 272 | 273 | assert.equal(client.secure, true); 274 | assert.equal(client.style, 'virtualHosted'); 275 | assert.equal(client.region, 'us-standard'); 276 | assert.equal(client.endpoint, 's3.amazonaws.com'); 277 | assert.equal(client.url('file'), 'https://misc.s3.amazonaws.com/file'); 278 | }); 279 | 280 | it('should set secure to false and update the URL when given a port', function () { 281 | var client = knox.createClient({ 282 | key: 'foobar' 283 | , secret: 'baz' 284 | , bucket: 'misc' 285 | , style: 'virtualHosted' 286 | , region: 'us-west-1' 287 | , port: 1234 288 | }); 289 | 290 | assert.equal(client.secure, false); 291 | assert.equal(client.style, 'virtualHosted'); 292 | assert.equal(client.region, 'us-west-1'); 293 | assert.equal(client.endpoint, 's3-us-west-1.amazonaws.com'); 294 | assert.equal(client.url('file'), 'http://misc.s3-us-west-1.amazonaws.com:1234/file'); 295 | }); 296 | 297 | it('should let secure set to true override custom port defaulting it to false', function () { 298 | var client = knox.createClient({ 299 | key: 'foobar' 300 | , secret: 'baz' 301 | , bucket: 'misc' 302 | , style: 'virtualHosted' 303 | , region: 'us-west-1' 304 | , port: 1234 305 | , secure: true 306 | }); 307 | 308 | assert.equal(client.secure, true); 309 | assert.equal(client.style, 'virtualHosted'); 310 | assert.equal(client.region, 'us-west-1'); 311 | assert.equal(client.endpoint, 's3-us-west-1.amazonaws.com'); 312 | assert.equal(client.url('file'), 'https://misc.s3-us-west-1.amazonaws.com:1234/file'); 313 | }); 314 | }); 315 | 316 | describe('with path style', function () { 317 | it('should use a default region and endpoint given a bucket', function () { 318 | var client = knox.createClient({ 319 | key: 'foobar' 320 | , secret: 'baz' 321 | , bucket: 'misc' 322 | , style: 'path' 323 | }); 324 | 325 | assert.equal(client.secure, true); 326 | assert.equal(client.style, 'path'); 327 | assert.equal(client.region, 'us-standard'); 328 | assert.equal(client.endpoint, 's3.amazonaws.com'); 329 | assert.equal(client.url('file'), 'https://s3.amazonaws.com/misc/file'); 330 | }); 331 | 332 | it('should use a custom endpoint directly', function () { 333 | var client = knox.createClient({ 334 | key: 'foobar' 335 | , secret: 'baz' 336 | , bucket: 'misc' 337 | , style: 'path' 338 | , endpoint: 'objects.dreamhost.com' 339 | }); 340 | 341 | assert.equal(client.secure, true); 342 | assert.equal(client.style, 'path'); 343 | assert.equal(client.region, undefined); 344 | assert.equal(client.domain, undefined); 345 | assert.equal(client.endpoint, 'objects.dreamhost.com'); 346 | assert.equal(client.url('file'), 'https://objects.dreamhost.com/misc/file'); 347 | }); 348 | 349 | it('should derive endpoint correctly from a region', function () { 350 | var client = knox.createClient({ 351 | key: 'foobar' 352 | , secret: 'baz' 353 | , bucket: 'misc' 354 | , style: 'path' 355 | , region: 'us-west-1' 356 | }); 357 | 358 | assert.equal(client.secure, true); 359 | assert.equal(client.style, 'path'); 360 | assert.equal(client.region, 'us-west-1'); 361 | assert.equal(client.endpoint, 's3-us-west-1.amazonaws.com'); 362 | assert.equal(client.url('file'), 'https://s3-us-west-1.amazonaws.com/misc/file'); 363 | }); 364 | 365 | it('should derive endpoint correctly from explicit us-standard region', function () { 366 | var client = knox.createClient({ 367 | key: 'foobar' 368 | , secret: 'baz' 369 | , bucket: 'misc' 370 | , style: 'path' 371 | , region: 'us-standard' 372 | }); 373 | 374 | assert.equal(client.secure, true); 375 | assert.equal(client.style, 'path'); 376 | assert.equal(client.region, 'us-standard'); 377 | assert.equal(client.endpoint, 's3.amazonaws.com'); 378 | assert.equal(client.url('file'), 'https://s3.amazonaws.com/misc/file'); 379 | }); 380 | 381 | it('should set secure to false and update the URL when given a port', function () { 382 | var client = knox.createClient({ 383 | key: 'foobar' 384 | , secret: 'baz' 385 | , bucket: 'misc' 386 | , style: 'path' 387 | , region: 'us-west-1' 388 | , port: 1234 389 | }); 390 | 391 | assert.equal(client.secure, false); 392 | assert.equal(client.style, 'path'); 393 | assert.equal(client.region, 'us-west-1'); 394 | assert.equal(client.endpoint, 's3-us-west-1.amazonaws.com'); 395 | assert.equal(client.url('file'), 'http://s3-us-west-1.amazonaws.com:1234/misc/file'); 396 | }); 397 | 398 | it('should let secure set to true override custom port defaulting it to false', function () { 399 | var client = knox.createClient({ 400 | key: 'foobar' 401 | , secret: 'baz' 402 | , bucket: 'misc' 403 | , style: 'path' 404 | , region: 'us-west-1' 405 | , port: 1234 406 | , secure: true 407 | }); 408 | 409 | assert.equal(client.secure, true); 410 | assert.equal(client.style, 'path'); 411 | assert.equal(client.region, 'us-west-1'); 412 | assert.equal(client.endpoint, 's3-us-west-1.amazonaws.com'); 413 | assert.equal(client.url('file'), 'https://s3-us-west-1.amazonaws.com:1234/misc/file'); 414 | }); 415 | }); 416 | 417 | describe('with automatic style determination', function () { 418 | it('should choose virtual hosted style by default', function () { 419 | var client = knox.createClient({ 420 | key: 'foobar' 421 | , secret: 'baz' 422 | , bucket: 'misc' 423 | }); 424 | 425 | assert.equal(client.secure, true); 426 | assert.equal(client.style, 'virtualHosted'); 427 | assert.equal(client.region, 'us-standard'); 428 | assert.equal(client.endpoint, 's3.amazonaws.com'); 429 | assert.equal(client.url('file'), 'https://misc.s3.amazonaws.com/file'); 430 | }); 431 | 432 | it('should choose path style if the bucket name contains a period', function () { 433 | var client = knox.createClient({ 434 | key: 'foobar' 435 | , secret: 'baz' 436 | , bucket: 'misc.bucket' 437 | }); 438 | 439 | assert.equal(client.secure, true); 440 | assert.equal(client.style, 'path'); 441 | assert.equal(client.region, 'us-standard'); 442 | assert.equal(client.endpoint, 's3.amazonaws.com'); 443 | assert.equal(client.url('file'), 'https://s3.amazonaws.com/misc.bucket/file'); 444 | }); 445 | 446 | it('should choose path style if the bucket name contains an upper-case character', function () { 447 | var client = knox.createClient({ 448 | key: 'foobar' 449 | , secret: 'baz' 450 | , bucket: 'MiscBucket' 451 | }); 452 | 453 | assert.equal(client.secure, true); 454 | assert.equal(client.style, 'path'); 455 | assert.equal(client.region, 'us-standard'); 456 | assert.equal(client.endpoint, 's3.amazonaws.com'); 457 | assert.equal(client.url('file'), 'https://s3.amazonaws.com/MiscBucket/file'); 458 | }); 459 | 460 | it('should choose path style if the bucket name contains a non-DNS-compliant character', function () { 461 | var client = knox.createClient({ 462 | key: 'foobar' 463 | , secret: 'baz' 464 | , bucket: 'misc_bucket' 465 | }); 466 | 467 | assert.equal(client.secure, true); 468 | assert.equal(client.style, 'path'); 469 | assert.equal(client.region, 'us-standard'); 470 | assert.equal(client.endpoint, 's3.amazonaws.com'); 471 | assert.equal(client.url('file'), 'https://s3.amazonaws.com/misc_bucket/file'); 472 | }); 473 | 474 | it('should choose path style if the bucket name starts with a dash', function () { 475 | var client = knox.createClient({ 476 | key: 'foobar' 477 | , secret: 'baz' 478 | , bucket: '-bucket' 479 | }); 480 | 481 | assert.equal(client.secure, true); 482 | assert.equal(client.style, 'path'); 483 | assert.equal(client.region, 'us-standard'); 484 | assert.equal(client.endpoint, 's3.amazonaws.com'); 485 | assert.equal(client.url('file'), 'https://s3.amazonaws.com/-bucket/file'); 486 | }); 487 | 488 | it('should choose virtual hosted style if the bucket name contains a period but secure is set to false', 489 | function () { 490 | var client = knox.createClient({ 491 | key: 'foobar' 492 | , secret: 'baz' 493 | , bucket: 'misc.bucket' 494 | , secure: false 495 | }); 496 | 497 | assert.equal(client.secure, false); 498 | assert.equal(client.style, 'virtualHosted'); 499 | assert.equal(client.region, 'us-standard'); 500 | assert.equal(client.endpoint, 's3.amazonaws.com'); 501 | assert.equal(client.url('file'), 'http://misc.bucket.s3.amazonaws.com/file'); 502 | }); 503 | }); 504 | }); 505 | }); 506 | -------------------------------------------------------------------------------- /test/fixtures/user.json: -------------------------------------------------------------------------------- 1 | {"name":"tj"} -------------------------------------------------------------------------------- /test/initClients.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var knox = require('..') 4 | , utils = knox.utils 5 | , assert = require('assert'); 6 | 7 | module.exports = function(style){ 8 | var client, client2, clientUsWest2; 9 | 10 | try { 11 | var auth = utils.merge({ style: style }, require('./auth.json')); 12 | 13 | assert(auth.bucket, 'bucket must exist'); 14 | assert(auth.bucket2, 'bucket2 must exist'); 15 | assert(auth.bucketUsWest2, 'bucketUsWest2 must exist'); 16 | assert.notEqual(auth.bucket, auth.bucket2, 'bucket should not equal bucket2.'); 17 | assert.notEqual(auth.bucket, auth.bucketUsWest2, 'bucket should not equal bucketUsWest2.'); 18 | assert.notEqual(auth.bucket2, auth.bucketUsWest2, 'bucket2 should not equal bucketUsWest2.'); 19 | 20 | var auth1 = utils.merge({}, auth); 21 | client = knox.createClient(auth1); 22 | 23 | var auth2 = utils.merge({}, auth); 24 | auth2.bucket = auth2.bucket2; 25 | client2 = knox.createClient(auth2); 26 | 27 | var authUsWest2 = utils.merge({}, auth); 28 | authUsWest2.bucket = auth.bucketUsWest2; 29 | // Without this we get a 307 redirect 30 | // that putFile can't handle (issue #66). Later 31 | // when there is an implementation of #66 we can test 32 | // both with and without this option present, but it's 33 | // always a good idea for performance 34 | authUsWest2.region = 'us-west-2'; 35 | clientUsWest2 = knox.createClient(authUsWest2); 36 | } catch (err) { 37 | console.error(err); 38 | console.error('The tests require test/auth.json to contain JSON with ' + 39 | 'key, secret, bucket, bucket2, and bucketUsWest2 in order ' + 40 | 'to run tests. All three buckets must exist and should not ' + 41 | 'contain anything you want to keep. bucketUsWest2 should ' + 42 | 'be created in the us-west-2 (Oregon) region, not the ' + 43 | 'default region.'); 44 | process.exit(1); 45 | } 46 | 47 | return { client: client, client2: client2, clientUsWest2: clientUsWest2 }; 48 | }; 49 | -------------------------------------------------------------------------------- /test/knox.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var knox = require('..') 4 | , initClients = require('./initClients') 5 | , fs = require('fs') 6 | , http = require('http') 7 | , assert = require('assert') 8 | , crypto = require('crypto'); 9 | 10 | var jsonFixture = __dirname + '/fixtures/user.json'; 11 | 12 | runTestsForStyle('virtualHosted', 'virtual hosted'); 13 | runTestsForStyle('path', 'path'); 14 | 15 | function runTestsForStyle(style, userFriendlyName) { 16 | describe('Client operations: ' + userFriendlyName + '-style', function () { 17 | var clients = initClients(style); 18 | var client = clients.client; 19 | var client2 = clients.client2; 20 | var clientUsWest2 = clients.clientUsWest2; 21 | 22 | describe('put()', function () { 23 | specify('from a file statted and read into a buffer', function (done) { 24 | fs.stat(jsonFixture, function (err, stat) { 25 | assert.ifError(err); 26 | fs.readFile(jsonFixture, function (err, buffer) { 27 | assert.ifError(err); 28 | var req = client.put('/test/user.json', { 29 | 'Content-Length': stat.size 30 | , 'Content-Type': 'application/json' 31 | }); 32 | 33 | assert.equal(req.url, client.url('/test/user.json')); 34 | 35 | req.on('response', function (res) { 36 | assert.equal(res.statusCode, 200); 37 | done(); 38 | }); 39 | 40 | req.end(buffer); 41 | }); 42 | }); 43 | }); 44 | 45 | specify('piping from a file stream', function (done) { 46 | fs.stat(jsonFixture, function (err, stat) { 47 | assert.ifError(err); 48 | var req = client.put('/test/direct-pipe.json', { 49 | 'Content-Length': stat.size 50 | , 'Content-Type': 'application/json' 51 | }); 52 | 53 | req.on('response', function (res) { 54 | assert.equal(res.statusCode, 200); 55 | done(); 56 | }); 57 | 58 | var fileStream = fs.createReadStream(jsonFixture); 59 | fileStream.pipe(req); 60 | }); 61 | }); 62 | 63 | specify('from a string written to the request', function (done) { 64 | var string = 'hello I am a string'; 65 | var req = client.put('/test/string.txt', { 66 | 'Content-Length': string.length 67 | , 'Content-Type': 'text/plain' 68 | }); 69 | 70 | req.on('response', function (res) { 71 | assert.equal(res.statusCode, 200); 72 | done(); 73 | }); 74 | 75 | req.end(string); 76 | }); 77 | 78 | specify('from a string written to the request, into a filename with an apostrophe', function (done) { 79 | var string = 'hello I have a \' in my name'; 80 | var req = client.put('/test/apos\'trophe.txt', { 81 | 'Content-Length': string.length 82 | , 'Content-Type': 'text/plain' 83 | }); 84 | 85 | req.on('response', function (res) { 86 | assert.equal(res.statusCode, 200); 87 | done(); 88 | }); 89 | 90 | req.end(string); 91 | }); 92 | 93 | specify('from a string written to the request, into a filename with an \'#\' sign', function (done) { 94 | var string = 'hello I have a version \'#\' in my extension'; 95 | var req = client.put('/test/versioned.txt#1', { 96 | 'Content-Length': string.length 97 | , 'Content-Type': 'text/plain' 98 | }); 99 | 100 | req.on('response', function (res) { 101 | assert.equal(res.statusCode, 200); 102 | done(); 103 | }); 104 | 105 | req.end(string); 106 | }); 107 | 108 | specify('should upload keys with strange unicode values', function (done) { 109 | var data = 'knox'; 110 | 111 | var req = client.put('/ø', { 112 | 'Content-Length': data.length 113 | , 'Content-Type': 'text/plain' 114 | }); 115 | 116 | req.on('response', function (res) { 117 | assert.equal(res.statusCode, 200); 118 | done(); 119 | }); 120 | 121 | req.end(data); 122 | }); 123 | 124 | it('should lower-case headers on requests', function () { 125 | var headers = { 'X-Amz-Acl': 'private' }; 126 | var req = client.put('/test/user.json', headers); 127 | 128 | assert.equal(req.getHeader('x-amz-acl'), 'private'); 129 | 130 | req.on('error', function (){}); // swallow "socket hang up" from aborting 131 | req.abort(); 132 | }); 133 | }); 134 | 135 | describe('putStream()', function () { 136 | specify('from a file stream', function (done) { 137 | fs.stat(jsonFixture, function (err, stat) { 138 | assert.ifError(err); 139 | 140 | var headers = { 141 | 'Content-Length': stat.size 142 | , 'Content-Type': 'application/json' 143 | }; 144 | 145 | var fileStream = fs.createReadStream(jsonFixture); 146 | client.putStream(fileStream, '/test/user.json', headers, function (err, res) { 147 | assert.ifError(err); 148 | assert.equal(res.statusCode, 200); 149 | done(); 150 | }); 151 | }); 152 | }); 153 | 154 | specify('from a HTTP stream', function (done) { 155 | http.get({ host: 'google.com', path: '/' }, function (res) { 156 | var headers = { 157 | 'Content-Length': res.headers['content-length'] 158 | , 'Content-Type': res.headers['content-type'] 159 | }; 160 | client.putStream(res, '/google', headers, function (err, res) { 161 | assert.ifError(err); 162 | assert.equal(res.statusCode, 200); 163 | done(); 164 | }); 165 | }); 166 | }); 167 | 168 | it('should emit "progress" events', function (done) { 169 | http.get({ host: 'google.com', path: '/' }, function (res) { 170 | var headers = { 171 | 'Content-Length': res.headers['content-length'] 172 | , 'Content-Type': res.headers['content-type'] 173 | }; 174 | 175 | var progressHappened = false; 176 | 177 | var req = client.putStream(res, '/google', headers, function (err, res) { 178 | assert.ifError(err); 179 | assert.equal(res.statusCode, 200); 180 | assert(progressHappened); 181 | done(); 182 | }); 183 | 184 | req.on('progress', function (event) { 185 | progressHappened = true; 186 | assert(event.percent); 187 | assert(event.total); 188 | assert(event.written); 189 | }); 190 | }); 191 | }); 192 | 193 | it('should error early if there is no "Content-Length" header', function (done) { 194 | var stream = fs.createReadStream(jsonFixture); 195 | var headers = { 'Content-Type': 'application/json' }; 196 | client.putStream(stream, '/test/user.json', headers, function (err, res) { 197 | assert(err); 198 | assert(/Content-Length/.test(err.message)); 199 | done(); 200 | }); 201 | }); 202 | 203 | it('should work with a lower-case "content-length" header', function (done) { 204 | fs.stat(jsonFixture, function (err, stat) { 205 | assert.ifError(err); 206 | 207 | var headers = { 208 | 'content-length': stat.size 209 | , 'Content-Type': 'application/json' 210 | }; 211 | 212 | var fileStream = fs.createReadStream(jsonFixture); 213 | client.putStream(fileStream, '/test/user.json', headers, function (err, res) { 214 | assert.ifError(err); 215 | assert.equal(res.statusCode, 200); 216 | done(); 217 | }); 218 | }); 219 | }); 220 | }); 221 | 222 | describe('putFile()', function () { 223 | specify('the basic case', function (done) { 224 | client.putFile(jsonFixture, '/test/user2.json', function (err, res) { 225 | assert.ifError(err); 226 | assert.equal(res.statusCode, 200); 227 | 228 | client.get('/test/user2.json').on('response', function (res) { 229 | assert.equal(res.headers['content-type'], 'application/json'); 230 | done(); 231 | }).end(); 232 | }); 233 | }); 234 | 235 | it('should work the same in us-west-2', function (done) { 236 | clientUsWest2.putFile(jsonFixture, '/test/user2.json', function (err, res) { 237 | assert.ifError(err); 238 | assert.equal(res.statusCode, 200); 239 | 240 | clientUsWest2.get('/test/user2.json').on('response', function (res) { 241 | assert.equal(res.headers['content-type'], 'application/json'); 242 | done(); 243 | }).end(); 244 | }); 245 | }); 246 | 247 | it('should emit "progress" events', function (done) { 248 | var progressHappened = false; 249 | 250 | var file = client.putFile(jsonFixture, '/test/user2.json', function (err, res) { 251 | assert.ifError(err); 252 | assert.equal(res.statusCode, 200); 253 | 254 | clientUsWest2.get('/test/user2.json').on('response', function (res) { 255 | assert.equal(res.headers['content-type'], 'application/json'); 256 | assert(progressHappened); 257 | done(); 258 | }).end(); 259 | }); 260 | 261 | file.on('progress', function (event) { 262 | progressHappened = true; 263 | assert(event.percent); 264 | assert(event.total); 265 | assert(event.written); 266 | }); 267 | }); 268 | }); 269 | 270 | describe('putBuffer()', function () { 271 | specify('the basic case', function (done) { 272 | var buffer = new Buffer('a string of stuff'); 273 | var headers = { 'Content-Type': 'text/plain' }; 274 | 275 | client.putBuffer(buffer, '/buffer.txt', headers, function (err, res) { 276 | assert.ifError(err); 277 | assert.equal(res.statusCode, 200); 278 | done(); 279 | }); 280 | }); 281 | 282 | specify('with a lower-case "content-type" header', function (done) { 283 | var buffer = new Buffer('a string of stuff'); 284 | var headers = { 'content-type': 'text/plain' }; 285 | 286 | client.putBuffer(buffer, '/buffer2.txt', headers, function (err, res) { 287 | assert.ifError(err); 288 | assert.equal(res.statusCode, 200); 289 | 290 | client.getFile('/buffer2.txt', function (err, res) { 291 | assert.ifError(err); 292 | assert.equal(res.statusCode, 200); 293 | assert.equal(res.headers['content-type'], 'text/plain'); 294 | done(); 295 | }); 296 | }); 297 | }); 298 | 299 | specify('with spaces in the file name', function (done) { 300 | var buffer = new Buffer('a string of stuff'); 301 | var headers = { 'Content-Type': 'text/plain' }; 302 | 303 | client.putBuffer(buffer, '/buffer with spaces.txt', headers, function (err, res) { 304 | assert.ifError(err); 305 | assert.equal(res.statusCode, 200); 306 | done(); 307 | }); 308 | }); 309 | 310 | specify('with pluses in the file name', function (done) { 311 | var buffer = new Buffer('a string of stuff'); 312 | var headers = { 'Content-Type': 'text/plain' }; 313 | 314 | client.putBuffer(buffer, '/buffer+with+pluses.txt', headers, function (err, res) { 315 | assert.ifError(err); 316 | assert.equal(res.statusCode, 200); 317 | done(); 318 | }); 319 | }); 320 | 321 | specify('with ? in the file name', function (done) { 322 | var buffer = new Buffer('a string of stuff'); 323 | var headers = { 'Content-Type': 'text/plain' }; 324 | 325 | client.putBuffer(buffer, '/buffer?with?questions.txt', headers, function (err, res) { 326 | assert.ifError(err); 327 | assert.equal(res.statusCode, 200); 328 | done(); 329 | }); 330 | }); 331 | }); 332 | 333 | describe('copy()', function () { 334 | it('should return with 200 OK', function (done) { 335 | client.copy('/test/user.json', '/test/user3.json').on('response', function (res) { 336 | assert.equal(res.statusCode, 200); 337 | done(); 338 | }).end(); 339 | }); 340 | }); 341 | 342 | describe('copy() with unicode characters', function() { 343 | it('should return with 200 OK', function(done) { 344 | client.copy('/ø', '/ø/ø').on('response', function(res) { 345 | assert.equal(res.statusCode, 200); 346 | done(); 347 | }).end(); 348 | }); 349 | }); 350 | 351 | describe('copyTo()', function () { 352 | it('should return with 200 OK', function (done) { 353 | client.copyTo('/test/user.json', client2.bucket, '/test/user3.json').on('response', function (res) { 354 | assert.equal(res.statusCode, 200); 355 | done(); 356 | }).end(); 357 | }); 358 | }); 359 | 360 | describe('copyFile()', function () { 361 | it('should return with 200 OK', function (done) { 362 | client.copyFile('/test/user.json', '/test/user4.json', function (err, res) { 363 | assert.ifError(err); 364 | assert.equal(res.statusCode, 200); 365 | done(); 366 | }).end(); 367 | }); 368 | }); 369 | 370 | describe('copyFileTo()', function () { 371 | it('should return with 200 OK', function (done) { 372 | client.copyFileTo('/test/user4.json', client2.bucket, '/test/user4.json', function (err, res) { 373 | assert.ifError(err); 374 | assert.equal(res.statusCode, 200); 375 | done(); 376 | }).end(); 377 | }); 378 | }); 379 | 380 | describe('get()', function () { 381 | specify('the basic case', function (done) { 382 | client.get('/test/user.json').on('response', function (res) { 383 | assert.equal(res.statusCode, 200); 384 | assert.equal(res.headers['content-type'], 'application/json'); 385 | assert.equal(res.headers['content-length'], 13); 386 | done(); 387 | }).end(); 388 | }); 389 | 390 | it('should work without a leading slash', function (done) { 391 | client.get('test/user.json').on('response', function (res) { 392 | assert.equal(res.statusCode, 200); 393 | assert.equal(res.headers['content-type'], 'application/json'); 394 | assert.equal(res.headers['content-length'], 13); 395 | done(); 396 | }).end(); 397 | }); 398 | 399 | it('should give a 404 for the file not found', function (done) { 400 | client.get('/test/whatever').on('response', function (res) { 401 | assert.equal(res.statusCode, 404); 402 | done(); 403 | }).end(); 404 | }); 405 | 406 | it('should set tokens passed to client construction as the x-amz-security-token header', function () { 407 | var client = knox.createClient({ 408 | key: 'foobar' 409 | , secret: 'baz' 410 | , bucket: 'misc' 411 | , token: 'foo' 412 | }); 413 | 414 | var req = client.get('/'); 415 | assert.equal(req.getHeader('x-amz-security-token'), 'foo'); 416 | 417 | req.on('error', function (){}); // swallow "socket hang up" from aborting 418 | req.abort(); 419 | }); 420 | 421 | it('should not set a token header if the token option is undefined', function () { 422 | var client = knox.createClient({ 423 | key: 'foobar' 424 | , secret: 'baz' 425 | , bucket: 'misc' 426 | , token: undefined 427 | }); 428 | 429 | var req = client.get('/'); 430 | assert(!req.getHeader('x-amz-security-token')); 431 | 432 | req.on('error', function (){}); // swallow "socket hang up" from aborting 433 | req.abort(); 434 | }); 435 | }); 436 | 437 | describe('getFile()', function () { 438 | specify('the basic case', function (done) { 439 | client.getFile('/test/user.json', function(err, res){ 440 | assert.ifError(err); 441 | assert.equal(res.statusCode, 200); 442 | assert.equal(res.headers['content-type'], 'application/json'); 443 | assert.equal(res.headers['content-length'], 13); 444 | done(); 445 | }); 446 | }); 447 | }); 448 | 449 | describe('head()', function () { 450 | specify('the basic case', function (done) { 451 | client.head('/test/user.json').on('response', function (res) { 452 | assert.equal(res.statusCode, 200); 453 | assert.equal(res.headers['content-type'], 'application/json'); 454 | assert.equal(res.headers['content-length'], 13); 455 | done(); 456 | }).end(); 457 | }); 458 | 459 | it('should work without a leading slash', function (done) { 460 | client.head('test/user.json').on('response', function (res) { 461 | assert.equal(res.statusCode, 200); 462 | assert.equal(res.headers['content-type'], 'application/json'); 463 | assert.equal(res.headers['content-length'], 13); 464 | done(); 465 | }).end(); 466 | }); 467 | 468 | it('should give a 404 for the file not found', function (done) { 469 | client.head('/test/whatever').on('response', function (res) { 470 | assert.equal(res.statusCode, 404); 471 | done(); 472 | }).end(); 473 | }); 474 | }); 475 | 476 | describe('headFile()', function () { 477 | specify('the basic case', function (done) { 478 | client.headFile('/test/user.json', function(err, res){ 479 | assert.ifError(err); 480 | assert.equal(res.statusCode, 200); 481 | assert.equal(res.headers['content-type'], 'application/json'); 482 | assert.equal(res.headers['content-length'], 13); 483 | done(); 484 | }); 485 | }); 486 | }); 487 | 488 | describe('list()', function () { 489 | it('should list files with a specified prefix', function (done) { 490 | var files = ['/list/user1.json', '/list/user2.json']; 491 | 492 | client.putFile(jsonFixture, files[0], function (err, res) { 493 | assert.ifError(err); 494 | client.putFile(jsonFixture, files[1], function (err, res) { 495 | assert.ifError(err); 496 | client.list({ prefix: 'list' }, function (err, data) { 497 | assert.ifError(err); 498 | 499 | assert.strictEqual(data.Prefix, 'list'); 500 | assert.strictEqual(data.IsTruncated, false); 501 | assert.strictEqual(data.MaxKeys, 1000); 502 | assert.strictEqual(data.Contents.length, 2); 503 | assert(data.Contents[0].LastModified instanceof Date); 504 | assert.strictEqual(typeof data.Contents[0].Size, 'number'); 505 | assert.deepEqual( 506 | Object.keys(data.Contents[0]), 507 | ['Key', 'LastModified', 'ETag', 'Size', 'Owner', 'StorageClass'] 508 | ); 509 | 510 | done(); 511 | }); 512 | }); 513 | }); 514 | }); 515 | 516 | it('should list files with a specified prefix with slash', function (done) { 517 | var files = ['/list/slash-1.json', '/list/slash-2.json']; 518 | 519 | client.putFile(jsonFixture, files[0], function (err, res) { 520 | assert.ifError(err); 521 | client.putFile(jsonFixture, files[1], function (err, res) { 522 | assert.ifError(err); 523 | client.list({ prefix: 'list/slash-' }, function (err, data) { 524 | assert.ifError(err); 525 | 526 | assert.strictEqual(data.Prefix, 'list/slash-'); 527 | assert.strictEqual(data.IsTruncated, false); 528 | assert.strictEqual(data.MaxKeys, 1000); 529 | assert.strictEqual(data.Contents.length, 2); 530 | assert(data.Contents[0].LastModified instanceof Date); 531 | assert.strictEqual(typeof data.Contents[0].Size, 'number'); 532 | assert.deepEqual( 533 | Object.keys(data.Contents[0]), 534 | ['Key', 'LastModified', 'ETag', 'Size', 'Owner', 'StorageClass'] 535 | ); 536 | 537 | done(); 538 | }); 539 | }); 540 | }); 541 | }); 542 | 543 | }); 544 | 545 | describe('request()', function () { 546 | it('should work to get an object\'s ACL via ?acl', function (done) { 547 | var req = client.request('GET', '/test/user3.json?acl') 548 | .on('error', done) 549 | .on('response', function (res) { 550 | var data = ''; 551 | res.on('data', function (chunk) { 552 | data += chunk; 553 | }).on('end', function () { 554 | assert(data.indexOf('FULL_CONTROL') !== -1); 555 | done(); 556 | }).on('error', done); 557 | }).end(); 558 | }); 559 | 560 | it('should work to delete files via ?delete', function (done) { 561 | var xml = '' + 562 | 'test/user4.json' + 563 | 'test/direct-pipe.json' + 564 | 'list/user1.json' + 565 | 'list/user2.json' + 566 | 'list/slash-1.json' + 567 | 'list/slash-2.json' + 568 | ''; 569 | 570 | var req = client.request('POST', '/?delete', { 571 | 'Content-Length': xml.length, 572 | 'Content-MD5': crypto.createHash('md5').update(xml).digest('base64'), 573 | 'Accept:': '*\/*' 574 | }) 575 | .on('error',done) 576 | .on('response', function (res) { 577 | assert.equal(res.statusCode, 200); 578 | done(); 579 | }) 580 | .end(xml); 581 | }); 582 | }); 583 | 584 | describe('del()', function () { 585 | it('should return with 204 No Content', function (done) { 586 | client.del('/test/user.json').on('response', function (res) { 587 | assert.equal(res.statusCode, 204); 588 | done(); 589 | }).end(); 590 | }); 591 | }); 592 | 593 | describe('deleteFile()', function () { 594 | it('should return with 204 No Content', function (done) { 595 | client.deleteFile('/test/user2.json', function (err, res) { 596 | assert.ifError(err); 597 | assert.equal(res.statusCode, 204); 598 | done(); 599 | }); 600 | }); 601 | }); 602 | 603 | describe('deleteMultiple()', function () { 604 | it('should remove the files as seen in list()', function (done) { 605 | // Intentionally mix no leading slashes or leading slashes: see #121. 606 | var files = ['/test/user3.json', 'test/string.txt', '/test/apos\'trophe.txt', '/buffer.txt', '/buffer2.txt', 607 | 'google', 'buffer with spaces.txt', 'buffer+with+pluses.txt', 'buffer?with?questions.txt', 608 | '/ø', 'ø/ø', '/test/versioned.txt#1']; 609 | client.deleteMultiple(files, function (err, res) { 610 | assert.ifError(err); 611 | assert.equal(res.statusCode, 200); 612 | 613 | client.list(function (err, data) { 614 | assert.ifError(err); 615 | var keys = data.Contents.map(function (entry) { return entry.Key; }); 616 | 617 | assert(keys.indexOf('test/user3.json') === -1); 618 | assert(keys.indexOf('test/string.txt') === -1); 619 | assert(keys.indexOf('test/apos\'trophe.txt') === -1); 620 | assert(keys.indexOf('buffer.txt') === -1); 621 | assert(keys.indexOf('buffer2.txt') === -1); 622 | assert(keys.indexOf('google') === -1); 623 | assert(keys.indexOf('buffer with spaces.txt') === -1); 624 | assert(keys.indexOf('buffer+with+pluses.txt') === -1); 625 | assert(keys.indexOf('buffer?with?questions.txt') === -1); 626 | assert(keys.indexOf('ø') === -1); 627 | assert(keys.indexOf('ø/ø') === -1); 628 | assert(keys.indexOf('/test/versioned.txt#1') === -1); 629 | 630 | done(); 631 | }); 632 | }); 633 | }); 634 | 635 | it('should work in bucket2', function (done) { 636 | client2.deleteMultiple(['test/user3.json', 'test/user4.json'], function (err, res) { 637 | assert.ifError(err); 638 | assert.equal(res.statusCode, 200); 639 | done(); 640 | }); 641 | }); 642 | 643 | it('should work in bucketUsWest2', function (done) { 644 | clientUsWest2.deleteMultiple(['test/user2.json'], function (err, res) { 645 | assert.ifError(err); 646 | assert.equal(res.statusCode, 200); 647 | done(); 648 | }); 649 | }); 650 | }); 651 | 652 | describe('we should clean up and not use people\'s S3 $$$', function () { 653 | specify('in bucket', function (done) { 654 | client.list(function (err, data) { 655 | assert.ifError(err); 656 | 657 | // Do the assertion like this for nicer error reporting. 658 | var keys = data.Contents.map(function (entry) { return entry.Key; }); 659 | assert.deepEqual(keys, []); 660 | 661 | done(); 662 | }); 663 | }); 664 | 665 | specify('in bucket2', function (done) { 666 | client2.list(function (err, data) { 667 | assert.ifError(err); 668 | 669 | // Do the assertion like this for nicer error reporting. 670 | var keys = data.Contents.map(function (entry) { return entry.Key; }); 671 | assert.deepEqual(keys, []); 672 | 673 | done(); 674 | }); 675 | }); 676 | 677 | specify('in bucketUsWest2', function (done) { 678 | clientUsWest2.list(function (err, data) { 679 | assert.ifError(err); 680 | 681 | // Do the assertion like this for nicer error reporting. 682 | var keys = data.Contents.map(function (entry) { return entry.Key; }); 683 | assert.deepEqual(keys, []); 684 | 685 | done(); 686 | }); 687 | }); 688 | }); 689 | }); 690 | } 691 | -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --ui bdd 2 | --slow 1000ms 3 | --timeout 5000ms 4 | --reporter spec 5 | -------------------------------------------------------------------------------- /test/signedUrl.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var knox = require('..') 4 | , initClients = require('./initClients') 5 | , signQuery = require('../lib/auth').signQuery 6 | , https = require('https') 7 | , assert = require('assert') 8 | , parseUrl = require('url').parse 9 | , qs = require('querystring'); 10 | 11 | var jsonString = JSON.stringify({ name: 'Domenic '}); 12 | 13 | runTestsForStyle('virtualHosted', 'virtual hosted'); 14 | runTestsForStyle('path', 'path'); 15 | 16 | function runTestsForStyle(style, userFriendlyName) { 17 | describe('client.signedUrl(): ' + userFriendlyName + '-style', function () { 18 | var client = initClients(style).client; 19 | var tokenClient = knox.createClient({ 20 | bucket: 'example' 21 | , key: 'foo' 22 | , secret: 'bar' 23 | , token: 'baz' 24 | , style: style 25 | }); 26 | 27 | describe('using the signed URL to perform HTTP requests', function () { 28 | // These tests seem to fail if we don't slow down a bit, probably due to 29 | // Amazon throttling us. So insert a delay. 30 | beforeEach(function (done) { 31 | setTimeout(done, 2000); 32 | }); 33 | 34 | specify('PUT', function (done) { 35 | var signedUrl = client.signedUrl( 36 | '/test/user.json' 37 | , new Date(Date.now() + 50000) 38 | , { verb: 'PUT', contentType: 'application/json' } 39 | ); 40 | 41 | var options = parseUrl(signedUrl); 42 | options.method = 'PUT'; 43 | options.headers = { 44 | 'Content-Length': jsonString.length, 45 | 'Content-Type': 'application/json' 46 | }; 47 | 48 | https.request(options).on('response', function (res) { 49 | assert.equal(res.statusCode, 200); 50 | done(); 51 | }) 52 | .on('error', assert.ifError) 53 | .end(jsonString); 54 | }); 55 | 56 | specify('PUT (with spaces in the file name)', function (done) { 57 | var signedUrl = client.signedUrl( 58 | 'user with spaces.json' 59 | , new Date(Date.now() + 50000) 60 | , { verb: 'PUT', contentType: 'application/json' } 61 | ); 62 | 63 | var options = parseUrl(signedUrl); 64 | options.method = 'PUT'; 65 | options.headers = { 66 | 'Content-Length': jsonString.length, 67 | 'Content-Type': 'application/json' 68 | }; 69 | 70 | https.request(options).on('response', function (res) { 71 | assert.equal(res.statusCode, 200); 72 | done(); 73 | }) 74 | .on('error', assert.ifError) 75 | .end(jsonString); 76 | }); 77 | 78 | specify('PUT (with x-amz-acl)', function (done) { 79 | var signedUrl = client.signedUrl( 80 | 'acl.json' 81 | , new Date(Date.now() + 50000) 82 | , { 83 | verb: 'PUT' 84 | , contentType: 'application/json' 85 | , 'x-amz-acl': 'public-read' 86 | } 87 | ); 88 | 89 | var options = parseUrl(signedUrl); 90 | options.method = 'PUT'; 91 | options.headers = { 92 | 'Content-Length': jsonString.length, 93 | 'Content-Type': 'application/json' 94 | }; 95 | 96 | https.request(options).on('response', function (res) { 97 | assert.equal(res.statusCode, 200); 98 | done(); 99 | }) 100 | .on('error', assert.ifError) 101 | .end(jsonString); 102 | }); 103 | 104 | specify('PUT (with x-amz-meta-myField)', function (done) { 105 | var signedUrl = client.signedUrl( 106 | 'acl.json' 107 | , new Date(Date.now() + 50000) 108 | , { 109 | verb: 'PUT' 110 | , contentType: 'application/json' 111 | , extraHeaders: {'x-amz-meta-myField': 'mySignedFieldValue'} 112 | } 113 | ); 114 | 115 | var options = parseUrl(signedUrl); 116 | options.method = 'PUT'; 117 | options.headers = { 118 | 'Content-Length': jsonString.length, 119 | 'Content-Type': 'application/json', 120 | 'x-amz-meta-myField': 'mySignedFieldValue' 121 | }; 122 | 123 | https.request(options).on('response', function (res) { 124 | assert.equal(res.statusCode, 200); 125 | done(); 126 | }) 127 | .on('error', assert.ifError) 128 | .end(jsonString); 129 | }); 130 | 131 | 132 | specify('GET (with leading slash)', function (done) { 133 | var signedUrl = client.signedUrl( 134 | '/test/user.json' 135 | , new Date(Date.now() + 50000) 136 | ); 137 | 138 | https.get(signedUrl).on('response', function (res) { 139 | assert.equal(res.statusCode, 200); 140 | assert.equal(res.headers['content-type'], 'application/json'); 141 | assert.equal(res.headers['content-length'], jsonString.length); 142 | done(); 143 | }) 144 | .on('error', assert.ifError) 145 | .end(); 146 | }); 147 | 148 | specify('GET (without leading slash)', function (done) { 149 | var signedUrl = client.signedUrl( 150 | 'test/user.json' 151 | , new Date(Date.now() + 50000) 152 | ); 153 | 154 | https.get(signedUrl).on('response', function (res) { 155 | assert.equal(res.statusCode, 200); 156 | assert.equal(res.headers['content-type'], 'application/json'); 157 | assert.equal(res.headers['content-length'], jsonString.length); 158 | done(); 159 | }) 160 | .on('error', assert.ifError) 161 | .end(); 162 | }); 163 | 164 | specify('GET (with explicit verb option)', function (done) { 165 | var signedUrl = client.signedUrl( 166 | '/test/user.json' 167 | , new Date(Date.now() + 50000) 168 | , { verb: 'GET' } 169 | ); 170 | 171 | https.get(signedUrl).on('response', function (res) { 172 | assert.equal(res.statusCode, 200); 173 | assert.equal(res.headers['content-type'], 'application/json'); 174 | assert.equal(res.headers['content-length'], jsonString.length); 175 | done(); 176 | }) 177 | .on('error', assert.ifError) 178 | .end(); 179 | }); 180 | 181 | specify('GET (with Unicode in query string)', function (done) { 182 | var contentDisposition = 'attachment; filename="ümläüt.txt";'; 183 | var signedUrl = client.signedUrl( 184 | '/test/user.json' 185 | , new Date(Date.now() + 50000) 186 | , { qs: { 'response-content-disposition': contentDisposition } } 187 | ); 188 | 189 | https.get(signedUrl).on('response', function (res) { 190 | assert.equal(res.statusCode, 200); 191 | assert.equal(res.headers['content-type'], 'application/json'); 192 | assert.equal(res.headers['content-length'], jsonString.length); 193 | 194 | // TODO: why aren't these equal? Amazon's fault, or ours? 195 | // assert.equal(res.headers['content-disposition'], contentDisposition); 196 | 197 | done(); 198 | }) 199 | .on('error', assert.ifError) 200 | .end(); 201 | }); 202 | 203 | specify('GET (with spaces in the name)', function (done) { 204 | var signedUrl = client.signedUrl( 205 | 'user with spaces.json' 206 | , new Date(Date.now() + 50000) 207 | ); 208 | 209 | https.get(signedUrl).on('response', function (res) { 210 | assert.equal(res.statusCode, 200); 211 | assert.equal(res.headers['content-type'], 'application/json'); 212 | assert.equal(res.headers['content-length'], jsonString.length); 213 | done(); 214 | }) 215 | .on('error', assert.ifError) 216 | .end(); 217 | }); 218 | 219 | specify('HEAD', function (done) { 220 | var signedUrl = client.signedUrl( 221 | '/test/user.json' 222 | , new Date(Date.now() + 50000) 223 | , { verb: 'HEAD' } 224 | ); 225 | 226 | var options = parseUrl(signedUrl); 227 | options.method = 'HEAD'; 228 | https.request(options).on('response', function (res) { 229 | assert.equal(res.statusCode, 200); 230 | assert.equal(res.headers['content-type'], 'application/json'); 231 | assert.equal(res.headers['content-length'], jsonString.length); 232 | done(); 233 | }) 234 | .on('error', assert.ifError) 235 | .end(); 236 | }); 237 | 238 | specify('DELETE', function (done) { 239 | var signedUrl = client.signedUrl( 240 | '/test/user.json' 241 | , new Date(Date.now() + 50000) 242 | , { verb: 'DELETE' } 243 | ); 244 | 245 | var options = parseUrl(signedUrl); 246 | options.method = 'DELETE'; 247 | https.request(options).on('response', function (res) { 248 | assert.equal(res.statusCode, 204); 249 | done(); 250 | }) 251 | .on('error', assert.ifError) 252 | .end(); 253 | }); 254 | 255 | specify('DELETE (with spaces in the file name)', function (done) { 256 | var signedUrl = client.signedUrl( 257 | 'user with spaces.json' 258 | , new Date(Date.now() + 50000) 259 | , { verb: 'DELETE' } 260 | ); 261 | 262 | var options = parseUrl(signedUrl); 263 | options.method = 'DELETE'; 264 | https.request(options).on('response', function (res) { 265 | assert.equal(res.statusCode, 204); 266 | done(); 267 | }) 268 | .on('error', assert.ifError) 269 | .end(); 270 | }); 271 | 272 | specify('DELETE (clean up the x-amz-acl test)', function (done) { 273 | var signedUrl = client.signedUrl( 274 | 'acl.json' 275 | , new Date(Date.now() + 50000) 276 | , { verb: 'DELETE' } 277 | ); 278 | 279 | var options = parseUrl(signedUrl); 280 | options.method = 'DELETE'; 281 | https.request(options).on('response', function (res) { 282 | assert.equal(res.statusCode, 204); 283 | done(); 284 | }) 285 | .on('error', assert.ifError) 286 | .end(); 287 | }); 288 | }); 289 | 290 | describe('checking the signed URL against a known results', function () { 291 | specify('with extra parameters in the querystring', function () { 292 | var date = new Date(2020, 1, 1); 293 | var timestamp = date.getTime() * 0.001; 294 | var otherParams = { 295 | filename: 'my?Fi&le.json', 296 | 'response-content-disposition': 'attachment' 297 | }; 298 | var signedUrl = client.signedUrl( 299 | '/test/user.json', 300 | date, 301 | { qs: otherParams } 302 | ); 303 | 304 | var signature = signQuery({ 305 | secret: client.secret 306 | , date: timestamp 307 | , resource: '/' + client.bucket + '/test/user.json?' + 308 | decodeURIComponent(qs.stringify(otherParams)) 309 | }); 310 | 311 | assert.equal(signedUrl, 312 | client.url('/test/user.json') + 313 | '?Expires=' + timestamp + 314 | '&AWSAccessKeyId=' + client.key + 315 | '&Signature=' + encodeURIComponent(signature) + 316 | '&filename=' + encodeURIComponent('my?Fi&le.json') + 317 | '&response-content-disposition=attachment'); 318 | }); 319 | 320 | specify('with a STS token', function () { 321 | var date = new Date(2020, 1, 1); 322 | var timestamp = date.getTime() * 0.001; 323 | var token = tokenClient.token; 324 | var signedUrl = tokenClient.signedUrl('/test/user.json', date); 325 | 326 | var signature = signQuery({ 327 | secret: tokenClient.secret 328 | , date: timestamp 329 | , resource: '/' + tokenClient.bucket + '/test/user.json' 330 | , token: token 331 | }); 332 | 333 | assert.equal(signedUrl, 334 | tokenClient.url('/test/user.json') + 335 | '?Expires=' + timestamp + 336 | '&AWSAccessKeyId=' + tokenClient.key + 337 | '&Signature=' + encodeURIComponent(signature) + 338 | '&x-amz-security-token=' + encodeURIComponent(token)); 339 | }); 340 | }); 341 | }); 342 | } 343 | -------------------------------------------------------------------------------- /test/utils.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var knox = require('..') 4 | , utils = knox.utils 5 | , assert = require('assert'); 6 | 7 | describe('knox.utils', function () { 8 | specify('.base64.encode()', function () { 9 | assert.equal('aGV5', utils.base64.encode('hey')); 10 | }); 11 | 12 | specify('.base64.decode()', function () { 13 | assert.equal('hey', utils.base64.decode('aGV5')); 14 | }); 15 | }); 16 | --------------------------------------------------------------------------------