├── package.json ├── .gitignore ├── README.md ├── yarn.lock └── index.js /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aws-s3-zipper", 3 | "version": "1.4.0", 4 | "description": "Zip files from and to an Amazon AWS S3 Bucket directory as a stream, file or fragments. Allows filtering files.", 5 | "main": "index.js", 6 | "dependencies": { 7 | "archiver": "0.21.*", 8 | "async": "1.5.*", 9 | "aws-sdk": "2.814.*", 10 | "@auth0/s3": "latest" 11 | }, 12 | "repository": "https://github.com/DanielHindi/aws-s3-zipper.git", 13 | "keywords": [ 14 | "zip", 15 | "amazon", 16 | "aws", 17 | "s3" 18 | ], 19 | "author": "Daniel Hindi", 20 | "license": "MIT", 21 | "readmeFilename": "README.md" 22 | } 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 18 | .grunt 19 | 20 | # node-waf configuration 21 | .lock-wscript 22 | 23 | # Compiled binary addons (http://nodejs.org/api/addons.html) 24 | build/Release 25 | 26 | # Dependency directory 27 | node_modules 28 | 29 | # Optional npm cache directory 30 | .npm 31 | 32 | # Optional REPL history 33 | .node_repl_history 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Amazon S3 Zipping tool (aws-s3-zipper) 2 | 3 | ## What does it do? 4 | ### 1. Zips S3 files 5 | Takes an amazon s3 bucket folder and zips it to a: 6 | * Stream 7 | * Local File 8 | * Local File Fragments (zip multiple files broken up by max number of files or size) 9 | * S3 File (ie uploads the zip back to s3) 10 | * S3 File Fragments (upload multiple zip files broken up by max number of files or size) 11 | 12 | ### 2. Differential zipping 13 | It also allows you to do *differential* zips. You can save the key of the last file you zipped and then zip files that have been uploaded after the last zip. 14 | 15 | ### 3. Fragmented Zips 16 | If a zip file has the potential of getting too big. You can provide limits to breakup the compression into multiple zip files. You can limit based on file count or total size (pre zip) 17 | 18 | ### 4. Filter Files to zip 19 | You can filter out files you dont want zipped based on any criteria you need 20 | 21 | 22 | 23 | ## How do i use it? 24 | ### Setup 25 | ``` 26 | var S3Zipper = require ('aws-s3-zipper'); 27 | 28 | var config ={ 29 | accessKeyId: "XXXX", 30 | secretAccessKey: "XXXX", 31 | region: "us-west-2", 32 | bucket: 'XXX' 33 | }; 34 | var zipper = new S3Zipper(config); 35 | ``` 36 | 37 | 38 | ### Filter out Files 39 | ``` 40 | zipper.filterOutFiles= function(file){ 41 | if(file.Key.indexOf('.tmp') >= 0) // filter out temp files 42 | return null; 43 | else 44 | return file; 45 | }; 46 | ``` 47 | 48 | ### Zip to local file 49 | ``` 50 | zipper.zipToFile ({ 51 | s3FolderName: 'myBucketFolderName' 52 | , startKey: 'keyOfLastFileIZipped' // could keep null 53 | , zipFileName: './myLocalFile.zip' 54 | , recursive: true 55 | } 56 | ,function(err,result){ 57 | if(err) 58 | console.error(err); 59 | else{ 60 | var lastFile = result.zippedFiles[result.zippedFiles.length-1]; 61 | if(lastFile) 62 | console.log('last key ', lastFile.Key); // next time start from here 63 | } 64 | }); 65 | ``` 66 | 67 | ### Pipe zip data to stream (using Express.js) 68 | ``` 69 | app.all('/', function (request, response) { 70 | response.set('content-type', 'application/zip') // optional 71 | zipper.streamZipDataTo({ 72 | pipe: response 73 | , folderName: 'myBucketFolderName' 74 | , startKey: 'keyOfLastFileIZipped' // could keep null 75 | , recursive: true 76 | } 77 | ,function (err, result) { 78 | if(err) 79 | console.error(err); 80 | else{ 81 | console.log(result) 82 | } 83 | }) 84 | }) 85 | ``` 86 | 87 | ### Zip fragments to local file system with the filename pattern with a maximum file count 88 | ``` 89 | zipper.zipToFileFragments ({ 90 | s3FolderName:'myBucketFolderName' 91 | ,startKey: null 92 | ,zipFileName './myLocalFile.zip' 93 | ,maxFileCount: 5 94 | ,maxFileSize: 1024*1024 95 | }, function(err,results){ 96 | if(err) 97 | console.error(err); 98 | else{ 99 | if(results.length > 0) { 100 | var result = results[results.length - 1]; 101 | var lastFile = result.zippedFiles[result.zippedFiles.length - 1]; 102 | if (lastFile) 103 | console.log('last key ', lastFile.Key); // next time start from here 104 | } 105 | } 106 | }); 107 | ``` 108 | 109 | 110 | ### Zip to S3 file 111 | ``` 112 | /// if no path is given to S3 zip file then it will be placed in the same folder 113 | zipper.zipToS3File ({ 114 | s3FolderName: 'myBucketFolderName' 115 | , startKey: 'keyOfLastFileIZipped' // optional 116 | , s3ZipFileName: 'myS3File.zip' 117 | , tmpDir: "/tmp" // optional, defaults to node_modules/aws-s3-zipper 118 | },function(err,result){ 119 | if(err) 120 | console.error(err); 121 | else{ 122 | var lastFile = result.zippedFiles[result.zippedFiles.length-1]; 123 | if(lastFile) 124 | console.log('last key ', lastFile.Key); // next time start from here 125 | } 126 | }); 127 | ``` 128 | 129 | ### Zip fragments to S3 130 | ``` 131 | zipper.zipToS3FileFragments({ 132 | s3FolderName: 'myBucketFolderName' 133 | , startKey: 'keyOfLastFileIZipped' // optional 134 | , s3ZipFileName: 'myS3File.zip' 135 | , maxFileCount: 5 136 | , maxFileSize: 1024*1024 137 | , tmpDir: "/tmp" // optional, defaults to node_modules/aws-s3-zipper 138 | },function(err, results){ 139 | if(err) 140 | console.error(err); 141 | else if(results.length > 0) { 142 | var result = results[results.length - 1]; 143 | var lastFile = result.zippedFiles[result.zippedFiles.length - 1]; 144 | if (lastFile) 145 | console.log('last key ', lastFile.Key); // next time start from here 146 | } 147 | 148 | }); 149 | ``` 150 | 151 | ## The Details 152 | ### `init` 153 | Either from the constructor or from the `init(config)` function you can pass along the AWS config object 154 | ``` 155 | { 156 | accessKeyId: [Your access id], 157 | secretAccessKey: [your access key], 158 | region: [the region of your S3 bucket], 159 | bucket: [your bucket name], 160 | endpoint: [optional, for use with S3-compatible services] 161 | } 162 | ``` 163 | If using temporary credentials 164 | ``` 165 | { 166 | accessKeyId: [Your access id], 167 | secretAccessKey: [your access key], 168 | sessionToken: [your session token], 169 | region: [the region of your S3 bucket], 170 | bucket: [your bucket name], 171 | endpoint: [optional, for use with S3-compatible services] 172 | } 173 | ``` 174 | ### `filterOutFiles(file)` 175 | Override this function when you want to filter out certain files. The `file` param that is passed to you is the format of the aws file 176 | * file 177 | ``` 178 | /// as of when this document was written 179 | { 180 | Key: [file key], // this is what you use to keep track of where you left off 181 | ETag: [file tag], 182 | LastModified: [i'm sure you get it], 183 | Owner: {}, 184 | Size: [in bytes], 185 | StorageClass: [type of storage] 186 | } 187 | ``` 188 | 189 | ### `getFiles: function(params,callback)` 190 | Get a list of files in the bucket folder 191 | * `params` object 192 | * `folderName` : the name of the folder in the bucket 193 | * `startKey`: optional. return files listed after this file key 194 | * `recursive`: bool optional. to zip nested folders or not 195 | * `callback(err,result)`: the function you want called when the list returns 196 | * `err`: error object if it exists 197 | * `result`: 198 | * `files`: array of files found 199 | * `totalFilesScanned`: total number of files scanned including filter out files from the `filterOutFiles` function 200 | 201 | ### `streamZipDataTo: function (params, callback)` 202 | If you want to get a stream to pipe raw data to. For example if you wanted to stream the zip file directly to an http response 203 | * `params` object 204 | * `pipe`: the pipe to which you want the stream to feed 205 | * `folderName`: the name of the bucket folder you want to stream 206 | * `startKey`: optional. start zipping after this file key 207 | * `recursive`: bool optional. to zip nested folders or not 208 | * `callback(err,result)`: call this function when done 209 | * `err`: the error object if any 210 | * `result`: the resulting archiver zip object with attached property 'manifest' whcih is an array of files it zipped 211 | 212 | ### `zipToS3File: function (params ,callback)` 213 | Zip files in an s3 folder and place the zip file back on s3 214 | * `params` object 215 | * `s3FolderName`: the name of the bucket folder you want to stream 216 | * `startKey`: optional. start zipping after this file key 217 | * `s3FilerName`: the name of the new s3 zip file including its path. if no path is given it will defult to the same s3 folder 218 | * `recursive`: bool optional. to zip nested folders or not 219 | * `callback(err,result)`: call this function when done 220 | * `err`: the error object if any 221 | * `result`: the resulting archiver zip object with attached property 'manifest' whcih is an array of files it zipped 222 | 223 | ### `zipToS3FileFragments: function (params , callback)` 224 | * `params` object 225 | * `s3FolderName`: the name of the bucket folder you want to stream 226 | * `startKey`: optional. start zipping after this file key 227 | * `s3ZipFileName`: the pattern of the name of the S3 zip files to be uploaded. Fragments will have an underscore and index at the end of the file name example ["allImages_1.zip","allImages_2.zip","allImages_3.zip"] 228 | * `maxFileCount`: Optional. maximum number of files to zip in a single fragment. 229 | * `maxFileSize`: Optional. Maximum Bytes to fit into a single zip fragment. Note: If a file is found larger than the limit a separate fragment will becreated just for it. 230 | * `recursive`: bool optional. to zip nested folders or not 231 | * `callback(err,result)`: call this function when done 232 | * `err`: the error object if any 233 | * `results`: the array of results 234 | 235 | ### `zipToFile: function (params ,callback)` 236 | Zip files to a local zip file. 237 | * `params` object 238 | * `s3FolderName`: the name of the bucket folder you want to stream 239 | * `startKey`: optional. start zipping after this file key 240 | * `zipFileName`: the name of the new local zip file including its path. 241 | * `recursive`: bool optional. to zip nested folders or not 242 | * `callback(err,result)`: call this function when done 243 | * `err`: the error object if any 244 | * `result`: the resulting archiver zip object with attached property 'manifest' whcih is an array of files it zipped 245 | 246 | ### `zipToFileFragments: function (params,callback)` 247 | * `params` object 248 | * `s3FolderName`: the name of the bucket folder you want to stream 249 | * `startKey`: optional. start zipping after this file key 250 | * `zipFileName`: the pattern of the name of the zip files to be uploaded. Fragments will have an underscore and index at the end of the file name example ["allImages_1.zip","allImages_2.zip","allImages_3.zip"] 251 | * `maxFileCount`: Optional. maximum number of files to zip in a single fragment. 252 | * `maxFileSize`: Optional. Maximum Bytes to fit into a single zip fragment. Note: If a file is found larger than the limit a separate fragment will becreated just for it. 253 | * `recursive`: bool optional. to zip nested folders or not 254 | * `callback(err,result)`: call this function when done 255 | * `err`: the error object if any 256 | * `results`: the array of results 257 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | "@auth0/s3@latest": 6 | version "1.0.0" 7 | resolved "https://registry.yarnpkg.com/@auth0/s3/-/s3-1.0.0.tgz#2f4cd16e04e583955fe594407918315bfa76a622" 8 | integrity sha512-O8PTXJnA7n8ONBSwqlWa+aZ/vlOdZYnSCGQt25h87ALWNItY/Yij79TOnzIkMTJZ8aCpGXQPuIRziLmBliV++Q== 9 | dependencies: 10 | aws-sdk "^2.346.0" 11 | fd-slicer "~1.0.0" 12 | findit2 "~2.2.3" 13 | graceful-fs "~4.1.4" 14 | mime "^2.3.1" 15 | mkdirp "~0.5.0" 16 | pend "~1.2.0" 17 | rimraf "~2.2.8" 18 | streamsink "~1.2.0" 19 | 20 | archiver-utils@~0.3.0: 21 | version "0.3.0" 22 | resolved "https://registry.yarnpkg.com/archiver-utils/-/archiver-utils-0.3.0.tgz#9d85340892fe9e7c6c1b1b6af3c33692cf595b70" 23 | integrity sha1-nYU0CJL+nnxsGxtq88M2ks9ZW3A= 24 | dependencies: 25 | glob "~6.0.0" 26 | lazystream "~0.1.0" 27 | lodash "~3.10.0" 28 | normalize-path "~2.0.0" 29 | readable-stream "~2.0.0" 30 | 31 | archiver@0.21.*: 32 | version "0.21.0" 33 | resolved "https://registry.yarnpkg.com/archiver/-/archiver-0.21.0.tgz#129da0c8eae1a9bf08a012d8a4e2043a1e2909ce" 34 | integrity sha1-Ep2gyOrhqb8IoBLYpOIEOh4pCc4= 35 | dependencies: 36 | archiver-utils "~0.3.0" 37 | async "~1.5.0" 38 | buffer-crc32 "~0.2.1" 39 | glob "~6.0.0" 40 | lodash "~3.10.0" 41 | readable-stream "~2.0.0" 42 | tar-stream "~1.3.1" 43 | zip-stream "~0.8.0" 44 | 45 | async@1.5.*, async@~1.5.0: 46 | version "1.5.2" 47 | resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" 48 | integrity sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo= 49 | 50 | aws-sdk@2.814.*, aws-sdk@^2.346.0: 51 | version "2.814.0" 52 | resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.814.0.tgz#7a1c36006e0b5826f14bd2511b1d229ef6814bb0" 53 | integrity sha512-empd1m/J/MAkL6d9OeRpmg9thobULu0wk4v8W3JToaxGi2TD7PIdvE6yliZKyOVAdJINhBWEBhxR4OUIHhcGbQ== 54 | dependencies: 55 | buffer "4.9.2" 56 | events "1.1.1" 57 | ieee754 "1.1.13" 58 | jmespath "0.15.0" 59 | querystring "0.2.0" 60 | sax "1.2.1" 61 | url "0.10.3" 62 | uuid "3.3.2" 63 | xml2js "0.4.19" 64 | 65 | balanced-match@^1.0.0: 66 | version "1.0.0" 67 | resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" 68 | integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= 69 | 70 | base64-js@^1.0.2: 71 | version "1.3.1" 72 | resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" 73 | integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g== 74 | 75 | bl@^1.0.0: 76 | version "1.2.3" 77 | resolved "https://registry.yarnpkg.com/bl/-/bl-1.2.3.tgz#1e8dd80142eac80d7158c9dccc047fb620e035e7" 78 | integrity sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww== 79 | dependencies: 80 | readable-stream "^2.3.5" 81 | safe-buffer "^5.1.1" 82 | 83 | brace-expansion@^1.1.7: 84 | version "1.1.11" 85 | resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" 86 | integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== 87 | dependencies: 88 | balanced-match "^1.0.0" 89 | concat-map "0.0.1" 90 | 91 | buffer-crc32@~0.2.1: 92 | version "0.2.13" 93 | resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" 94 | integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI= 95 | 96 | buffer@4.9.2: 97 | version "4.9.2" 98 | resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" 99 | integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== 100 | dependencies: 101 | base64-js "^1.0.2" 102 | ieee754 "^1.1.4" 103 | isarray "^1.0.0" 104 | 105 | compress-commons@~0.4.0: 106 | version "0.4.2" 107 | resolved "https://registry.yarnpkg.com/compress-commons/-/compress-commons-0.4.2.tgz#1a0847489f0d713d0dd76343c0c2d055dc9cd892" 108 | integrity sha1-GghHSJ8NcT0N12NDwMLQVdyc2JI= 109 | dependencies: 110 | buffer-crc32 "~0.2.1" 111 | crc32-stream "~0.4.0" 112 | node-int64 "~0.4.0" 113 | normalize-path "~2.0.0" 114 | readable-stream "~2.0.0" 115 | 116 | concat-map@0.0.1: 117 | version "0.0.1" 118 | resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" 119 | integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= 120 | 121 | core-util-is@~1.0.0: 122 | version "1.0.3" 123 | resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" 124 | integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== 125 | 126 | crc32-stream@~0.4.0: 127 | version "0.4.0" 128 | resolved "https://registry.yarnpkg.com/crc32-stream/-/crc32-stream-0.4.0.tgz#b54d4c6eefd35b53e653d062b306edc6316ae26d" 129 | integrity sha1-tU1Mbu/TW1PmU9BiswbtxjFq4m0= 130 | dependencies: 131 | buffer-crc32 "~0.2.1" 132 | readable-stream "~2.0.0" 133 | 134 | end-of-stream@^1.0.0: 135 | version "1.4.4" 136 | resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" 137 | integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== 138 | dependencies: 139 | once "^1.4.0" 140 | 141 | events@1.1.1: 142 | version "1.1.1" 143 | resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" 144 | integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= 145 | 146 | fd-slicer@~1.0.0: 147 | version "1.0.1" 148 | resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.0.1.tgz#8b5bcbd9ec327c5041bf9ab023fd6750f1177e65" 149 | integrity sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU= 150 | dependencies: 151 | pend "~1.2.0" 152 | 153 | findit2@~2.2.3: 154 | version "2.2.3" 155 | resolved "https://registry.yarnpkg.com/findit2/-/findit2-2.2.3.tgz#58a466697df8a6205cdfdbf395536b8bd777a5f6" 156 | integrity sha1-WKRmaX34piBc39vzlVNri9d3pfY= 157 | 158 | glob@~6.0.0: 159 | version "6.0.4" 160 | resolved "https://registry.yarnpkg.com/glob/-/glob-6.0.4.tgz#0f08860f6a155127b2fadd4f9ce24b1aab6e4d22" 161 | integrity sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI= 162 | dependencies: 163 | inflight "^1.0.4" 164 | inherits "2" 165 | minimatch "2 || 3" 166 | once "^1.3.0" 167 | path-is-absolute "^1.0.0" 168 | 169 | graceful-fs@~4.1.4: 170 | version "4.1.15" 171 | resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00" 172 | integrity sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA== 173 | 174 | ieee754@1.1.13, ieee754@^1.1.4: 175 | version "1.1.13" 176 | resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" 177 | integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== 178 | 179 | inflight@^1.0.4: 180 | version "1.0.6" 181 | resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" 182 | integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= 183 | dependencies: 184 | once "^1.3.0" 185 | wrappy "1" 186 | 187 | inherits@2, inherits@~2.0.1, inherits@~2.0.3: 188 | version "2.0.4" 189 | resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" 190 | integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== 191 | 192 | isarray@0.0.1: 193 | version "0.0.1" 194 | resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" 195 | integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8= 196 | 197 | isarray@^1.0.0, isarray@~1.0.0: 198 | version "1.0.0" 199 | resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" 200 | integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= 201 | 202 | jmespath@0.15.0: 203 | version "0.15.0" 204 | resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" 205 | integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= 206 | 207 | lazystream@~0.1.0: 208 | version "0.1.0" 209 | resolved "https://registry.yarnpkg.com/lazystream/-/lazystream-0.1.0.tgz#1b25d63c772a4c20f0a5ed0a9d77f484b6e16920" 210 | integrity sha1-GyXWPHcqTCDwpe0KnXf0hLbhaSA= 211 | dependencies: 212 | readable-stream "~1.0.2" 213 | 214 | lodash@~3.10.0, lodash@~3.10.1: 215 | version "3.10.1" 216 | resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6" 217 | integrity sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y= 218 | 219 | mime@^2.3.1: 220 | version "2.4.4" 221 | resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.4.tgz#bd7b91135fc6b01cde3e9bae33d659b63d8857e5" 222 | integrity sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA== 223 | 224 | "minimatch@2 || 3": 225 | version "3.0.4" 226 | resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" 227 | integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== 228 | dependencies: 229 | brace-expansion "^1.1.7" 230 | 231 | minimist@0.0.8: 232 | version "0.0.8" 233 | resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" 234 | integrity sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0= 235 | 236 | mkdirp@~0.5.0: 237 | version "0.5.1" 238 | resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" 239 | integrity sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM= 240 | dependencies: 241 | minimist "0.0.8" 242 | 243 | node-int64@~0.4.0: 244 | version "0.4.0" 245 | resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" 246 | integrity sha1-h6kGXNs1XTGC2PlM4RGIuCXGijs= 247 | 248 | normalize-path@~2.0.0: 249 | version "2.0.1" 250 | resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-2.0.1.tgz#47886ac1662760d4261b7d979d241709d3ce3f7a" 251 | integrity sha1-R4hqwWYnYNQmG32XnSQXCdPOP3o= 252 | 253 | once@^1.3.0, once@^1.4.0: 254 | version "1.4.0" 255 | resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" 256 | integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= 257 | dependencies: 258 | wrappy "1" 259 | 260 | path-is-absolute@^1.0.0: 261 | version "1.0.1" 262 | resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" 263 | integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= 264 | 265 | pend@~1.2.0: 266 | version "1.2.0" 267 | resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" 268 | integrity sha1-elfrVQpng/kRUzH89GY9XI4AelA= 269 | 270 | process-nextick-args@~1.0.6: 271 | version "1.0.7" 272 | resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3" 273 | integrity sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M= 274 | 275 | process-nextick-args@~2.0.0: 276 | version "2.0.1" 277 | resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" 278 | integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== 279 | 280 | punycode@1.3.2: 281 | version "1.3.2" 282 | resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" 283 | integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= 284 | 285 | querystring@0.2.0: 286 | version "0.2.0" 287 | resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" 288 | integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= 289 | 290 | readable-stream@^2.0.0, readable-stream@^2.3.5: 291 | version "2.3.7" 292 | resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" 293 | integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== 294 | dependencies: 295 | core-util-is "~1.0.0" 296 | inherits "~2.0.3" 297 | isarray "~1.0.0" 298 | process-nextick-args "~2.0.0" 299 | safe-buffer "~5.1.1" 300 | string_decoder "~1.1.1" 301 | util-deprecate "~1.0.1" 302 | 303 | readable-stream@~1.0.2: 304 | version "1.0.34" 305 | resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" 306 | integrity sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw= 307 | dependencies: 308 | core-util-is "~1.0.0" 309 | inherits "~2.0.1" 310 | isarray "0.0.1" 311 | string_decoder "~0.10.x" 312 | 313 | readable-stream@~2.0.0: 314 | version "2.0.6" 315 | resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.0.6.tgz#8f90341e68a53ccc928788dacfcd11b36eb9b78e" 316 | integrity sha1-j5A0HmilPMySh4jaz80Rs265t44= 317 | dependencies: 318 | core-util-is "~1.0.0" 319 | inherits "~2.0.1" 320 | isarray "~1.0.0" 321 | process-nextick-args "~1.0.6" 322 | string_decoder "~0.10.x" 323 | util-deprecate "~1.0.1" 324 | 325 | rimraf@~2.2.8: 326 | version "2.2.8" 327 | resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.2.8.tgz#e439be2aaee327321952730f99a8929e4fc50582" 328 | integrity sha1-5Dm+Kq7jJzIZUnMPmaiSnk/FBYI= 329 | 330 | safe-buffer@^5.1.1: 331 | version "5.2.1" 332 | resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" 333 | integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== 334 | 335 | safe-buffer@~5.1.0, safe-buffer@~5.1.1: 336 | version "5.1.2" 337 | resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" 338 | integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== 339 | 340 | sax@1.2.1: 341 | version "1.2.1" 342 | resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" 343 | integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= 344 | 345 | sax@>=0.6.0: 346 | version "1.2.4" 347 | resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" 348 | integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== 349 | 350 | streamsink@~1.2.0: 351 | version "1.2.0" 352 | resolved "https://registry.yarnpkg.com/streamsink/-/streamsink-1.2.0.tgz#efafee9f1e22d3591ed7de3dcaa95c3f5e79f73c" 353 | integrity sha1-76/unx4i01ke1949yqlcP1559zw= 354 | 355 | string_decoder@~0.10.x: 356 | version "0.10.31" 357 | resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" 358 | integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ= 359 | 360 | string_decoder@~1.1.1: 361 | version "1.1.1" 362 | resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" 363 | integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== 364 | dependencies: 365 | safe-buffer "~5.1.0" 366 | 367 | tar-stream@~1.3.1: 368 | version "1.3.2" 369 | resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-1.3.2.tgz#724d1ab4801c9b3149cdea765fe8c90ea71f6606" 370 | integrity sha1-ck0atIAcmzFJzep2X+jJDqcfZgY= 371 | dependencies: 372 | bl "^1.0.0" 373 | end-of-stream "^1.0.0" 374 | readable-stream "^2.0.0" 375 | xtend "^4.0.0" 376 | 377 | url@0.10.3: 378 | version "0.10.3" 379 | resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" 380 | integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= 381 | dependencies: 382 | punycode "1.3.2" 383 | querystring "0.2.0" 384 | 385 | util-deprecate@~1.0.1: 386 | version "1.0.2" 387 | resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" 388 | integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= 389 | 390 | uuid@3.3.2: 391 | version "3.3.2" 392 | resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" 393 | integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== 394 | 395 | wrappy@1: 396 | version "1.0.2" 397 | resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" 398 | integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= 399 | 400 | xml2js@0.4.19: 401 | version "0.4.19" 402 | resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" 403 | integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== 404 | dependencies: 405 | sax ">=0.6.0" 406 | xmlbuilder "~9.0.1" 407 | 408 | xmlbuilder@~9.0.1: 409 | version "9.0.7" 410 | resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" 411 | integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= 412 | 413 | xtend@^4.0.0: 414 | version "4.0.2" 415 | resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" 416 | integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== 417 | 418 | zip-stream@~0.8.0: 419 | version "0.8.0" 420 | resolved "https://registry.yarnpkg.com/zip-stream/-/zip-stream-0.8.0.tgz#236b2fe25823cb4f48e8336f5bfa743aa5ae9dbd" 421 | integrity sha1-I2sv4lgjy09I6DNvW/p0OqWunb0= 422 | dependencies: 423 | archiver-utils "~0.3.0" 424 | compress-commons "~0.4.0" 425 | lodash "~3.10.1" 426 | readable-stream "~2.0.0" 427 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var archiver = require('archiver'); 3 | var async = require('async'); 4 | var AWS = require('aws-sdk'); 5 | var fs = require('fs'); 6 | var s3 = require('@auth0/s3'); 7 | 8 | function S3Zipper(awsConfig) { 9 | var self = this 10 | assert.ok(awsConfig, 'AWS S3 options must be defined.'); 11 | assert.notEqual(awsConfig.accessKeyId, undefined, 'Requires S3 AWS Key.'); 12 | assert.notEqual(awsConfig.secretAccessKey, undefined, 'Requires S3 AWS Secret'); 13 | assert.notEqual(awsConfig.region, undefined, 'Requires AWS S3 region.'); 14 | assert.notEqual(awsConfig.bucket, undefined, 'Requires AWS S3 bucket.'); 15 | 16 | if(awsConfig.sessionToken) { 17 | AWS.config.update({ 18 | accessKeyId: awsConfig.accessKeyId, 19 | secretAccessKey: awsConfig.secretAccessKey, 20 | sessionToken: awsConfig.sessionToken, 21 | region: awsConfig.region 22 | }); } 23 | else { 24 | AWS.config.update({ 25 | accessKeyId: awsConfig.accessKeyId, 26 | secretAccessKey: awsConfig.secretAccessKey, 27 | region: awsConfig.region 28 | } 29 | 30 | self.init(awsConfig); 31 | } 32 | 33 | S3Zipper.prototype = { 34 | init: function (awsConfig) { 35 | this.awsConfig = awsConfig; 36 | var self = this 37 | self.s3bucket = new AWS.S3({ 38 | params: { 39 | Bucket: self.awsConfig.bucket 40 | } 41 | }); 42 | } 43 | , filterOutFiles: function (fileObj) { 44 | return fileObj; 45 | } 46 | , calculateFileName: function (f) { 47 | var name = f.Key.split("/"); 48 | name.shift(); 49 | name = name.join("/"); 50 | return name; 51 | 52 | } 53 | 54 | /* 55 | params={ 56 | folderName: the name of the folder within the s3 bucket 57 | , startKey: the key of the file you want to start after. keep null if you want to start from the first file 58 | , maxFileCount: an integer that caps off how many files to zip at a time 59 | , maxFileSize: max total size of files before they are zipped 60 | , recursive: option to loop through nested folders 61 | }; 62 | callback = function that is called back when completed 63 | * */ 64 | , getFiles: function (params,callback) { 65 | 66 | if(arguments.length == 5){ //for backwards comparability 67 | params={ 68 | folderName: arguments[0] 69 | , startKey: arguments[1] 70 | , maxFileCount: arguments[2] 71 | , maxFileSize: arguments[3] 72 | , recursive:false 73 | }; 74 | callback = arguments[4]; 75 | } 76 | 77 | 78 | var bucketParams = { 79 | Bucket: this.awsConfig.bucket, /* required */ 80 | Delimiter: "/", 81 | Prefix: params.folderName + "/" 82 | }; 83 | 84 | if (params.startKey) 85 | bucketParams.Marker = params.startKey; 86 | 87 | if (typeof(params.maxFileCount) == "function" && typeof(callback) == "undefined") { 88 | callback = params.maxFileCount; 89 | params.maxFileCount = null; 90 | } 91 | else if (params.maxFileCount > 0) 92 | bucketParams.MaxKeys = params.maxFileCount; 93 | 94 | var t = this; 95 | 96 | var files ={}; 97 | files.Contents = []; 98 | 99 | var options = { 100 | s3Client: this.s3bucket 101 | // more options available. See API docs below. 102 | }; 103 | var client = s3.createClient(options); 104 | 105 | var realParams = { 106 | s3Params: bucketParams, 107 | recursive: params.recursive 108 | }; 109 | 110 | var emitter = client.listObjects(realParams); 111 | emitter.on('data', function (data) { 112 | if(data && data.Contents) { 113 | files.Contents = files.Contents.concat(data.Contents); 114 | } 115 | }); 116 | 117 | emitter.on('error', function(err) { 118 | console.error('unable to get files:', err.stack); 119 | callback(err); 120 | }); 121 | 122 | emitter.on('end', function () { 123 | var data = files; 124 | console.log('end'); 125 | var result = []; 126 | var totalSizeOfPassedFiles = 0; 127 | var lastScannedFile; 128 | for (var i = 0; i < data.Contents.length; i++) { 129 | 130 | var passedFile = t.filterOutFiles(data.Contents[i]); 131 | if (passedFile) { 132 | if (params.maxFileSize && params.maxFileSize < passedFile.Size) { 133 | console.warn('Single file size exceeds max allowed size', data.Contents[i].Size, '>', params.maxFileSize, passedFile); 134 | if (result.length == 0) { 135 | console.warn('Will zip large file on its own', passedFile.Key); 136 | result.push(passedFile); 137 | totalSizeOfPassedFiles += passedFile.Size; 138 | } 139 | else 140 | break; 141 | } 142 | else if (params.maxFileSize && totalSizeOfPassedFiles + data.Contents[i].Size > params.maxFileSize) { 143 | console.log('Hit max size limit. Split fragment'); 144 | break; 145 | } 146 | else { 147 | result.push(passedFile); 148 | totalSizeOfPassedFiles += passedFile.Size; 149 | } 150 | } 151 | 152 | lastScannedFile = data.Contents[i]; 153 | } 154 | 155 | callback(null, {files: result, totalFilesScanned: data.Contents.length, lastScannedFile: lastScannedFile}); 156 | }); 157 | } 158 | 159 | /* 160 | params: { 161 | pipe : pipe stream 162 | , folderName: folder name to zip 163 | , startKey: the key of the file you want to start after. keep null if you want to start from the first file 164 | , maxFileCount: an integer that caps off how many files to zip at a time 165 | , maxFileSize: max total size of files before they are zipped 166 | , recursive: option to loop through nested folders 167 | } 168 | , callback : function 169 | */ 170 | , streamZipDataTo: function (params, callback) { 171 | if (!params || !params.folderName) { 172 | console.error('folderName required'); 173 | return null; 174 | } 175 | 176 | 177 | var zip = new archiver.create('zip'); 178 | if (params.pipe) zip.pipe(params.pipe); 179 | 180 | var t = this; 181 | 182 | this.getFiles(params, function (err, clearedFiles) { 183 | if (err) 184 | console.error(err); 185 | else { 186 | var files = clearedFiles.files; 187 | console.log("files", files); 188 | async.map(files, function (f, callback) { 189 | t.s3bucket.getObject({Bucket: t.awsConfig.bucket, Key: f.Key}, function (err, data) { 190 | if (err) 191 | callback(err); 192 | else { 193 | 194 | var name = t.calculateFileName(f); 195 | 196 | if (name === ""){ 197 | callback(null, f); 198 | return; 199 | } 200 | else { 201 | console.log('zipping ', name, '...'); 202 | 203 | zip.append(data.Body, {name: name}); 204 | callback(null, f); 205 | } 206 | 207 | } 208 | 209 | }); 210 | 211 | }, function (err, results) { 212 | zip.manifest = results; 213 | zip.on('finish',function(){ 214 | callback(err, { 215 | zip: zip, 216 | zippedFiles: results, 217 | totalFilesScanned: clearedFiles.totalFilesScanned, 218 | lastScannedFile: clearedFiles.lastScannedFile 219 | }); 220 | }); 221 | zip.finalize(); 222 | 223 | 224 | 225 | }); 226 | } 227 | }); 228 | 229 | } 230 | 231 | 232 | , uploadLocalFileToS3: function (localFileName, s3ZipFileName, callback) { 233 | console.log('uploading ', s3ZipFileName, '...'); 234 | var readStream = fs.createReadStream(localFileName);//tempFile 235 | 236 | this.s3bucket.upload({ 237 | Bucket: this.awsConfig.bucket 238 | , Key: s3ZipFileName 239 | , ContentType: "application/zip" 240 | , Body: readStream 241 | }) 242 | .on('httpUploadProgress', function (e) { 243 | var p = Math.round(e.loaded / e.total * 100); 244 | if (p % 10 == 0) 245 | console.log('upload progress', p, '%'); 246 | 247 | }) 248 | .send(function (err, result) { 249 | readStream.close(); 250 | if (err) 251 | callback(err); 252 | else { 253 | console.log('upload completed.'); 254 | callback(null, result); 255 | } 256 | }); 257 | } 258 | //all these timeouts are because streams arent done writing when they say they are 259 | 260 | /* 261 | params: { 262 | s3FolderName: the name of the folder within the S3 bucket 263 | , startKey: the key of the file you want to start after. keep null if you want to start from the first file 264 | , s3ZipFileName: the name of the file you to zip to and upload to S3 265 | , tmpDir: specifies the directory of the temporal zip file, default is node_modules/aws-s3-zipper 266 | , recursive: indicates to zip nested folders or not 267 | } 268 | , callback: function 269 | */ 270 | , zipToS3File: function (params, callback) { 271 | 272 | if(arguments.length == 5){ 273 | // for backward compatibility 274 | params = { 275 | s3FolderName:arguments[0] 276 | ,startKey:arguments[1] 277 | ,s3ZipFileName:arguments[2] 278 | ,tmpDir:arguments[3] 279 | ,recursive: false 280 | }; 281 | callback= arguments[4]; 282 | } 283 | 284 | var t = this; 285 | params.tmpDir = params.tmpDir?params.tmpDir+"/":"" 286 | params.zipFileName = params.tmpDir+'__' + Date.now() + '.zip'; 287 | 288 | if (params.s3ZipFileName.indexOf('/') < 0) 289 | params.s3ZipFileName = params.s3FolderName + "/" + params.s3ZipFileName; 290 | 291 | 292 | this.zipToFile(params, function (err, r) { 293 | 294 | if (r && r.zippedFiles && r.zippedFiles.length) { 295 | t.uploadLocalFileToS3( params.zipFileName, params.s3ZipFileName, function (err, result) { 296 | callback(null, { 297 | zipFileETag: result.ETag, 298 | zipFileLocation: result.Location, 299 | zippedFiles: r.zippedFiles 300 | }); 301 | fs.unlinkSync(params.zipFileName); 302 | }); 303 | } 304 | else { 305 | console.log('no files zipped. nothing to upload'); 306 | fs.unlinkSync(params.zipFileName); 307 | callback(null, { 308 | zipFileETag: null, 309 | zipFileLocation: null, 310 | zippedFiles: [] 311 | }); 312 | } 313 | }); 314 | 315 | 316 | } 317 | 318 | /* 319 | params: { 320 | s3FolderName: the name of the folder within the S3 bucket 321 | , startKey: the key of the file you want to start after. keep null if you want to start from the first file 322 | , s3ZipFileName: the name of the file you to zip to and upload to S3 323 | , maxFileCount: an integer that caps off how many files to zip at a time 324 | , maxFileSize: max total size of files before they are zipped 325 | , tmpDir: specifies the directory of the temporal zip file, default is node_modules/aws-s3-zipper 326 | , recursive: indicates to zip nested folders or not 327 | } 328 | , callback: function 329 | */ 330 | , zipToS3FileFragments: function (params, callback) { 331 | 332 | if(arguments.length == 7){ 333 | // for backward compatibility 334 | params = { 335 | s3FolderName:arguments[0] 336 | , startKey:arguments[1] 337 | , s3ZipFileName:arguments[2] 338 | , maxFileCount:arguments[3] 339 | , maxFileSize:arguments[4] 340 | , tmpDir:arguments[5] 341 | , recursive: false 342 | }; 343 | callback= arguments[6]; 344 | } 345 | 346 | var t = this; 347 | ///local file 348 | params.tmpDir = params.tmpDir?params.tmpDir+"/":"" 349 | params.zipFileName = params.tmpDir+'__' + Date.now() + '.zip'; 350 | 351 | if (params.s3ZipFileName.indexOf('/') < 0) 352 | params.s3ZipFileName = params.s3FolderName + "/" + params.s3ZipFileName; 353 | 354 | var finalResult; 355 | 356 | var count = 0; 357 | this.zipToFileFragments(params, function (err, result) { 358 | if (err) 359 | callback(err); 360 | else { 361 | finalResult = result; 362 | if (!result || result.length == 0) 363 | callback(null, result); /// dont need to wait for uploads 364 | } 365 | }) 366 | .onFileZipped = function (fragFileName, result) { 367 | var s3fn = params.s3ZipFileName.replace(".zip", "_" + count + ".zip"); 368 | count++; 369 | uploadFrag(s3fn, fragFileName, result); 370 | }; 371 | 372 | var pendingUploads = 0;// prevent race condition 373 | function uploadFrag(s3FragName, localFragName, result) { 374 | pendingUploads++; 375 | t.uploadLocalFileToS3(localFragName, s3FragName, function (err, uploadResult) { 376 | 377 | if (uploadResult) { 378 | result.uploadedFile = uploadResult; 379 | console.log('remove temp file ', localFragName); 380 | fs.unlinkSync(localFragName); 381 | } 382 | pendingUploads--; 383 | if (pendingUploads == 0 && finalResult) { 384 | callback(null, finalResult); 385 | } 386 | }); 387 | } 388 | 389 | 390 | } 391 | /* 392 | params={ 393 | s3FolderName: the name of the folder within the s3 bucket 394 | , startKey: the key of the file you want to start after. keep null if you want to start from the first file 395 | , zipFileName: zip file name 396 | , recursive: option to loop through nested folders 397 | }; 398 | callback = function that is called back when completed 399 | * */ 400 | , zipToFile: function (params, callback) { 401 | 402 | if(arguments.length == 4){ 403 | // for backward compatibility 404 | params = { 405 | s3FolderName:arguments[0] 406 | , startKey:arguments[1] 407 | , zipFileName:arguments[2] 408 | , recursive: false 409 | }; 410 | callback= arguments[3]; 411 | } 412 | 413 | var filestream = fs.createWriteStream(params.zipFileName); 414 | this.streamZipDataTo({ 415 | pipe: filestream 416 | ,folderName: params.s3FolderName 417 | , startKey: params.startKey 418 | , maxFileCount: params.maxFileCount 419 | , maxFileSize: params.maxFileSize 420 | , recursive: params.recursive 421 | }, function (err, result) { 422 | setTimeout(function () { 423 | callback(err, result); 424 | filestream.close(); 425 | }, 1000); 426 | }); 427 | } 428 | 429 | /* 430 | params: { 431 | s3FolderName: the name of the folder within the S3 bucket 432 | , startKey: the key of the file you want to start after. keep null if you want to start from the first file 433 | , zipFileName: the name of the file you to zip to and upload to S3 434 | , maxFileCount: an integer that caps off how many files to zip at a time 435 | , maxFileSize: max total size of files before they are zipped 436 | , recursive: indicates to zip nested folders or not 437 | } 438 | , callback: function 439 | */ 440 | , zipToFileFragments: function (params, callback) { 441 | 442 | if(arguments.length == 6){ 443 | // for backward compatibility 444 | params = { 445 | s3FolderName:arguments[0] 446 | , startKey:arguments[1] 447 | , s3ZipFileName:arguments[2] 448 | , maxFileCount:arguments[3] 449 | , maxFileSize:arguments[4] 450 | , recursive: false 451 | }; 452 | callback= arguments[5]; 453 | } 454 | 455 | var events = { 456 | onFileZipped: function () { 457 | } 458 | }; 459 | 460 | var report = { 461 | results: [] 462 | , errors: [] 463 | , lastKey: null 464 | }; 465 | 466 | if (params.maxFileSize && params.maxFileSize < 1024) 467 | console.warn('Max File Size is really low. This may cause no files to be zipped, maxFileSize set to ', params.maxFileSize); 468 | 469 | if (params.zipFileName.indexOf(".zip") < 0) 470 | params.zipFileName += ".zip"; 471 | 472 | var t = this; 473 | 474 | function garbageCollector(fileStream, result, fragFileName) { 475 | 476 | setTimeout(function () { 477 | 478 | fileStream.close(); 479 | if (result.zippedFiles.length == 0) /// its an empty zip file get rid of it 480 | 481 | fs.unlinkSync(fragFileName); 482 | 483 | else 484 | events.onFileZipped(fragFileName, result); 485 | }, 1000); /// TODO: Zip needs a bit more time to finishing writing. I'm sure there is a better way 486 | } 487 | 488 | var counter = 0; 489 | 490 | function recursiveLoop(startKey, fragFileName, callback) { 491 | var fileStream = fs.createWriteStream(fragFileName); 492 | t.streamZipDataTo({ 493 | pipe : fileStream 494 | , folderName: params.s3FolderName 495 | , startKey:startKey 496 | , maxFileCount:params.maxFileCount 497 | , maxFileSize:params.maxFileSize 498 | , recursive: params.recursive }, function (err, result) { 499 | 500 | if (err) 501 | report.errors.push(err); 502 | else { 503 | if (result.zippedFiles.length > 0) { 504 | report.results.push(result); 505 | report.lastKey = result.zippedFiles[result.zippedFiles.length - 1].Key; 506 | } 507 | 508 | 509 | /// you may have not zipped anything but you scanned files and there may be more 510 | if (result.totalFilesScanned > 0) 511 | recursiveLoop(result.lastScannedFile.Key, params.zipFileName.replace(".zip", "_" + counter + ".zip"), callback); 512 | else ///you're done time to go home 513 | callback(err, result); 514 | 515 | counter++; 516 | /// clean up your trash you filthy animal 517 | garbageCollector(fileStream, result, fragFileName); 518 | 519 | } 520 | 521 | }); 522 | } 523 | 524 | recursiveLoop(params.startKey, params.zipFileName, function () { 525 | 526 | if (report.errors.length > 0) 527 | callback(report.errors, report.results); 528 | else 529 | callback(null, report.results); 530 | 531 | }); 532 | 533 | return events; 534 | 535 | } 536 | }; 537 | 538 | module.exports = S3Zipper; 539 | --------------------------------------------------------------------------------