├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── clients.json ├── package.json └── s3motion.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # node-waf configuration 20 | .lock-wscript 21 | 22 | # Compiled binary addons (http://nodejs.org/api/addons.html) 23 | build/Release 24 | 25 | # Dependency directory 26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 27 | node_modules -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ############################################################ 2 | # Dockerfile to start the s3motion REST interface in a container 3 | # Based on Node.js 4 | ############################################################ 5 | 6 | # Set the base image to Ubuntu 7 | FROM dockerfile/nodejs 8 | 9 | # File Author / Maintainer 10 | MAINTAINER Kendrick Coleman (kendrickcoleman@gmail.com) 11 | 12 | # Update the repository sources list 13 | RUN apt-get update 14 | 15 | ################## BEGIN INSTALLATION ###################### 16 | # Install s3motion 17 | RUN npm install s3motion -g 18 | 19 | ##################### INSTALLATION END ##################### 20 | # Expose the default port 8080 21 | EXPOSE 8080 22 | 23 | # Set default container command 24 | ENTRYPOINT ["s3motion"] 25 | 26 | # Start the REST service 27 | CMD ["--REST"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | s3motion 2 | ====================== 3 | s3motion is a combination of both a command line utility and REST based microservice used to upload and download local objects and transfer objects between S3 compatible storage. 4 | 5 | ## Description 6 | s3motion creates a simple CLI user interface or REST based microservice for migrating, copying, uploading, and downloading of objects to S3 compatible storage. The uses cases can be: 7 | - a command line tool to upload/download objects 8 | - a move/copy individual objects at scheduled intervals between buckets 9 | - migrate between services. copy entire bucket, then change your application to point to new service 10 | 11 | ## Installation 12 | Make sure node.js and npm is installed first, then install it globally by issuing: `npm install s3motion -g` 13 | 14 | To run it as a microservice by utilizing the REST implementation, deploy it in a docker container that can be called from any application. `docker run -d -p 8080:8080 kacole2/s3motion`. To interact with the container for CLI purposes, you can `docker run -ti --entrypoint=/bin/bash kacole2/s3motion` 15 | 16 | ## CLI Usage 17 | All commands are accessible via the `-h` or `--help` flag. Only one flag can be used during a single command. There are two modes to run the CLI utility. You can choose to pass all arguments through a single command, or run the wizard. To run the wizard simply type `wiz` or `wizard` after your chosen flag (ie `s3motion -n wizard`). 18 | 19 | - `-n` or `--newClient`: Add a new client to a locally stored s3motionClients.json object. The clients are used as a way to store credential information locally and pipe those in for operational use. The s3motionClients.json will be stores in the current users working home directory. Here are the following arguments that can be passed for a single line command: 20 | - `--name`: This is an arbitrary name you are using to identify your client. This name must be unique and cannot be the same as one previously used. If you want to use a previously configured name, edit the s3motionClients.json object. This is a required argument. 21 | - `--accessKeyId`: This is the S3 Access Key/ID. This is a required argument. 22 | - `--secretAccessKey`: This is the S3 Secret Access Key. This is a required argument. 23 | - `--endpoint`: Only specify this if you are trying to access third-party S3 compatible storage. If you are adding a new client for AWS, leave this blank. The endpoint assumes `https` and port `443`, therefore, a DNS or IP address will work such as `vipr.emc.com`. If your storage uses unsecured access, then you must specify the protocol and port such as `http://vipr.emc.com:80`. If you're endpoint uses a different port that can be specified as `vipr.emc.com:10101` which assumes `https`. 24 | - `-L` or `--listClients`: List all the clients available in the s3motionClients.json object. No arguments needed for this command. 25 | - `-b` or `--listBuckets`: List all the avialable buckets for a specific client. Here are the following arguments that can be passed for a single line command: 26 | - `--client`: Specify the name of the locally configued client. This is a required argument. 27 | - `-N` or `--newBucket`: Create a new bucket for a client. Here are the following arguments that can be passed for a single line command: 28 | - `--client`: Specify the name of the locally configued client. This is a required argument. 29 | - `--name`: Specify the name of the new bucket. This is a required argument. 30 | - `-l` or `--listObjects`: List all the objects in a bucket. Here are the following arguments that can be passed for a single line command: 31 | - `--client`: Specify the name of the locally configued client. This is a required argument. 32 | - `--bucket`: Specify the name of the bucket. This is a required argument. 33 | - `-d` or `--downloadObject`: Download an object(s) from a bucket. Here are the following arguments that can be passed for a single line command: 34 | - `--client`: Specify the name of the locally configued client. This is a required argument. 35 | - `--bucket`: Specify the name of the bucket. This is a required argument. 36 | - `--object`: Specify the name of the object. If the object is nested within a folder, specify the directory listing as well: `images/myimages/avatar.png`. You can also specify multiple objects to download by using a comma and no spaces such as `object1.png,object2.png,images/myimages/avatar.png`. This is a required argument. 37 | - `--folder`: Specify the location on your local machine where the object(s) will be download to. `/Users/me/Desktop`. If no folder is specified, then the object is downloaded to the current working directory. This is an optional argument. 38 | - `-u` or `--uploadObject`: Upload an object(s) to a bucket. Here are the following arguments that can be passed for a single line command: 39 | - `--client`: Specify the name of the locally configued client. This is a required argument. 40 | - `--bucket`: Specify the name of the bucket. This is a required argument. 41 | - `--object`: Specify the name of the object. By default, uploaded objects will be placed in the root directory. This is a required argument. 42 | - `--folder`: Specify the location on your local machine where the object(s) will be uploaded from. `/Users/me/Desktop`. If no folder is specified, then the object is downloaded to the current working directory. This is an optional argument. 43 | - `-D` or `--deleteObject`: Delete an object(s) from a bucket. Here are the following arguments that can be passed for a single line command: 44 | - `--client`: Specify the name of the locally configued client. This is a required argument. 45 | - `--bucket`: Specify the name of the bucket. This is a required argument. 46 | - `--object`: Specify the name of the object. If the object is nested within a folder, specify the directory listing as well: `images/myimages/avatar.png`. You can also specify multiple objects to delete by using a comma and no spaces such as `object1.png,object2.png,images/myimages/avatar.png`. This is a required argument. 47 | - `-c` or `--copyObject`: Copy an object(s) from one client to another. Here are the following arguments that can be passed for a single line command: 48 | - `--sourceClient`: Specify the name of the locally configued client where the object is stored. This is a required argument. 49 | - `--sourceBucket`: Specify the name of the bucket where the object is stored. This is a required argument. 50 | - `--object`: Specify the name of the object. If the object is nested within a folder, specify the directory listing as well: `images/myimages/avatar.png`. You can also specify multiple objects to copy by using a comma and no spaces such as `object1.png,object2.png,images/myimages/avatar.png`. This is a required argument. 51 | - `--destClient`: Specify the name of the locally configued client where the object will be copied to. This is a required argument. 52 | - `--destBucket`: Specify the name of the bucket where the object will be copied to. This is a required argument. 53 | - `--delete`: Default is `n` which means the source copy will remain in the bucket. Specify `Y` if the source object should be deleted upon a successful transfer. 54 | - `-C` or `--copyBucket`: Bulk copy process for all object from one bucket to another. Here are the following arguments that can be passed for a single line command: 55 | - `--sourceClient`: Specify the name of the locally configued client where the object is stored. This is a required argument. 56 | - `--sourceBucket`: Specify the name of the bucket where the object is stored. This is a required argument. 57 | - `--destClient`: Specify the name of the locally configued client where the object will be copied to. This is a required argument. 58 | - `--destBucket`: Specify the name of the bucket where the object will be copied to. This is a required argument. 59 | 60 | - `s3motion -R` or `s3motion --REST`: Start a webservice listening on port 8080 for REST based commands. 61 | 62 | ## REST Usage 63 | In addition to running this as a command line, it can also accept REST requests. To begin the microservice to accept REST commands use `s3motion -R`. You will see a message that says **s3motion microservice started on port 8080**. All requests must go through /api, for example `http://myserver.mycompany.com:8080/api`. All requests are returned with JSON. The default port used is always `:8080`. 64 | 65 | The following REST commands are available to you. 66 | 67 | - /api/clients 68 | - GET: Returns a JSON object with all clients configured in the s3motionClients.json object 69 | 70 | GET http://127.0.0.1:8080/api/clients 71 | Status: 200 OK 72 | JSON: 73 | { 74 | clients: [3] 75 | 0: { 76 | name: "vipronline" 77 | accessKeyId: "*******" 78 | secretAccessKey: "*********" 79 | endpoint: "object.vipronline.com" 80 | }- 81 | 1: { 82 | name: "aws" 83 | accessKeyId: "*******" 84 | secretAccessKey: "********" 85 | }- 86 | 2: {} 87 | } 88 | - POST: Creates a new client in the s3motionClients.json object. Payload requires `name`, `accessKeyId`, and `secretAccessKey` while `endpoint` is optional. 89 | 90 | POST http://127.0.0.1:8080/api/clients 91 | Status: 200 OK 92 | JSON: 93 | { 94 | operation: "newClient" 95 | client: "APItest" 96 | accessKeyId: "gniowrngosejbrjs90u390289r342nv" 97 | secretAccessKey: "fwefew7&^7@(*fec9#**vcuovcuyvwu" 98 | status: "success" 99 | } 100 | 101 | - /api/buckets/:client 102 | - GET: Returns a JSON object with all buckets for a client 103 | 104 | GET http://127.0.0.1:8080/api/buckets/vipronline 105 | Status: 200 OK 106 | JSON: 107 | { 108 | Buckets: [2] 109 | 0: { 110 | Name: "s3jump" 111 | CreationDate: "2015-01-30T16:59:28.026Z" 112 | } 113 | 1: { 114 | Name: "s3motion_vipr01" 115 | CreationDate: "2015-01-15T18:37:41.182Z" 116 | } 117 | Owner: { 118 | DisplayName: "user056" 119 | ID: "user056" 120 | } 121 | } 122 | - POST: Creates a new bucket for a client. Payload requires `name` for the bucket. 123 | 124 | POST http://127.0.0.1:8080/api/clients 125 | Status: 200 OK 126 | JSON: 127 | { 128 | Location: "/apitest" 129 | } 130 | 131 | - /api/bucket/copy 132 | - POST: Copies one bucket to an another in its entirety. Payload requires `sourceClient`, `sourceBucket`, `destClient`, and `destBucket`. You will never get a "success" status message as a response because depending on the amount of objects that must be copied over could take a while and the browser will timeout. 133 | 134 | POST http://127.0.0.1:8080/api/bucket/copy 135 | Status: 200 OK 136 | JSON: 137 | { 138 | operation: "copyBucket" 139 | sourceClient: "aws" 140 | sourceBucket: "s3motion01" 141 | destClient: "vipronline" 142 | destBucket: "s3motion_vipr01" 143 | status: "running" 144 | } 145 | 146 | - /api/objects/:client/:bucket 147 | - GET: Returns a JSON object with all objects in a bucket. Depending on the amount of objects, the browser may timeout. The timeout is currently set to 4 minutes for Express.js, but most browsers will timeout after 2 minutes. You will need a browser with a configurable timeout to wait for the objects to be collected. Anything greater than >100,000 objects will more than likely timeout after 2 minutes. 148 | 149 | GET http://127.0.0.1:8080/api/objects/vipronline/s3motion_vipr01 150 | Status: 200 OK 151 | JSON: 152 | [1] 153 | 0: [4] 154 | 0: { 155 | Key: "object1.ics" 156 | LastModified: "2015-02-05T15:30:28.846Z" 157 | ETag: ""699e612ef53db0c730ba2b809935509d"" 158 | Size: 3391 159 | StorageClass: "STANDARD" 160 | Owner: { 161 | DisplayName: "user056" 162 | ID: "user056" 163 | } 164 | } 165 | 1: { 166 | Key: "object2.xlsx" 167 | LastModified: "2015-02-05T15:30:29.459Z" 168 | ETag: ""fcb4a7c70c7c70df8484f0a44d34b22f"" 169 | Size: 26483 170 | StorageClass: "STANDARD" 171 | Owner: { 172 | DisplayName: "user056" 173 | ID: "user056" 174 | } 175 | } 176 | 2: { 177 | Key: "object3.xlsx" 178 | LastModified: "2015-02-05T15:30:28.927Z" 179 | ETag: ""ad6e381175f42a9b2b26f90a068a3823"" 180 | Size: 13428 181 | StorageClass: "STANDARD" 182 | Owner: { 183 | DisplayName: "user056" 184 | ID: "user056" 185 | } 186 | } 187 | 3: { 188 | Key: "object4.csv" 189 | LastModified: "2015-02-05T15:30:28.918Z" 190 | ETag: ""9a058b5b07578848b8d2406ded823d7e"" 191 | Size: 7792 192 | StorageClass: "STANDARD" 193 | Owner: { 194 | DisplayName: "user056" 195 | ID: "user056" 196 | } 197 | } 198 | - POST: Uploads an object to a specific bucket and client. Payload requires `object`. `object` can be in the form of comma seperated values. `folder` is an optional parameter to specify where on the host object system the object is located. By default, uploads will go to the root of the bucket. 199 | 200 | POST http://127.0.0.1:8080/api/objects/vipronline/s3motion_vipr01 201 | Status: 200 OK 202 | JSON: 203 | { 204 | operation: "objectUpload" 205 | objects: "object1.png,object2.jpg" 206 | folder: "/home/kcoleman" 207 | client: "vipronline" 208 | bucket: "s3motion_vipr01" 209 | status: "complete" 210 | } 211 | - DELETE: Deletes an object(s) in a specific bucket and client. Payload requires `object`. `object` can be in the form of comma seperated values. 212 | 213 | DELETE http://127.0.0.1:8080/api/objects/vipronline/s3motion_vipr01 214 | Status: 200 OK 215 | JSON: 216 | { 217 | operation: "objectDelete" 218 | objects: "object1.jpg,object2.gif" 219 | client: "vipronline" 220 | bucket: "s3motion_vipr01" 221 | status: "complete" 222 | } 223 | 224 | - /api/object/copy 225 | - POST: Copies an object(s) from one bucket to another. Payload requires `sourceClient`, `sourceBucket`, `destClient`, `destBucket`, and `object`. `object` can be in the form of comma seperated values. You will never get a "success" status message as a response because depending on the amount of objects that must be copied over could take a while and the browser will timeout. 226 | 227 | POST http://127.0.0.1:8080/api/bucket/copy 228 | Status: 200 OK 229 | JSON: 230 | { 231 | operation: "objectCopy" 232 | objects: "object1.jpg,object2.gif" 233 | sourceClient: "aws" 234 | sourceBucket: "s3motion01" 235 | destClient: "vipronline" 236 | destBucket: "s3motion01_vipr01" 237 | status: "running" 238 | } 239 | 240 | - /api/object/download 241 | - POST: Downloads an object(s) from one bucket to the host running the microservice. Payload requires `client`, `bucket`, and `object`. `object` can be in the form of comma seperated values. `folder` is an optional value to specify the download location on the host. 242 | 243 | POST http://127.0.0.1:8080/api/bucket/copy 244 | Status: 200 OK 245 | JSON: 246 | { 247 | operation: "downloadObject" 248 | object: "object.json" 249 | folder: "/home/user" 250 | client: "vipronline" 251 | bucket: "s3motion_vipr01" 252 | status: "complete" 253 | } 254 | 255 | ## Troubleshooting 256 | When using a 3rd party S3 storage service (not AWS) there are instances when the server returns a header without `content-length` and instead specifies `transfer-encoding: chunked`. 257 | ``` 258 | { date: 'Fri, 06 Feb 2015 20:18:09 GMT', 259 | server: 'ViPR/1.0', 260 | 'x-amz-request-id': '0a6c5fc3:14af4ae3238:f7f1:b', 261 | 'x-amz-id-2': '97e9f1ba70052a7b85e9db09cd13f09454828f6f0a5b92e5f691c07656a7ec10', 262 | etag: '"e1f0060d53cbfab7f917642abaa7a1c6"', 263 | 'last-modified': 'Fri, 06 Feb 2015 16:09:15 GMT', 264 | 'x-emc-mtime': '1423238955945', 265 | 'content-type': 'application/zip', 266 | 'transfer-encoding': 'chunked' } 267 | ``` 268 | This means the download process for a file will not happen because of how the [node-s3-client](https://github.com/andrewrk/node-s3-client) handles it. [Pull-request 76](https://github.com/andrewrk/node-s3-client/pull/76) will fix this behavior. 269 | 270 | ## Future 271 | - Add these functions depending on necessity 272 | - delete bucket including all objects (scary) 273 | - sync buckets between endpoints 274 | - upload object to a specific folder inside a bucket (just needs another param) 275 | - error messages for incorrect bucket spelling 276 | - bucket copy process don't specify destination Bucket, just copy the name of the source bucket. 277 | - bucket copy process needs to copy empty folders or create those entries on the destination bucket. 278 | - Continue Microservice using Express.js 279 | - Clean up the code 280 | - break out into multiple objects 281 | - Web front end 282 | - Create logging functionality 283 | 284 | ## Contribution 285 | - Fork it, merge it 286 | 287 | Licensing 288 | --------- 289 | Licensed under the Apache License, Version 2.0 (the “License”); you may not use this file except in compliance with the License. You may obtain a copy of the License at 290 | 291 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 292 | 293 | Support 294 | ------- 295 | Please file bugs and issues on the Github issues page for this project. This is to help keep track and document everything related to this repo. The code and documentation are released with no warranties or SLAs and are intended to be supported through a community driven process. 296 | -------------------------------------------------------------------------------- /clients.json: -------------------------------------------------------------------------------- 1 | {"clients":[ 2 | { 3 | } 4 | ] 5 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "s3motion", 3 | "version": "0.1.2", 4 | "description": "copy & move objects between s3 instances with CLI and REST", 5 | "main": "s3motion.js", 6 | "dependencies": { 7 | "async": "^0.9.0", 8 | "aws-sdk": "^2.1.5", 9 | "body-parser": "^1.11.0", 10 | "chalk": "^0.5.1", 11 | "commander": "^2.6.0", 12 | "connect-timeout": "^1.5.0", 13 | "express": "^4.11.2", 14 | "optimist": "^0.6.1", 15 | "osenv": "^0.1.0", 16 | "progress": "^1.1.8", 17 | "prompt": "^0.2.14", 18 | "s3": "^4.3.1" 19 | }, 20 | "devDependencies": {}, 21 | "scripts": { 22 | "test": "echo \"Error: no test specified\" && exit 1" 23 | }, 24 | "repository": { 25 | "type": "git", 26 | "url": "https://github.com/kacole2/s3motion.git" 27 | }, 28 | "keywords": [ 29 | "s3", 30 | "amazon", 31 | "vipr", 32 | "copy", 33 | "move", 34 | "kacole2", 35 | "emc", 36 | "transfer", 37 | "motion", 38 | "migrate", 39 | "migration", 40 | "object", 41 | "file", 42 | "riakcs", 43 | "ceph", 44 | "cleversafe", 45 | "rest" 46 | ], 47 | "author": "Kendrick Coleman", 48 | "license": "Apache 2.0", 49 | "preferGlobal": true, 50 | "bin": { 51 | "s3motion": "s3motion.js" 52 | }, 53 | "readmeFilename": "Readme.md", 54 | "bugs": { 55 | "url": "https://github.com/kacole2/s3motion/issues" 56 | }, 57 | "homepage": "https://github.com/kacole2/s3motion" 58 | } 59 | -------------------------------------------------------------------------------- /s3motion.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | 3 | var AWS = require('aws-sdk'), 4 | fs = require('fs'), 5 | s3 = require('s3'), 6 | async = require('async'), 7 | ProgressBar = require('progress'), 8 | chalk = require('chalk'), 9 | program = require('commander'), 10 | prompt = require('prompt'), 11 | optimist = require('optimist'), 12 | osenv = require('osenv'); 13 | 14 | //Functions that make all the magic happen 15 | //============================================================= 16 | // @description Returns a client object for the AWS SDK 17 | // @object awsClientArgs Paramters passed from outside functions. 18 | // @string accessKeyId Pipes in the access key for S3 access 19 | // @string secretAccessKey Pipes in the secret key for S3 access 20 | // @string endpoint Pipes in an endpoint for 3rd party S3 services 21 | // @return client object for AWS SDK 22 | var awsClient = function(awsClientArgs){ 23 | var client = new AWS.S3({accessKeyId: awsClientArgs.accessKeyId, secretAccessKey: awsClientArgs.secretAccessKey, endpoint: awsClientArgs.endpoint}); 24 | return client; 25 | } 26 | 27 | // @description Returns a client object for the S3 NPM package 28 | // @object s3ClientArgs Paramters passed from outside functions. 29 | // @string accessKeyId Pipes in the access key for S3 access 30 | // @string secretAccessKey Pipes in the secret key for S3 access 31 | // @string endpoint Pipes in an endpoint for 3rd party S3 services 32 | // @return client object for S3 NPM package 33 | var s3Client = function(s3ClientArgs) { 34 | var client = s3.createClient({ 35 | s3Options: { 36 | accessKeyId: s3ClientArgs.accessKeyId, 37 | secretAccessKey: s3ClientArgs.secretAccessKey, 38 | endpoint: s3ClientArgs.endpoint 39 | // any other options are passed to new AWS.S3() 40 | // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#constructor-property 41 | }, 42 | }); 43 | return client; 44 | } 45 | 46 | // @description Creates a new client json object 47 | // @object newClientArgs Paramters passed from outside functions. 48 | var newClient = function(newClientArgs, done){ 49 | var home = osenv.home(); //get the home directory for the current user 50 | // Two String functions needed to searching and inserting text to the json file 51 | String.prototype.insert = function (index, string) { 52 | if (index > 0) 53 | return this.substring(0, index) + string + this.substring(index, this.length); 54 | else 55 | return string + this; 56 | }; 57 | String.prototype.includes = function() {'use strict'; 58 | return String.prototype.indexOf.apply(this, arguments) !== -1; 59 | }; 60 | 61 | // Read the s3motionClients.json file to add a new client 62 | fs.readFile(home + '/s3motionClients.json', function (err, data) { //buffer file into memory 63 | if (err) { 64 | // if s3motionClients.json does not exist, create it and then continue 65 | console.log("creating 's3motionClients.json' file in " + home); 66 | fs.writeFileSync(home + '/s3motionClients.json', '{"clients":[\n\t{\n\t}\n\t]\n}'); 67 | var data = fs.readFileSync(home + '/s3motionClients.json'); 68 | } 69 | var content = data.toString('utf8'); //change buffer content to string for manipulation 70 | var contentJSON = JSON.parse(content); //parse the content to JSON 71 | var clients = contentJSON['clients']; //tab over to clients 72 | var clientExists = false; //set to false for error checking 73 | for (var i = 0; i < clients.length; i++) { //loop through all clients to see if the specified name exists 74 | if(clients[i]['name'] == newClientArgs.name) clientExists = true; //if it exists, set error handler to true 75 | } 76 | 77 | if(clientExists == true){ //if the error is true kill the process and send message to console 78 | done(chalk.yellow.bold(newClientArgs.name) + chalk.red(" already exists. Please use a different name or edit the 's3motionClients.json' file")); 79 | } else { 80 | if(typeof newClientArgs.endpoint === 'undefined'){ //if endpoint is undefined (AWS), remove it from the insertion 81 | content = content.insert(13, '\t{ \n\t\t"name": "' + newClientArgs.name + '",\n\t\t"accessKeyId": "' + newClientArgs.accessKeyId + '",\n\t\t"secretAccessKey": "' + newClientArgs.secretAccessKey + '"\n\t},\n'); 82 | } else { 83 | content = content.insert(13, '\t{ \n\t\t"name": "' + newClientArgs.name + '",\n\t\t"accessKeyId": "' + newClientArgs.accessKeyId + '",\n\t\t"secretAccessKey": "' + newClientArgs.secretAccessKey + '",\n\t\t"endpoint": "' + newClientArgs.endpoint + '"\n\t},\n'); 84 | }; 85 | fs.writeFile(home + '/s3motionClients.json', content, function (err) { //save the file with the new content 86 | if (err) throw err; 87 | done(chalk.yellow.bold(newClientArgs.name) + chalk.green.bold(" client created")); //deliver success message 88 | }); 89 | } 90 | }); 91 | } 92 | 93 | // @description Retrieve client from JSON file and return client 94 | // @object getClientArgs Paramters passed from outside functions. 95 | // @return client object 96 | var getClient = function(getClientArgs, done){ 97 | var home = osenv.home(); //get current home directory 98 | fs.readFile(home + '/s3motionClients.json', function (err, data) { //read the file into buffer 99 | if (err) throw err; //error if file isn't found. 100 | var clientExists = false; //set error handler to FALSE 101 | var clients = JSON.parse(data); //put buffer data to JSON 102 | clients = clients['clients']; //tab into JSON 103 | for (var i = 0; i < clients.length; i++) { //loop through all JSON objects and 104 | if(clients[i]['name'] == getClientArgs.name) { //find the matching name supplied from the user input 105 | var clientExists = true; //set error handler to TRUE 106 | if(getClientArgs.client == 's3'){ //run function depending on type of client needed 107 | var s3clientReturn = s3Client({accessKeyId: clients[i]['accessKeyId'], secretAccessKey: clients[i]['secretAccessKey'], endpoint: clients[i]['endpoint']}); 108 | done(s3clientReturn); 109 | } else if (getClientArgs.client == 'aws') { 110 | var awsClientReturn = awsClient({accessKeyId: clients[i]['accessKeyId'], secretAccessKey: clients[i]['secretAccessKey'], endpoint: clients[i]['endpoint']}); 111 | done(awsClientReturn) ; 112 | } 113 | } 114 | } 115 | // if the supplied client isn't found, let the user know. Using '' as a return for nothing 116 | if(clientExists == false) { 117 | done(''); 118 | } 119 | }); 120 | } 121 | 122 | // @description List all clients 123 | // @return JSON object 124 | var listClients = function(done){ 125 | var home = osenv.home(); 126 | fs.readFile(home + '/s3motionClients.json', function (err, data) { //read the file into buffer 127 | if (err) { 128 | done({message : 'Error: cannot open or find s3motionClients.json file in ' + home}); 129 | } else { 130 | var clients = JSON.parse(data); 131 | done(clients); //pass all clients as JSON object back 132 | } 133 | }); 134 | } 135 | 136 | // @description List all buckets for a client 137 | // @object clientArgs Paramters passed from outside functions. 138 | // @return JSON object if passed. console error message. 139 | var listBuckets = function(clientArgs, done){ 140 | clientArgs.site.listBuckets(function(err, data) { 141 | if (err) { 142 | done(chalk.red('Could not retrieve buckets. Error: ' + err)); // error is Response.error 143 | } else { 144 | done(data); // data is Response.data 145 | } 146 | }); 147 | } 148 | 149 | // @description Create new bucket for a client 150 | // @object newBucketParams Paramters passed from outside functions. 151 | // @return JSON object if passed. Sting on error message 152 | var newBucket = function(newBucketParams, done){ 153 | var params = { 154 | Bucket: newBucketParams.bucket //new bucket name is passed as a parameter 155 | }; 156 | newBucketParams.site.createBucket(params, function(err, data) { 157 | if (err) done(err, err.stack); // an error occurred 158 | else { 159 | done(data); 160 | } 161 | }); 162 | } 163 | 164 | // @description List objects in a bucket for a client 165 | // @object listArgs Paramters passed from outside functions. 166 | // @return Array 167 | var listObjects = function(listArgs, done) { 168 | var objectLists = []; //set the array to hold list of all Objects 169 | var objectCount = 0; //count how many objects are discovered 170 | var params = { 171 | s3Params: { 172 | Bucket: listArgs.bucket, //bucket name is passed as a parameter 173 | }, 174 | }; 175 | var lister = listArgs.site.listObjects(params); //specify site, passed as a listArgs param, to list bucket items 176 | lister.on('error', function(err) { 177 | done("unable to list " + listArgs.bucket + " : " + err.stack); 178 | //console.error(chalk.red("unable to list: " + chalk.red.bold(listArgs.bucket) + ""), chalk.yellow(err.stack)); 179 | }); 180 | lister.on('data', function(data) { //keep streaming data as its discovered 181 | var objects = data['Contents']; //drill down one layer in JSON 182 | objectCount += data['Contents'].length; //add the discovered amount to the Count 183 | objectLists.push(objects); //add this discovered list to the array 184 | console.log(chalk.blue('Gathering list of objects from ' + listArgs.bucket + '... Discovered ' + objectCount + ' objects so far')); 185 | }); 186 | lister.on('end', function(data) { 187 | done(objectLists); //pass the array full of JSON back to the callback 188 | }); 189 | } 190 | 191 | // @description Download an object or set of objects from a bucket for a client 192 | // @object downloadArgs Paramters passed from outside functions. 193 | // @return Array for progress and String on completion for each object download 194 | var downloadObject = function(downloadArgs, done) { 195 | var folder; //set the folder variable and run if/else if folder is not passed as a param to set a default. 196 | if(typeof downloadArgs.folder === 'undefined'){ 197 | folder = '' 198 | } else { 199 | var lastChar = downloadArgs.folder.substr(downloadArgs.folder.length - 1); 200 | if (lastChar == '/') { 201 | folder = downloadArgs.folder 202 | } else { 203 | folder = downloadArgs.folder + '/' 204 | } 205 | }; 206 | 207 | var downloadObjectProcess = function(downloadArgs, done){ 208 | var params = { 209 | //specify download location and the object name. no name changes are made to the object on the fly 210 | localFile: downloadArgs.folder + downloadArgs.object, 211 | 212 | s3Params: { 213 | Bucket: downloadArgs.bucket, //specify bucket to download from 214 | Key: downloadArgs.object, //specify name of object to download 215 | }, 216 | }; 217 | var downloader = downloadArgs.site.downloadFile(params); 218 | downloader.on('error', function(err) { 219 | done(chalk.red("Unable to download " + chalk.red.bold(downloadArgs.object) + ". Check bucket name and object name :"), chalk.yellow(err.stack)); 220 | }); 221 | downloader.on('progress', function() { 222 | done([downloadArgs.folder + downloadArgs.object, downloader.progressAmount, downloader.progressTotal]); 223 | }); 224 | downloader.on('end', function() { 225 | done(downloadArgs.object + chalk.green.bold(" downloaded")); 226 | }); 227 | 228 | } 229 | //Using CLI & REST, all objects are added to this function as an array. This 'if' statement is here in case this becomes 230 | //a npm package for inclusion in other projects where objects are not passed in as arrays from the original function 231 | //When objects are passed in as an array, only do 10 objects at a time until completed 232 | if (downloadArgs.object instanceof Array){ 233 | async.eachLimit(downloadArgs.object, 10, function(object, callback){ 234 | downloadObjectProcess({site: downloadArgs.site, bucket: downloadArgs.bucket, object: object, folder: folder}, function(data){ 235 | done(data); //send data to the done callback from the source function 236 | callback(); //send a callback response to the async callback. 237 | }); 238 | }); 239 | } else { 240 | downloadObjectProcess({site: downloadArgs.site, bucket: downloadArgs.bucket, object: downloadArgs.object, folder: folder}, function(data){ 241 | done(data); //send data to the done callback from the source function 242 | }); 243 | } 244 | 245 | 246 | } 247 | 248 | // @description Upload an object or set of objects from a local location for a client 249 | // @object uploadArgs Paramters passed from outside functions. 250 | // @return Array for progress and String on completion for each object upload 251 | var uploadObject = function(uploadArgs, done) { 252 | var folder; //set the folder variable and run if/else if folder is not passed as a param to set a default. 253 | if(typeof uploadArgs.folder === 'undefined'){ 254 | folder = '' 255 | } else { 256 | //add the trailing / if not specified in the folder param 257 | var lastChar = uploadArgs.folder.substr(uploadArgs.folder.length - 1); 258 | if (lastChar == '/') { 259 | folder = uploadArgs.folder 260 | } else { 261 | folder = uploadArgs.folder + '/' 262 | } 263 | }; 264 | var uploadObjectProcess = function(uploadArgs, done){ 265 | var params = { 266 | localFile: uploadArgs.folder + uploadArgs.object, //specifying upload location 267 | 268 | s3Params: { 269 | Bucket: uploadArgs.bucket, //specify bucket name to upload to 270 | Key: uploadArgs.object, //specify what the object will be called. In our case we are using same names. no changes 271 | }, 272 | }; 273 | var uploader = uploadArgs.site.uploadFile(params); 274 | var i = 0; //set variables for progress 275 | var l = 0; 276 | uploader.on('error', function(err) { 277 | done(chalk.red("unable to upload " + chalk.red.bold(uploadArgs.object) + ":"), chalk.yellow(err.stack)); 278 | }); 279 | uploader.on('progress', function() { 280 | //if statement to run because of an incorrect amount of callback returns causing errors on the progress bar 281 | // once the progressAmount equals the progressTotal then we don't need to send anymore progress data 282 | // data is returned as Array 283 | if (i != 0){ 284 | if(l == 0){ 285 | done([uploadArgs.folder + uploadArgs.object, uploader.progressAmount, uploader.progressTotal]); 286 | if(uploader.progressAmount == uploader.progressTotal){ 287 | l += 1; 288 | } 289 | } 290 | } 291 | i += 1; 292 | }); 293 | uploader.on('end', function() { 294 | //send success String 295 | done(chalk.green.bold(uploadArgs.object + " uploaded")); 296 | }); 297 | } 298 | //Using CLI & REST, all objects are added to this function as an array. This 'if' statement is here in case this becomes 299 | //a npm package for inclusion in other projects where objects are not passed in as arrays from the original function 300 | //When objects are passed in as an array, only do 10 objects at a time until completed 301 | if (uploadArgs.object instanceof Array){ 302 | async.eachLimit(uploadArgs.object, 10, function(object, callback){ 303 | uploadObjectProcess({site: uploadArgs.site, bucket: uploadArgs.bucket, object: object, folder: folder}, function(data){ 304 | done(data); //send data to the done callback from the source function 305 | callback(); //send a callback response to the async callback. 306 | }); 307 | }); 308 | } else { 309 | uploadObjectProcess({site: uploadArgs.site, bucket: uploadArgs.bucket, object: uploadArgs.object, folder: folder}, function(data){ 310 | done(data); 311 | }); 312 | } 313 | } 314 | 315 | // @description Copy an object or set of objects from an S3 bucket to another bucket and between clients 316 | // @object copyArgs Paramters passed from outside functions. 317 | // @return String on completion 318 | var copyObject = function(copyArgs, done) { 319 | var copyProcess = function(copyProcessArgs, done) { 320 | //create a progress bar to show the copy process 321 | var bar = new ProgressBar('copy: ' + copyProcessArgs.object + ' [:bar :title] ', { 322 | complete: chalk.green('='), 323 | incomplete: ' ', 324 | width: 30, 325 | total: 3 326 | }); 327 | //process goes to download object, upload object, and remove from local filesystem. must be sequential. 328 | async.series([ 329 | function(done) { 330 | bar.tick({ title: chalk.blue('downloading') }); 331 | downloadObject({site: copyProcessArgs.sourceSite, bucket: copyProcessArgs.sourceBucket, object: copyProcessArgs.object, folder: __dirname + '/s3motionTransfer/', transfer: true}, function(data){ 332 | if(data == copyProcessArgs.object + chalk.green.bold(" downloaded")) { 333 | done(); //async callback to go to the next 334 | } 335 | }); 336 | }, 337 | function(done) { 338 | bar.tick({ title: chalk.blue('uploading') }); 339 | uploadObject({site: copyProcessArgs.destinationSite, bucket: copyProcessArgs.destinationBucket, object: copyProcessArgs.object, folder: __dirname + '/s3motionTransfer/', transfer: true}, function(data){ 340 | if(data == chalk.green.bold(copyProcessArgs.object + " uploaded")) { 341 | done(); //async callback to go to the next 342 | } 343 | }); 344 | }, 345 | function(done) { 346 | fs.unlink(__dirname + '/s3motionTransfer/' + copyProcessArgs.object, function (err) { 347 | if (err) throw err; 348 | bar.tick({ title: chalk.cyan('complete') }); 349 | done(); //async callback to go to the next 350 | }); 351 | } 352 | ], 353 | function(err, results){ 354 | if (err) { 355 | done(chalk.red.bold('error occured: ') + chalk.yellow(err)); 356 | } 357 | done(copyProcessArgs.object + chalk.green('successfully copied')); 358 | }); 359 | } 360 | //Using CLI & REST, all objects are added to this function as an array. This 'if' statement is here in case this becomes 361 | //a npm package for inclusion in other projects where objects are not passed in as arrays from the original function 362 | //When objects are passed in as an array, only do 10 objects at a time until completed 363 | if (copyArgs.object instanceof Array){ 364 | async.eachLimit(copyArgs.object, 10, function(object, callback){ 365 | copyProcess({sourceSite: copyArgs.sourceSite, sourceBucket: copyArgs.sourceBucket, object: object, destinationSite: copyArgs.destinationSite, destinationBucket: copyArgs.destinationBucket}, function(data){ 366 | done(); //send data to the done callback from the source function 367 | callback(); //send a callback response to the async callback. 368 | }); 369 | }); 370 | } else { 371 | copyProcess({sourceSite: copyArgs.sourceSite, sourceBucket: copyArgs.sourceBucket, object: copyArgs.object, destinationSite: copyArgs.destinationSite, destinationBucket: copyArgs.destinationBucket}, function(data){ 372 | done(); 373 | }); 374 | } 375 | } 376 | 377 | // @description Delete an object or set of objects from an S3 bucket 378 | // @object deleteArgs Paramters passed from outside functions. 379 | // @return Array for progress and String on completion for each object upload 380 | var deleteObject = function(deleteArgs, done){ 381 | var objectsToDelete = []; //array syntax needed for multi-object deletion in s3param payload 382 | if (deleteArgs.object instanceof Array){ 383 | length = deleteArgs.object.length; 384 | for (var i = 0; i < length; i++) { 385 | var obj = {Key: deleteArgs.object[i],}; 386 | objectsToDelete.push(obj); 387 | } 388 | } else { 389 | var obj = {Key: deleteArgs.object,}; 390 | objectsToDelete.push(obj); 391 | } 392 | 393 | var s3params = { 394 | Bucket: deleteArgs.bucket, 395 | Delete: { 396 | Objects: 397 | objectsToDelete, 398 | }, 399 | }; 400 | 401 | var deleter = deleteArgs.site.deleteObjects(s3params); 402 | deleter.on('error', function(err) { 403 | done(chalk.red.bold("unable to delete:"), chalk.yellow(err.stack)); 404 | }); 405 | deleter.on('progress', function() { 406 | done([deleter.progressAmount, deleter.progressTotal]); 407 | }); 408 | deleter.on('end', function() { 409 | done(deleteArgs.object + chalk.red(" deleted")); 410 | }); 411 | } 412 | 413 | // @description Move an object or set of objects from an S3 bucket. Same as copy but calls the delete function in addition on the source 414 | // @object moveArgs Paramters passed from outside functions. 415 | // @return String on completion for each object upload 416 | var moveObject = function(moveArgs, done) { 417 | var moveProcess = function(moveProcessArgs, done) { 418 | var bar = new ProgressBar('move: ' + moveProcessArgs.object + ' [:bar :title] ', { 419 | complete: chalk.green('='), 420 | incomplete: ' ', 421 | width: 30, 422 | total: 5 423 | }); 424 | 425 | async.series([ 426 | function(done) { 427 | bar.tick({ title: chalk.blue('downloading') }); 428 | downloadObject({site: moveProcessArgs.sourceSite, bucket: moveProcessArgs.sourceBucket, object: moveProcessArgs.object, folder: __dirname + 's3motionTransfer/', transfer: true}, function(data){ 429 | if(data == moveProcessArgs.object + chalk.green.bold(" downloaded")) { 430 | done(); 431 | } 432 | }); 433 | }, 434 | function(done) { 435 | bar.tick({ title: chalk.blue('uploading') }); 436 | uploadObject({site: moveProcessArgs.destinationSite, bucket: moveProcessArgs.destinationBucket, object: moveProcessArgs.object, folder: __dirname + 's3motionTransfer/', transfer: true}, function(data){ 437 | if(data == chalk.green.bold(moveProcessArgs.object + " uploaded")) { 438 | done(); 439 | } 440 | }); 441 | }, 442 | function(done) { 443 | bar.tick({ title: chalk.blue('deleting locally') }); 444 | fs.unlink(__dirname + 's3motionTransfer/' + moveProcessArgs.object, function (err) { 445 | if (err) throw err; 446 | done(); 447 | }); 448 | }, 449 | function(done) { 450 | bar.tick({ title: chalk.blue('deleting from source' ) }); 451 | deleteObject({site: moveProcessArgs.sourceSite, bucket: moveProcessArgs.sourceBucket, object: moveProcessArgs.object, transfer: true}, function(data){ 452 | done(); 453 | }); 454 | }, 455 | function(done) { 456 | bar.tick({ title: chalk.cyan('complete') }); 457 | done(); 458 | } 459 | ], 460 | function(err, results){ 461 | if (err) { 462 | done(chalk.red.bold('error occured while moving ' + moveProcessArgs.object + ': ') + chalk.yellow(err)); 463 | } 464 | done(moveProcessArgs.object + chalk.green('successfully moved')); 465 | }); 466 | } 467 | //Using CLI & REST, all objects are added to this function as an array. This 'if' statement is here in case this becomes 468 | //a npm package for inclusion in other projects where objects are not passed in as arrays from the original function 469 | //When objects are passed in as an array, only do 10 objects at a time until completed 470 | if (moveArgs.object instanceof Array){ 471 | async.eachLimit(moveArgs.object, 10, function(object, callback){ 472 | moveProcess({sourceSite: moveArgs.sourceSite, sourceBucket: moveArgs.sourceBucket, object: object, destinationSite: moveArgs.destinationSite, destinationBucket: moveArgs.destinationBucket}, function(data){ 473 | done(); 474 | callback(); 475 | }); 476 | }); 477 | } else { 478 | moveProcess({sourceSite: moveArgs.sourceSite, sourceBucket: moveArgs.sourceBucket, object: moveArgs.object, destinationSite: moveArgs.destinationSite, destinationBucket: moveArgs.destinationBucket}, function(data){ 479 | done(); 480 | }); 481 | } 482 | } 483 | 484 | // @description Copy all object within a bucket to another bucket between. 485 | // @object copyArgs Paramters passed from outside functions. 486 | // @return String on completion for each object upload 487 | var copyBucket = function(copyArgs, done) { 488 | var objectLists = []; //array to hold all the list of objects gatehres from the lister 489 | var objectCount = 0; //iterating counter for all objects found 490 | var params = { 491 | s3Params: { 492 | Bucket: copyArgs.sourceBucket, 493 | }, 494 | }; 495 | 496 | var lister = copyArgs.sourceSite.listObjects(params); 497 | lister.on('error', function(err) { 498 | console.error(chalk.red.bold("unable to list:"), chalk.yellow(err.stack)); 499 | }); 500 | lister.on('data', function(data) { 501 | var objects = data['Contents']; //drill down one 502 | objectCount += data['Contents'].length; //add the amount of discovered objects to array 503 | objectLists.push(objects); //add the list to the array. this includes empty folders that will not transfer 504 | console.log(chalk.blue('Gathering list of objects from ' + copyArgs.sourceBucket + '... Discovered ' + objectCount + ' objects so far')); 505 | }); 506 | lister.on('end', function(data) { 507 | console.log(chalk.blue(objectCount + ' objects found. Beginning transfer:')); 508 | 509 | //the eachSeries call will do 1 objectList at a time and won't start the next until it has completed 510 | async.eachSeries(objectLists, 511 | function(objectList, callback){ 512 | var copyProcess = function(copyProcessArgs, done) { 513 | var bar = new ProgressBar('copy: ' + copyProcessArgs.object + ' [:bar :title] ', { 514 | complete: chalk.green('='), 515 | incomplete: ' ', 516 | width: 30, 517 | total: 3 518 | }); 519 | async.series([ 520 | function(done) { 521 | bar.tick({ title: chalk.blue('downloading') }); 522 | downloadObject({site: copyProcessArgs.sourceSite, bucket: copyProcessArgs.sourceBucket, object: copyProcessArgs.object, folder: __dirname + '/s3motionTransfer/', transfer: true}, function(data){ 523 | if(data == copyProcessArgs.object + chalk.green.bold(" downloaded")) { 524 | done(); 525 | } 526 | }); 527 | }, 528 | function(done) { 529 | bar.tick({ title: chalk.blue('uploading') }); 530 | uploadObject({site: copyProcessArgs.destinationSite, bucket: copyProcessArgs.destinationBucket, object: copyProcessArgs.object, folder: __dirname + '/s3motionTransfer/', transfer: true}, function(data){ 531 | if(data == chalk.green.bold(copyProcessArgs.object + " uploaded")) { 532 | done(); 533 | } 534 | }); 535 | }, 536 | function(done) { 537 | fs.unlink(__dirname + '/s3motionTransfer/' + copyProcessArgs.object, function (err) { 538 | if (err) throw err; 539 | bar.tick({ title: chalk.cyan('complete') }); 540 | done(); 541 | }); 542 | } 543 | ], 544 | function(err, results){ 545 | if (err) { 546 | done(chalk.red.bold('error occured with ' + copyProcessArgs.object + ': ') + chalk.yellow(err)); 547 | } 548 | done(copyProcessArgs.object + chalk.green('successfully copied')); 549 | }); 550 | } 551 | //Using CLI & REST, all objects are added to this function as an array. This 'if' statement is here in case this becomes 552 | //a npm package for inclusion in other projects where objects are not passed in as arrays from the original function 553 | //When objects are passed in as an array, only do 10 objects at a time until completed 554 | async.eachLimit(objectList, 10, 555 | function(object, callback){ 556 | copyProcess({sourceSite: copyArgs.sourceSite, sourceBucket: copyArgs.sourceBucket, object: object['Key'], destinationSite: copyArgs.destinationSite, destinationBucket: copyArgs.destinationBucket}, function(data){ 557 | done(); //send data to the done callback from the source function 558 | callback(); //send a callback response to the async callback. 559 | }); 560 | }, function(err){ 561 | 562 | } 563 | ); 564 | callback(); //send a callback response to the async callback to start the next List 565 | done(); //send data to the done callback from the source function 566 | }, function(err){ 567 | 568 | }); 569 | }); 570 | } 571 | 572 | // @description Starts a web service to accept REST requests 573 | var microservice = function() { 574 | // call the packages we need to run the web service 575 | var express = require('express'); // call express 576 | var app = express(); // define our app using express 577 | var timeout = require('connect-timeout');// set the timeout because waiting for object lists takes a while! 578 | var bodyParser = require('body-parser'); // read POST 579 | 580 | // configure app to use bodyParser() 581 | // this will let us get the data from a POST 582 | app.use(bodyParser.urlencoded({ extended: true })); 583 | app.use(bodyParser.json()); 584 | 585 | // set the application timeout to 5 minutes. 586 | app.use(timeout(300000)); 587 | app.use(haltOnTimedout); 588 | function haltOnTimedout(req, res, next){ 589 | if (!req.timedout) next(); 590 | } 591 | 592 | var port = process.env.PORT || 8080; // set our port 593 | 594 | // ROUTES FOR OUR API 595 | // ============================================================================= 596 | var router = express.Router(); // get an instance of the express Router 597 | 598 | // middleware to use for all requests 599 | router.use(function(req, res, next) { 600 | next(); // make sure we go to the next routes and don't stop here 601 | }); 602 | 603 | // test route to make sure everything is working (accessed at GET http://localhost:8080/api) 604 | router.get('/', function(req, res) { 605 | res.json({ message: 'This is the API. Go read the documentation on how to use it' }); 606 | }); 607 | 608 | //creating a psuedo-model for clients 609 | router.route('/clients') 610 | .post(function(req, res) { 611 | if (req.body.name == undefined || req.body.name == '' || req.body.accessKeyId == undefined || req.body.accessKeyId == '' || req.body.secretAccessKey == undefined || req.body.secretAccessKey == '') { 612 | res.status(400) 613 | .json({ 614 | operation: 'newClient', 615 | client: req.params.client, 616 | status: 'fail', 617 | message: 'Incorrect Payload params. "name", "accessKeyId", and "secretAccessKey" are required', 618 | }); 619 | } else { 620 | newClient({name: req.body.name, accessKeyId: req.body.accessKeyId, secretAccessKey: req.body.secretAccessKey, endpoint: req.body.endpoint}, function(data){ 621 | console.log(data); 622 | if (data == chalk.yellow.bold(req.body.name) + chalk.green.bold(" client created")){ 623 | res.json({ 624 | operation: 'newClient', 625 | client: req.body.name, 626 | accessKeyId: req.body.accessKeyId, 627 | secretAccessKey: req.body.secretAccessKey, 628 | endpoint: req.body.endpoint, 629 | status: 'success', 630 | }); 631 | } else if (data == chalk.yellow.bold(req.body.name) + chalk.red(" already exists. Please use a different name or edit the 's3motionClients.json' file")){ 632 | res.json({ 633 | operation: 'newClient', 634 | client: req.body.name, 635 | accessKeyId: req.body.accessKeyId, 636 | secretAccessKey: req.body.secretAccessKey, 637 | endpoint: req.body.endpoint, 638 | status: 'fail', 639 | message: req.body.name + ' already exists in s3motionClients.json' 640 | }); 641 | } 642 | }); 643 | } 644 | }) 645 | .get(function(req, res) { 646 | listClients(function(data){ 647 | res.json(data); 648 | }); 649 | }); 650 | //creating a psuedo-model for buckets 651 | router.route('/buckets/:client') 652 | .post(function(req, res) { 653 | if (req.body.name == undefined || req.body.name == '') { 654 | res.status(400) 655 | .json({ 656 | operation: 'newBucket', 657 | client: req.params.client, 658 | status: 'fail', 659 | message: '"name" value not passed as payload. Please specify a "name" for the new bucket', 660 | }); 661 | } else { 662 | var name = req.body.name; 663 | getClient({name: req.params.client, client: 'aws'}, function(client){ 664 | if (client == '') { 665 | res.status(404) 666 | .json({ 667 | operation: 'newBucket', 668 | client: req.params.client, 669 | status: 'fail', 670 | message: client + ' not found in s3motionClients.json. Create it using /clients' 671 | }); 672 | } else { 673 | newBucket({site: client, bucket: name}, function(data) { 674 | res.json({ 675 | operation: 'newBucket', 676 | client: req.params.client, 677 | status: 'complete', 678 | message: data 679 | }); 680 | }); 681 | } 682 | }); 683 | } 684 | }) 685 | .get(function(req, res) { 686 | getClient({name: req.params.client, client: 'aws'}, function(client){ 687 | if (client == '') { 688 | res.status(404) 689 | .json({ 690 | operation: 'listBuckets', 691 | client: req.params.client, 692 | status: 'fail', 693 | message: client + ' not found in s3motionClients.json. Create it using /clients' 694 | }); 695 | } else { 696 | listBuckets({site: client}, function(data) { 697 | res.json(data); 698 | }); 699 | } 700 | }); 701 | }); 702 | //not even a model, just a generic POST call for bucket copy operations 703 | router.route('/bucket/copy') 704 | .post(function(req, res) { 705 | if (req.body.sourceClient == undefined || req.body.sourceClient == '' || req.body.destClient == undefined || req.body.destClient == '' || req.body.sourceBucket == undefined || req.body.sourceBucket == '' || req.body.destBucket == undefined || req.body.destBucket == '') { 706 | res.status(400) 707 | .json({ 708 | operation: 'copyBucket', 709 | client: req.params.client, 710 | status: 'fail', 711 | message: 'Incorrect Payload params. "sourceClient", "sourceBucket", "destClient" and "destBucket" are required', 712 | }); 713 | } else { 714 | getClient({name: req.body.sourceClient, client: 's3'}, function(sclient){ 715 | if (sclient == '') { 716 | res.status(404) 717 | .json({ 718 | operation: 'copyBucket', 719 | sourceClient: req.body.sourceClient, 720 | sourceBucket: req.body.sourceBucket, 721 | destClient: req.body.destClient, 722 | destBucket: req.body.destBucket, 723 | status: 'fail', 724 | message: req.body.sourceClient + ' not found in s3motionClients.json. Create it using /clients' 725 | }); 726 | } 727 | var sourceClient = sclient; 728 | getClient({name: req.body.destClient, client: 's3'}, function(dclient){ 729 | if (dclient == '') { 730 | res.status(404) 731 | .json({ 732 | operation: 'copyBucket', 733 | sourceClient: req.body.sourceClient, 734 | sourceBucket: req.body.sourceBucket, 735 | destClient: req.body.destClient, 736 | destBucket: req.body.destBucket, 737 | status: 'fail', 738 | message: req.body.destClient + ' not found in s3motionClients.json. Create it using /clients' 739 | }); 740 | } 741 | var destClient = dclient 742 | copyBucket({sourceSite: sourceClient, sourceBucket: req.body.sourceBucket, destinationSite: destClient, destinationBucket: req.body.destBucket}, function(data) { 743 | 744 | }); 745 | res.json({ 746 | operation: 'copyBucket', 747 | sourceClient: req.body.sourceClient, 748 | sourceBucket: req.body.sourceBucket, 749 | destClient: req.body.destClient, 750 | destBucket: req.body.destBucket, 751 | status: 'running', 752 | }); 753 | }); 754 | }); 755 | } 756 | }); 757 | //creating a psuedo-model for objects 758 | router.route('/objects/:client/:bucket') 759 | .post(function(req, res) { 760 | if (req.body.object == undefined || req.body.object == '') { 761 | res.status(400) 762 | .json({ 763 | operation: 'objectUpload', 764 | client: req.params.client, 765 | status: 'fail', 766 | message: 'Incorrect Payload params. "object" is required', 767 | }); 768 | } else { 769 | var folder = req.body.folder; 770 | var objects = req.body.object.split(','); 771 | var objectsArrayLength = objects.length - 1; 772 | getClient({name: req.params.client, client: 's3'}, function(client){ 773 | if (client == '') { 774 | res.status(404) 775 | .json({ 776 | operation: 'objectUpload', 777 | objects: req.body.object, 778 | client: req.params.client, 779 | bucket: req.params.bucket, 780 | status: 'fail', 781 | message: req.params.client + ' not found in s3motionClients.json. Create it using /clients' 782 | }); 783 | } else { 784 | if(folder == ''){ 785 | folder = undefined; 786 | } 787 | uploadObject({site: client, bucket: req.params.bucket, object: objects, folder: folder}, function(data) { 788 | if (data instanceof Array){ 789 | // 790 | } else if (data == chalk.green.bold(objects[objectsArrayLength] + " uploaded")) { 791 | res.json({ 792 | operation: 'objectUpload', 793 | objects: req.body.object, 794 | folder: req.body.folder, 795 | client: req.params.client, 796 | bucket: req.params.bucket, 797 | status: 'complete' 798 | }); 799 | } 800 | }); 801 | } 802 | }); 803 | } 804 | }) 805 | .get(function(req, res) { 806 | getClient({name: req.params.client, client: 's3'}, function(client){ 807 | if (client == '') { 808 | res.status(404) 809 | .json({ 810 | operation: 'objectList', 811 | client: req.params.client, 812 | bucket: req.params.bucket, 813 | status: 'fail', 814 | message: client + ' not found in s3motionClients.json. Create it using /clients' 815 | }); 816 | } else { 817 | listObjects({site: client, bucket: req.params.bucket}, function(data) { 818 | res.json(data); 819 | }); 820 | } 821 | }); 822 | }) 823 | .delete(function(req, res) { 824 | if (req.body.object == undefined || req.body.object == '') { 825 | res.status(400) 826 | .json({ 827 | operation: 'objectUpload', 828 | client: req.params.client, 829 | bucket: req.params.bucket, 830 | status: 'fail', 831 | message: 'Incorrect Payload params. "object" is required', 832 | }); 833 | } else { 834 | var objects = req.body.object.split(','); 835 | getClient({name: req.params.client, client: 's3'}, function(client){ 836 | if (client == '') { 837 | res.status(404) 838 | .json({ 839 | operation: 'objectDelete', 840 | objects: objects.toString(), 841 | client: req.params.client, 842 | bucket: req.params.bucket, 843 | status: 'fail', 844 | message: client + ' not found in s3motionClients.json. Create it using /clients' 845 | }); 846 | } else { 847 | deleteObject({site: client, bucket: req.params.bucket, object: objects}, function(data) { 848 | if (data instanceof Array){ 849 | // 850 | } else { 851 | res.json({ 852 | operation: 'objectDelete', 853 | objects: objects.toString(), 854 | client: req.params.client, 855 | bucket: req.params.bucket, 856 | status: 'complete' 857 | }); 858 | } 859 | }); 860 | } 861 | }); 862 | } 863 | }); 864 | //not even a model, just a generic POST call for object copy operations 865 | router.route('/object/copy') 866 | .post(function(req, res) { 867 | var operation; 868 | if (req.body.delete == 'Y'){ 869 | operation = 'objectMove'; 870 | } else { 871 | operation = 'objectCopy'; 872 | } 873 | if (req.body.sourceClient == undefined || req.body.sourceClient == '' || req.body.sourceBucket == undefined || req.body.sourceBucket == '' || req.body.destClient == undefined || req.body.destClient == '' || req.body.destBucket == undefined || req.body.destBucket == '' || req.body.object == undefined || req.body.object == '') { 874 | res.status(400) 875 | .json({ 876 | operation: operation, 877 | status: 'fail', 878 | message: 'Incorrect Payload params. "sourceClient", "sourceBucket", "destClient", "destBucket", "object" is required', 879 | }); 880 | } else { 881 | getClient({name: req.body.sourceClient, client: 's3'}, function(sclient){ 882 | if (sclient == '') { 883 | res.status(404) 884 | .json({ 885 | operation: operation, 886 | objects: req.body.object, 887 | sourceClient: req.body.sourceClient, 888 | sourceBucket: req.body.sourceBucket, 889 | destClient: req.body.destClient, 890 | destBucket: req.body.destBucket, 891 | status: 'fail', 892 | message: req.body.sourceClient + ' not found in s3motionClients.json. Create it using /clients' 893 | }); 894 | } else { 895 | var sourceClient = sclient; 896 | getClient({name: req.body.destClient, client: 's3'}, function(dclient){ 897 | if (dclient == '') { 898 | res.status(404) 899 | .json({ 900 | operation: operation, 901 | objects: req.body.object, 902 | sourceClient: req.body.sourceClient, 903 | sourceBucket: req.body.sourceBucket, 904 | destClient: req.body.destClient, 905 | destBucket: req.body.destBucket, 906 | status: 'fail', 907 | message: req.body.destClient + ' not found in s3motionClients.json. Create it using /clients' 908 | }); 909 | } else { 910 | var destClient = dclient 911 | var objects = req.body.object.split(','); 912 | if (req.body.delete == 'Y'){ 913 | moveObject({sourceSite: sourceClient, sourceBucket: req.body.sourceBucket, object: objects, destinationSite: destClient, destinationBucket: req.body.destBucket}, function(data) { 914 | //console.log(data); 915 | }); 916 | } else { 917 | copyObject({sourceSite: sourceClient, sourceBucket: req.body.sourceBucket, object: objects, destinationSite: destClient, destinationBucket: req.body.destBucket}, function(data) { 918 | //console.log(data); 919 | }); 920 | } 921 | res.json({ 922 | operation: operation, 923 | objects: objects.toString(), 924 | sourceClient: req.body.sourceClient, 925 | sourceBucket: req.body.sourceBucket, 926 | destClient: req.body.destClient, 927 | destBucket: req.body.destBucket, 928 | status: 'running' 929 | }); 930 | } 931 | }); 932 | } 933 | }); 934 | } 935 | }); 936 | router.route('/object/download') 937 | .post(function(req, res) { 938 | if (req.body.client == undefined || req.body.client == '' || req.body.bucket == undefined || req.body.bucket == '' || req.body.object == undefined || req.body.object == '') { 939 | res.status(400) 940 | .json({ 941 | operation: 'downloadObject', 942 | status: 'fail', 943 | message: 'Incorrect Payload params. "client", "bucket", and "object" are required', 944 | }); 945 | } else { 946 | getClient({name: req.body.client, client: 's3'}, function(client){ 947 | if (client == '') { 948 | res.status(404) 949 | .json({ 950 | operation: 'downloadObject', 951 | object: req.body.object, 952 | folder: req.body.folder, 953 | client: req.body.client, 954 | bucket: req.body.bucket, 955 | status: 'fail', 956 | message: req.body.client + ' not found in s3motionClients.json. Create it using /clients' 957 | }); 958 | } else { 959 | if(req.body.folder == ''){ 960 | req.body.folder = undefined; 961 | } 962 | var objects = req.body.object.split(','); 963 | var objectsArrayLength = objects.length - 1; 964 | downloadObject({site: client, bucket: req.body.bucket, object: objects, folder: req.body.folder}, function(data) { 965 | if (data instanceof Array){ 966 | // 967 | } else if (data == objects[objectsArrayLength] + chalk.green.bold(" downloaded")) { 968 | res.json({ 969 | operation: 'downloadObject', 970 | object: req.body.object, 971 | folder: req.body.folder, 972 | client: req.body.client, 973 | bucket: req.body.bucket, 974 | status: 'complete' 975 | }); 976 | } 977 | }); 978 | } 979 | }); 980 | } 981 | }); 982 | 983 | // REGISTER OUR ROUTES ------------------------------- 984 | // all of our routes will be prefixed with /api 985 | app.use('/api', router); 986 | 987 | // START THE SERVER 988 | // ============================================================================= 989 | app.listen(port); 990 | console.log('s3motion microservice started on port ' + port); 991 | 992 | } 993 | 994 | // @description Commander for using s3motion as a Command Line (CLI) tool 995 | program 996 | .version('0.1.0') 997 | .usage('-flag Supply only 1 flag. Args will vary based on flag. To use the wizard, type the flag and "wizard" (ie. s3motion -n wizard)') 998 | .option('-n, --newClient <--name --accessKeyId --secretAccessKey --endpoint>', 'Add a New Client (will be stored in clients.json)') 999 | .option('-L, --listClients', 'List clients') 1000 | .option('-b, --listBuckets <--client>', 'List buckets for a specific client') 1001 | .option('-N, --newBucket <--client --name>', 'Create a new bucket') 1002 | .option('-l, --listObjects <--client --bucket>', 'List objects in a bucket') 1003 | .option('-d, --downloadObject <--client --bucket --object --folder>', 'Download object(s) from bucket. Multiple object download supported by using commas and no spaces.') 1004 | .option('-u, --uploadObject <--client --bucket --object --folder>', 'Upload object(s) to bucket. Multiple object upload supported by using commas and no spaces.') 1005 | .option('-D, --deleteObject <--client --bucket --object>', 'Delete object(s) from bucket. Multiple deletion supported by using commas and no spaces.') 1006 | .option('-c, --copyObject <--sourceClient --sourceBucket --object --destClient --destBucket --delete>', 'Copy object(s) between buckets. If --delete is Y, then source object is deleted after copy.') 1007 | .option('-C, --copyBucket <--sourceClient --sourceBucket --destClient --destBucket>', 'Copy objects between buckets') 1008 | .option('-R, --REST', 'Starts the REST based Web Service on port 8080') 1009 | .parse(process.argv); 1010 | 1011 | prompt.message = ""; 1012 | prompt.delimiter = ""; 1013 | prompt.colors = false; 1014 | prompt.override = optimist.argv; 1015 | 1016 | if (program.newClient) { 1017 | var questions = { 1018 | properties: { 1019 | name: { 1020 | description: 'Name: ', 1021 | required: true 1022 | }, 1023 | accessKeyId: { 1024 | description: 'Access Key: ', 1025 | required: true 1026 | }, 1027 | secretAccessKey: { 1028 | description: 'Secret Access Key: ', 1029 | required: true 1030 | }, 1031 | endpoint: { 1032 | description: 'Endpoint (leave blank for AWS): ', 1033 | required: false 1034 | } 1035 | } 1036 | }; 1037 | prompt.start(); 1038 | console.log('\n\n' + chalk.cyan('+----------------------+\n| |\n| CREATE NEW S3 CLIENT |\n| |\n+----------------------+') + '\n\n'); 1039 | prompt.get(questions, function (err, result) { 1040 | if (err) { 1041 | console.log('\n' + chalk.red(err)); 1042 | } else { 1043 | if(result.endpoint== ''){ 1044 | result.endpoint = undefined; 1045 | } 1046 | newClient({name: result.name, accessKeyId: result.accessKeyId, secretAccessKey: result.secretAccessKey, endpoint: result.endpoint}, function(data){ 1047 | console.log(data); 1048 | process.exit; 1049 | }); 1050 | } 1051 | }); 1052 | } 1053 | 1054 | if (program.listClients) { 1055 | console.log('\n\n' + chalk.cyan('+----------------------+\n| |\n| LIST CLIENTS |\n| |\n+----------------------+') + '\n\n'); 1056 | listClients(function(data){ 1057 | console.log(data); 1058 | }); 1059 | } 1060 | 1061 | if (program.listBuckets) { 1062 | var questions = { 1063 | properties: { 1064 | client: { 1065 | description: 'Client: ', 1066 | required: true 1067 | } 1068 | } 1069 | }; 1070 | prompt.start(); 1071 | console.log('\n\n' + chalk.cyan('+----------------------+\n| |\n| LIST BUCKETS |\n| |\n+----------------------+') + '\n\n'); 1072 | prompt.get(questions, function (err, result) { 1073 | if (err) { 1074 | console.log('\n' + chalk.red(err)); 1075 | } else { 1076 | getClient({name: result.client, client: 'aws'}, function(client){ 1077 | if (client == '') { 1078 | console.log(chalk.yellow(result.client) + chalk.red(" not found in s3motionClients.json. Create it using 's3motion -n wizard'")); 1079 | process.exit(1); 1080 | } 1081 | listBuckets({site: client}, function(data) { 1082 | console.log(data); 1083 | }); 1084 | }); 1085 | } 1086 | }); 1087 | } 1088 | 1089 | if (program.newBucket) { 1090 | var questions = { 1091 | properties: { 1092 | client: { 1093 | description: 'Client: ', 1094 | required: true 1095 | }, 1096 | name: { 1097 | description: 'New Bucket Name: ', 1098 | required: true 1099 | } 1100 | } 1101 | }; 1102 | prompt.start(); 1103 | console.log('\n\n' + chalk.cyan('+----------------------+\n| |\n| NEW BUCKET |\n| |\n+----------------------+') + '\n\n'); 1104 | prompt.get(questions, function (err, result) { 1105 | if (err) { 1106 | console.log('\n' + chalk.red(err)); 1107 | } else { 1108 | getClient({name: result.client, client: 'aws'}, function(client){ 1109 | if (client == '') { 1110 | console.log(chalk.yellow(result.client) + chalk.red(" not found in s3motionClients.json. Create it using 's3motion -n wizard'")); 1111 | process.exit(1); 1112 | } 1113 | newBucket({site: client, bucket: result.name}, function(data) { 1114 | data = data.toString(); 1115 | if (data == 'BucketAlreadyExists: The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again.'){ 1116 | console.log(chalk.red(data)); 1117 | } else { 1118 | console.log(chalk.green('Bucket successfully created: ') + '/' + result.name); 1119 | } 1120 | }); 1121 | }); 1122 | } 1123 | }); 1124 | } 1125 | 1126 | if (program.listObjects) { 1127 | var questions = { 1128 | properties: { 1129 | client: { 1130 | description: 'Client: ', 1131 | required: true 1132 | }, 1133 | bucket: { 1134 | description: 'Bucket: ', 1135 | required: true 1136 | } 1137 | } 1138 | }; 1139 | prompt.start(); 1140 | console.log('\n\n' + chalk.cyan('+----------------------+\n| |\n| LIST OBJECTS |\n| |\n+----------------------+') + '\n\n'); 1141 | prompt.get(questions, function (err, result) { 1142 | if (err) { 1143 | console.log('\n' + chalk.red(err)); 1144 | } else { 1145 | getClient({name: result.client, client: 's3'}, function(client){ 1146 | if (client == '') { 1147 | console.log(chalk.yellow(result.client) + chalk.red(" not found in s3motionClients.json. Create it using 's3motion -n wizard'")); 1148 | process.exit(1); 1149 | } 1150 | listObjects({site: client, bucket: result.bucket}, function(data) { 1151 | console.log(data); 1152 | }); 1153 | }); 1154 | } 1155 | }); 1156 | 1157 | } 1158 | 1159 | if (program.downloadObject) { 1160 | var questions = { 1161 | properties: { 1162 | client: { 1163 | description: 'Client: ', 1164 | required: true 1165 | }, 1166 | bucket: { 1167 | description: 'Bucket: ', 1168 | required: true 1169 | }, 1170 | object: { 1171 | description: "Object(s) (seperate with comma ',' and no spaces): ", 1172 | required: true 1173 | }, 1174 | folder: { 1175 | description: "Download location (optional, default is current directory): ", 1176 | required: false 1177 | } 1178 | } 1179 | }; 1180 | prompt.start(); 1181 | console.log('\n\n' + chalk.cyan('+----------------------+\n| |\n| DOWNLOAD OBJECT |\n| |\n+----------------------+') + '\n\n'); 1182 | prompt.get(questions, function (err, result) { 1183 | if (err) { 1184 | console.log('\n' + chalk.red(err)); 1185 | } else { 1186 | getClient({name: result.client, client: 's3'}, function(client){ 1187 | if (client == '') { 1188 | console.log(chalk.yellow(result.client) + chalk.red(" not found in s3motionClients.json. Create it using 's3motion -n wizard'")); 1189 | process.exit(1); 1190 | } 1191 | if(result.folder == ''){ 1192 | result.folder = undefined; 1193 | } 1194 | var objects = result.object.split(','); 1195 | downloadObject({site: client, bucket: result.bucket, object: objects, folder: result.folder}, function(data) { 1196 | if (data instanceof Array){ 1197 | var bar = new ProgressBar('downloading ' + chalk.blue(data[0]) + ' [:bar] ' + chalk.cyan(':percent'), { 1198 | complete: chalk.green('='), 1199 | incomplete: ' ', 1200 | width: 30, 1201 | total: data[2] 1202 | }); 1203 | bar.tick(data[1]); 1204 | } else { 1205 | //console.log(data); 1206 | } 1207 | }); 1208 | }); 1209 | } 1210 | }); 1211 | } 1212 | 1213 | if (program.uploadObject) { 1214 | var questions = { 1215 | properties: { 1216 | client: { 1217 | description: 'Client: ', 1218 | required: true 1219 | }, 1220 | bucket: { 1221 | description: 'Bucket: ', 1222 | required: true 1223 | }, 1224 | object: { 1225 | description: "Object(s) (seperate with comma ',' and no spaces): ", 1226 | required: true 1227 | }, 1228 | folder: { 1229 | description: "Upload location (optional, default is current directory): ", 1230 | required: false 1231 | } 1232 | } 1233 | }; 1234 | prompt.start(); 1235 | console.log('\n\n' + chalk.cyan('+----------------------+\n| |\n| UPLOAD OBJECT |\n| |\n+----------------------+') + '\n\n'); 1236 | prompt.get(questions, function (err, result) { 1237 | if (err) { 1238 | console.log('\n' + chalk.red(err)); 1239 | } else { 1240 | getClient({name: result.client, client: 's3'}, function(client){ 1241 | if (client == '') { 1242 | console.log(chalk.yellow(result.client) + chalk.red(" not found in s3motionClients.json. Create it using 's3motion -n wizard'")); 1243 | process.exit(1); 1244 | } 1245 | if(result.folder == ''){ 1246 | result.folder = undefined; 1247 | } 1248 | var objects = result.object.split(','); 1249 | uploadObject({site: client, bucket: result.bucket, object: objects, folder: result.folder}, function(data) { 1250 | if (data instanceof Array){ 1251 | var bar = new ProgressBar('uploading ' + chalk.blue(data[0]) + ' [:bar] ' + chalk.cyan(':percent'), { 1252 | complete: chalk.green('='), 1253 | incomplete: ' ', 1254 | width: 30, 1255 | total: data[2] 1256 | }); 1257 | bar.tick(data[1]); 1258 | } else { 1259 | //console.log(data); 1260 | } 1261 | }); 1262 | }); 1263 | } 1264 | }); 1265 | } 1266 | 1267 | if (program.deleteObject) { 1268 | var questions = { 1269 | properties: { 1270 | client: { 1271 | description: 'Client: ', 1272 | required: true 1273 | }, 1274 | bucket: { 1275 | description: 'Bucket: ', 1276 | required: true 1277 | }, 1278 | object: { 1279 | description: "Object(s) (seperate with comma ',' and no spaces): ", 1280 | required: true 1281 | } 1282 | } 1283 | }; 1284 | prompt.start(); 1285 | console.log('\n\n' + chalk.cyan('+----------------------+\n| |\n| DELETE OBJECT |\n| |\n+----------------------+') + '\n\n'); 1286 | prompt.get(questions, function (err, result) { 1287 | if (err) { 1288 | console.log('\n' + chalk.red(err)); 1289 | } else { 1290 | var object = result.object.split(','); 1291 | getClient({name: result.client, client: 's3'}, function(client){ 1292 | if (client == '') { 1293 | console.log(chalk.yellow(result.client) + chalk.red(" not found in s3motionClients.json. Create it using 's3motion -n wizard'")); 1294 | process.exit(1); 1295 | } 1296 | deleteObject({site: client, bucket: result.bucket, object: object}, function(data) { 1297 | if (data instanceof Array){ 1298 | var bar = new ProgressBar('deleting [:bar] ' + chalk.cyan(':percent'), { 1299 | complete: chalk.green('='), 1300 | incomplete: ' ', 1301 | width: 30, 1302 | total: data[1] 1303 | }); 1304 | bar.tick(data[0]); 1305 | } else { 1306 | console.log(data); 1307 | } 1308 | }); 1309 | }); 1310 | } 1311 | }); 1312 | } 1313 | 1314 | if (program.copyObject) { 1315 | var questions = { 1316 | properties: { 1317 | sourceClient: { 1318 | description: 'Source Client: ', 1319 | required: true 1320 | }, 1321 | sourceBucket: { 1322 | description: 'Source Bucket: ', 1323 | required: true 1324 | }, 1325 | object: { 1326 | description: "Object(s) (seperate with comma ',' and no spaces): ", 1327 | required: true 1328 | }, 1329 | destClient: { 1330 | description: 'Destination Client: ', 1331 | required: true 1332 | }, 1333 | destBucket: { 1334 | description: 'Destination Bucket: ', 1335 | required: true 1336 | }, 1337 | delete: { 1338 | description: 'Delete source object after copy? (default "n") (Y/n): ', 1339 | default: 'n', 1340 | required: false 1341 | } 1342 | } 1343 | }; 1344 | prompt.start(); 1345 | console.log('\n\n' + chalk.cyan('+----------------------+\n| |\n| COPY OBJECT |\n| |\n+----------------------+') + '\n\n'); 1346 | prompt.get(questions, function (err, result) { 1347 | if (err) { 1348 | console.log('\n' + chalk.red(err)); 1349 | } else { 1350 | getClient({name: result.sourceClient, client: 's3'}, function(sclient){ 1351 | var sourceClient = sclient; 1352 | 1353 | getClient({name: result.destClient, client: 's3'}, function(dclient){ 1354 | var destClient = dclient 1355 | var objects = result.object.split(','); 1356 | if (result.delete == 'Y'){ 1357 | moveObject({sourceSite: sourceClient, sourceBucket: result.sourceBucket, object: objects, destinationSite: destClient, destinationBucket: result.destBucket}, function(data) { 1358 | //console.log(data); 1359 | }); 1360 | } else { 1361 | copyObject({sourceSite: sourceClient, sourceBucket: result.sourceBucket, object: objects, destinationSite: destClient, destinationBucket: result.destBucket}, function(data) { 1362 | //console.log(data); 1363 | }); 1364 | } 1365 | }); 1366 | }); 1367 | } 1368 | }); 1369 | } 1370 | 1371 | if (program.copyBucket) { 1372 | var questions = { 1373 | properties: { 1374 | sourceClient: { 1375 | description: 'Source Client: ', 1376 | required: true 1377 | }, 1378 | sourceBucket: { 1379 | description: 'Source Bucket: ', 1380 | required: true 1381 | }, 1382 | destClient: { 1383 | description: 'Destination Client: ', 1384 | required: true 1385 | }, 1386 | destBucket: { 1387 | description: 'Destination Bucket: ', 1388 | required: true 1389 | } 1390 | } 1391 | }; 1392 | prompt.start(); 1393 | console.log('\n\n' + chalk.cyan('+----------------------+\n| |\n| COPY BUCKET |\n| |\n+----------------------+') + '\n\n'); 1394 | prompt.get(questions, function (err, result) { 1395 | if (err) { 1396 | console.log('\n' + chalk.red(err)); 1397 | } else { 1398 | getClient({name: result.sourceClient, client: 's3'}, function(sclient){ 1399 | if (sclient == '') { 1400 | console.log(chalk.yellow(result.client) + chalk.red(" not found in s3motionClients.json. Create it using 's3motion -n wizard'")); 1401 | process.exit(1); 1402 | } 1403 | var sourceClient = sclient; 1404 | 1405 | getClient({name: result.destClient, client: 's3'}, function(dclient){ 1406 | if (dclient == '') { 1407 | console.log(chalk.yellow(result.client) + chalk.red(" not found in s3motionClients.json. Create it using 's3motion -n wizard'")); 1408 | process.exit(1); 1409 | } 1410 | var destClient = dclient 1411 | copyBucket({sourceSite: sourceClient, sourceBucket: result.sourceBucket, destinationSite: destClient, destinationBucket: result.destBucket}, function(data) { 1412 | // 1413 | }); 1414 | }); 1415 | }); 1416 | } 1417 | }); 1418 | } 1419 | 1420 | if (program.REST) { 1421 | microservice(); 1422 | } --------------------------------------------------------------------------------