├── LICENSE ├── README.md ├── index.js ├── package.json └── policy.json /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Eleven41 Software Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # An AWS Lambda Based Function to Copy S3 Objects 2 | 3 | With this AWS Lambda function, you can copy objects from a source S3 bucket to one or more target S3 buckets as they are added to the source bucket. 4 | 5 | ## Configuration 6 | 7 | ### IAM Role 8 | 9 | Create an IAM role with the following policy: 10 | 11 | ```json 12 | { 13 | "Version": "2012-10-17", 14 | "Statement": [ 15 | { 16 | "Sid": "Stmt1430872797000", 17 | "Effect": "Allow", 18 | "Action": [ 19 | "s3:GetBucketTagging", 20 | "s3:GetObject", 21 | "s3:PutObject" 22 | ], 23 | "Resource": [ 24 | "*" 25 | ] 26 | }, 27 | { 28 | "Sid": "Stmt1430872844000", 29 | "Effect": "Allow", 30 | "Action": [ 31 | "cloudwatch:*" 32 | ], 33 | "Resource": [ 34 | "*" 35 | ] 36 | }, 37 | { 38 | "Sid": "Stmt1430872852000", 39 | "Effect": "Allow", 40 | "Action": [ 41 | "logs:*" 42 | ], 43 | "Resource": [ 44 | "*" 45 | ] 46 | } 47 | ] 48 | } 49 | ``` 50 | 51 | ### S3 Buckets 52 | 53 | 1. Ensure you have a source and target bucket. They do not need to reside in the same region. 54 | 2. Configure your S3 buckets (see below) 55 | 56 | ### Using the Release Packages 57 | 58 | Release packages can be found on the [Releases](https://github.com/eleven41/aws-lambda-copy-s3-objects/releases) page. 59 | 60 | ### Building the Lambda Package 61 | 62 | 1. Clone this repo 63 | 64 | ``` 65 | git clone git@github.com:eleven41/aws-lambda-copy-s3-objects.git 66 | cd aws-lambda-copy-s3-objects 67 | ``` 68 | 69 | 2. Install requirements 70 | 71 | ``` 72 | npm install async 73 | npm install aws-sdk 74 | ``` 75 | 76 | 3. Zip up the folder using your favourite zipping utility 77 | 78 | ### Lambda Function 79 | 80 | 1. Create a new Lambda function. 81 | 2. Upload the ZIP package to your lambda function. 82 | 3. Add an event source to your Lambda function: 83 | * Event Source Type: S3 84 | * Bucket: your source bucket 85 | * Event Type: Object Created 86 | 4. Set your Lambda function to execute using the IAM role you created above. 87 | 88 | ### Configuration 89 | 90 | Configuration is performed by setting tags on the source bucket. 91 | 92 | Tag Name | Required 93 | ---|--- 94 | TargetBucket | Yes 95 | 96 | **TargetBucket** - A space-separated list of buckets to which the objects will be copied. Optionally, the bucket names can contain a @ character followed by a region to indicate that the bucket resides in a different region. 97 | 98 | For example: `my-target-bucket1 my-target-bucket1@us-west-2 my-target-bucket3@us-east-1` 99 | 100 | 101 | At this point, if you upload a file to your source bucket, the file should be copied to the target bucket(s). 102 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | console.log("Version 0.2.0"); 2 | 3 | // Load up all dependencies 4 | var AWS = require('aws-sdk'); 5 | var async = require('async'); 6 | 7 | // CreateS3 8 | // 9 | // Create a reference to an S3 client 10 | // in the desired region. 11 | function createS3(regionName) { 12 | var config = { apiVersion: '2006-03-01' }; 13 | 14 | if (regionName != null) 15 | config.region = regionName; 16 | 17 | var s3 = new AWS.S3(config); 18 | return s3; 19 | } 20 | 21 | // This is the entry-point to the Lambda function. 22 | exports.handler = function (event, context) { 23 | 24 | if (event.Records == null) { 25 | context.fail('Error', "Event has no records."); 26 | return; 27 | } 28 | 29 | // Process all records in the event asynchronously. 30 | async.each(event.Records, processRecord, function (err) { 31 | if (err) { 32 | context.fail('Error', "One or more objects could not be copied."); 33 | } else { 34 | context.succeed(); 35 | } 36 | }); 37 | }; 38 | 39 | // processRecord 40 | // 41 | // Iterator function for async.each (called by the handler above). 42 | // 43 | // 1. Get the target bucket from the source bucket's tags 44 | // 2. Copy the object to each of the desired buckets. 45 | function processRecord(record, callback) { 46 | 47 | // The source bucket and source key are part of the event data 48 | var srcBucket = record.s3.bucket.name; 49 | var srcKey = decodeURIComponent(record.s3.object.key.replace(/\+/g, " ")); 50 | 51 | // Get the target bucket(s) based on the source bucket. 52 | // Once we have that, perform the copy. 53 | getTargetBuckets(srcBucket, function (err, targets) { 54 | if (err) { 55 | console.log("Error getting target bucket: "); // an error occurred 56 | console.log(err, err.stack); // an error occurred 57 | callback("Error getting target bucket from source bucket '" + srcBucket + "'"); 58 | return; 59 | } 60 | 61 | async.each(targets, function (target, callback) { 62 | 63 | var targetBucket = target.bucketName; 64 | var regionName = target.regionName; 65 | var targetBucketName = targetBucket; 66 | if (regionName != null) 67 | targetBucketName = targetBucketName + "@" + regionName; 68 | var targetKey = srcKey; 69 | 70 | console.log("Copying '" + srcKey + "' from '" + srcBucket + "' to '" + targetBucketName + "'"); 71 | 72 | // Copy the object from the source bucket 73 | var s3 = createS3(regionName); 74 | s3.copyObject({ 75 | Bucket: targetBucket, 76 | Key: targetKey, 77 | 78 | CopySource: encodeURIComponent(srcBucket + '/' + srcKey), 79 | MetadataDirective: 'COPY' 80 | }, function (err, data) { 81 | if (err) { 82 | console.log("Error copying '" + srcKey + "' from '" + srcBucket + "' to '" + targetBucketName + "'"); 83 | console.log(err, err.stack); // an error occurred 84 | callback("Error copying '" + srcKey + "' from '" + srcBucket + "' to '" + targetBucketName + "'"); 85 | } else { 86 | callback(); 87 | } 88 | }); 89 | }, function (err) { 90 | if (err) { 91 | callback(err); 92 | } else { 93 | callback(); 94 | } 95 | }); 96 | }); 97 | }; 98 | 99 | // getTargetBuckets 100 | // 101 | // Gets the tags for the named bucket, and 102 | // from those tags, finds the "TargetBucket" tag. 103 | // Once found, it calls the callback function passing 104 | // the tag value as the single parameter. 105 | function getTargetBuckets(bucketName, callback) { 106 | console.log("Getting tags for bucket '" + bucketName + "'"); 107 | 108 | var s3 = createS3(); 109 | s3.getBucketTagging({ 110 | Bucket: bucketName 111 | }, function (err, data) { 112 | if (err) { 113 | if (err.code == 'NoSuchTagSet') { 114 | // No tags on the bucket, so the bucket is not configured properly. 115 | callback("Source bucket '" + bucketName + "' is missing 'TargetBucket' tag.", null); 116 | } else { 117 | // Some other error 118 | callback(err, null); 119 | } 120 | return; 121 | } 122 | 123 | console.log(data); 124 | var tags = data.TagSet; 125 | 126 | console.log("Looking for 'TargetBucket' tag"); 127 | for (var i = 0; i < tags.length; ++i) { 128 | var tag = tags[i]; 129 | if (tag.Key == 'TargetBucket') { 130 | console.log("Tag 'TargetBucket' found with value '" + tag.Value + "'"); 131 | 132 | var tagValue = tag.Value.trim(); 133 | var buckets = tag.Value.split(' '); 134 | 135 | var targets = []; 136 | 137 | for (var i = 0; i < buckets.length; ++i) { 138 | var bucketSpec = buckets[i].trim(); 139 | if (bucketSpec.length == 0) 140 | continue; 141 | 142 | var specParts = bucketSpec.split('@'); 143 | 144 | var bucketName = specParts[0]; 145 | var regionName = specParts[1] 146 | 147 | targets.push({ bucketName: bucketName, regionName: regionName }); 148 | } 149 | 150 | callback(null, targets); 151 | return; 152 | } 153 | } 154 | 155 | callback("Tag 'TargetBucket' not found", null); 156 | }); 157 | } 158 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aws-lambda-copy-s3-objects", 3 | "version": "0.2.0", 4 | "private": true, 5 | "description": "AWS Lambda function to copy S3 objects from one bucket to another", 6 | "main": "index.js", 7 | "author": "Eleven41 Software Inc.", 8 | "license": "MIT", 9 | "dependencies": { 10 | "async": "^1.5.0", 11 | "aws-sdk": "^2.2.17" 12 | } 13 | } -------------------------------------------------------------------------------- /policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "Stmt1430872797000", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "s3:GetBucketTagging", 9 | "s3:GetObject", 10 | "s3:PutObject" 11 | ], 12 | "Resource": [ 13 | "*" 14 | ] 15 | }, 16 | { 17 | "Sid": "Stmt1430872844000", 18 | "Effect": "Allow", 19 | "Action": [ 20 | "cloudwatch:*" 21 | ], 22 | "Resource": [ 23 | "*" 24 | ] 25 | }, 26 | { 27 | "Sid": "Stmt1430872852000", 28 | "Effect": "Allow", 29 | "Action": [ 30 | "logs:*" 31 | ], 32 | "Resource": [ 33 | "*" 34 | ] 35 | } 36 | ] 37 | } 38 | --------------------------------------------------------------------------------