├── aws-s3-crr-dr.yaml ├── LICENSE ├── aws-s3-crr-primary.yaml ├── aws-s3-create-bucket-replicated.sh └── README.md /aws-s3-crr-dr.yaml: -------------------------------------------------------------------------------- 1 | Description: > 2 | Create a simple encrypted S3 bucket and suffix the region to the name 3 | 4 | Parameters: 5 | 6 | NAME: 7 | Type: String 8 | 9 | 10 | Resources: 11 | 12 | BUCKET: 13 | Type: "AWS::S3::Bucket" 14 | Properties: 15 | BucketName: !Sub ${NAME} 16 | BucketEncryption: 17 | ServerSideEncryptionConfiguration: 18 | - ServerSideEncryptionByDefault: 19 | SSEAlgorithm: AES256 20 | VersioningConfiguration: 21 | Status: "Enabled" 22 | 23 | Outputs: 24 | 25 | NAME: 26 | Description: Full name of bucket created 27 | Value: !Sub ${NAME} 28 | Export: 29 | Name: !Sub ${NAME} 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Doug Toppin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /aws-s3-crr-primary.yaml: -------------------------------------------------------------------------------- 1 | Description: > 2 | s3 crr testing 3 | 4 | Parameters: 5 | 6 | NAME: 7 | Type: String 8 | 9 | REGIONDEST: 10 | Type: String 11 | 12 | Resources: 13 | 14 | ReplicaRole: 15 | Type: AWS::IAM::Role 16 | Properties: 17 | AssumeRolePolicyDocument: 18 | Statement: 19 | - Action: ['sts:AssumeRole'] 20 | Effect: Allow 21 | Principal: 22 | Service: [s3.amazonaws.com] 23 | 24 | ReplicaPolicy: 25 | Type: AWS::IAM::Policy 26 | Properties: 27 | PolicyDocument: 28 | Statement: 29 | - Action: 30 | - s3:Get* 31 | - s3:ListBucket 32 | Resource: 33 | - !Sub arn:aws:s3:::${NAME}-${AWS::Region} 34 | - !Sub arn:aws:s3:::${NAME}-${AWS::Region}/* 35 | Effect: 'Allow' 36 | - Action: 37 | - s3:ReplicateObject 38 | - s3:ReplicateDelete 39 | - s3:ReplicateTags 40 | - s3:GetObjectVersionTagging 41 | Effect: 'Allow' 42 | Resource: !Sub arn:aws:s3:::${NAME}-${REGIONDEST}/* 43 | PolicyName: ReplicaPolicy 44 | Roles: [!Ref 'ReplicaRole'] 45 | 46 | S3Bucket: 47 | Type: AWS::S3::Bucket 48 | DeletionPolicy: Delete 49 | Properties: 50 | BucketName: !Sub ${NAME}-${AWS::Region} 51 | BucketEncryption: 52 | ServerSideEncryptionConfiguration: 53 | - ServerSideEncryptionByDefault: 54 | SSEAlgorithm: AES256 55 | ReplicationConfiguration: 56 | Role: !GetAtt [ReplicaRole, Arn] 57 | Rules: 58 | - Destination: 59 | Bucket: !Sub arn:aws:s3:::${NAME}-${REGIONDEST} 60 | StorageClass: STANDARD 61 | Id: Backup 62 | Prefix: '' 63 | Status: Enabled 64 | VersioningConfiguration: 65 | Status: Enabled 66 | 67 | Outputs: 68 | 69 | NAME: 70 | Description: Full name of bucket created 71 | Value: !Sub ${NAME}-${AWS::Region} 72 | Export: 73 | Name: !Sub ${NAME}-${AWS::Region} 74 | 75 | NAMEREP: 76 | Description: Full name of repication bucket 77 | Value: !Sub ${NAME}-${REGIONDEST} 78 | Export: 79 | Name: !Sub ${NAME}-${REGIONDEST} 80 | -------------------------------------------------------------------------------- /aws-s3-create-bucket-replicated.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # exit on any error 4 | set -e 5 | 6 | # aws-s3-create-bucket-replicated.sh 7 | 8 | # This is an example of how to create an AWS s3 bucket that is replicated to another region 9 | 10 | # fail on any error 11 | #set -e 12 | 13 | usage() { 14 | echo "Usage $0 (create|delete) bucket-name [ aws-profile ]" 15 | } 16 | 17 | if test $# -lt 2 18 | then 19 | usage 20 | exit 1 21 | fi 22 | 23 | AWSPROFILE="" 24 | 25 | if test $# -eq 3 26 | then 27 | # if a third argument was provided assume it is the aws profile to use for the operations 28 | AWSPROFILE=" --profile $3 " 29 | echo "using aws profile 3" 30 | fi 31 | 32 | 33 | # what to do 34 | ACTION="$1" 35 | 36 | # bucket name to be created, note that this will have source and destination regions suffixed to it 37 | NAME=$2 38 | 39 | # detination region to replicate to 40 | REGIONDEST=us-west-1 41 | 42 | # source region to create bucket to be replicated 43 | REGIONSRC=us-east-1 44 | 45 | NAMEDEST=$NAME-$REGIONDEST 46 | NAMESRC=$NAME-$REGIONSRC 47 | 48 | STACKMAIN=aws-s3-crr-primary 49 | STACKREP=aws-s3-crr-dr 50 | 51 | validate() { 52 | 53 | # always check for template validity before we do anything 54 | for i in $STACKMAIN.yaml $STACKREP.yaml 55 | do 56 | 57 | echo "validating $i" 58 | aws $AWSPROFILE cloudformation validate-template --template-body file://$i 59 | echo 60 | 61 | # cfn-lint (pip install cfn-lint) is a very good linter for templates, 62 | # if it is available then also run it 63 | if [ -x $(which cfn-lint) ]; then 64 | echo "running cfn-lint" 65 | cfn-lint $i 66 | else 67 | echo "cfn-lint not found, continuing" 68 | fi 69 | 70 | done 71 | 72 | } 73 | 74 | create() { 75 | 76 | # user is responsible for ensuring that the buckets to be created do not already exist 77 | 78 | 79 | echo "creating replication bucket" 80 | 81 | # the destination region bucket must be created first 82 | aws $AWSPROFILE cloudformation create-stack --stack-name $STACKREP \ 83 | --template-body file://$STACKREP.yaml \ 84 | --region "$REGIONDEST" \ 85 | --parameters ParameterKey=NAME,ParameterValue="$NAMEDEST" 86 | 87 | STATUS="" 88 | 89 | while :; do 90 | if test "$STATUS" == "CREATE_COMPLETE" 91 | then 92 | break 93 | fi 94 | echo "checking status" 95 | STATUS=$(aws cloudformation describe-stacks --stack-name $STACKREP --query Stacks[].StackStatus --output text --region $REGIONDEST) 96 | sleep 5 97 | done 98 | 99 | echo "creating primary bucket" 100 | 101 | aws $AWSPROFILE cloudformation create-stack --stack-name $STACKMAIN --template-body file://$STACKMAIN.yaml \ 102 | --region $REGIONSRC \ 103 | --capabilities CAPABILITY_NAMED_IAM \ 104 | --parameters ParameterKey=NAME,ParameterValue="$NAME" ParameterKey=REGIONDEST,ParameterValue="$REGIONDEST" 105 | 106 | 107 | } 108 | 109 | delete() { 110 | 111 | # user is responsible for ensuring that the buckets to be deleted have already been emptied 112 | 113 | echo "deleting stack $STACKMAIN" 114 | aws $AWSPROFILE cloudformation delete-stack --stack-name $STACKMAIN --region $REGIONSRC 115 | 116 | sleep 5 117 | 118 | echo "deleting stack $STACKREP" 119 | aws $AWSPROFILE cloudformation delete-stack --stack-name $STACKREP --region $REGIONDEST 120 | 121 | 122 | } 123 | 124 | validate 125 | 126 | case "$ACTION" in 127 | create) 128 | create $NAME 129 | ;; 130 | 131 | delete) 132 | delete 133 | ;; 134 | 135 | *) 136 | usage 137 | exit 1 138 | ;; 139 | 140 | esac 141 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Creating an AWS S3 bucket with replication to another region using CloudFormation 2 | 3 | AWS S3 buckets can be configured to replicate all objects put in them to another bucket in a different region. 4 | This is called ```Cross Region Replication```. 5 | 6 | Why this is useful is that objects stored in a bucket are kept only in the region that they were created in. 7 | Other regions may be able to access them if allowed but if a regional outage were to occur the contents of the buckets in that region may not be accessible. 8 | 9 | Note that because S3 buckets have a global namespace it is not possible to have a bucket with the same name in 2 different regions. 10 | 11 | Because of this it is useful to name a bucket with a suffix of the region that the bucket was created in. 12 | Doing that allows you to have uniquely named buckets that differ in name by only the region making the functions accessing the contents easier to write and manage. 13 | 14 | This is an example of using CloudFormation to create both a bucket to store objects in and a bucket to replicate those objects to. 15 | The CloudFormation stacks will be called ```aws-s3-crr-primary``` and ```aws-s3-crr-dr```. 16 | Because the stack names are fixed you cannot use this script as is to create multiple buckets. 17 | To do that change the script to use unique names for each stack. 18 | 19 | The regions to use are also set the script to us-east-1 for the primary and us-west-1 for the replica. 20 | 21 | The contents of this repository consists of a shell script to create and delete the buckets and the 2 CloudFormation templates to define how to create the buckets. 22 | 23 | * aws-s3-create-bucket-replicated.sh - shell script to create the CloudFormation stacks 24 | * aws-s3-crr-primary.yaml - primary bucket definition 25 | * aws-s3-crr-dr.yaml - replica bucket definition 26 | 27 | Note that that to enable the automatic copying of bucket contents a policy and role are attached to the source bucket. 28 | Both buckets must also have versioning enabled. 29 | Both buckets also have encryption enabled as an example. 30 | 31 | The replica bucket stack, defined by aws-s3-crr-dr.yaml, only requires that versioning be enabled. 32 | Other than that it is entirely normal. 33 | 34 | To get started run the script with a create argument and the name of a bucket to create. 35 | This will create the replication bucket in another region and suffix the region name to the bucket. 36 | When that operation has completed the main bucket will be created again with the region name suffixed. 37 | 38 | Example: 39 | 40 | ``` 41 | $ ./aws-s3-create-bucket-replicated.sh create my-unique-bucket-01 42 | validating aws-s3-crr-primary.yaml 43 | { 44 | "CapabilitiesReason": "The following resource(s) require capabilities: [AWS::IAM::Role]", 45 | "Description": "s3 crr testing\n", 46 | "Parameters": [ 47 | { 48 | "NoEcho": false, 49 | "ParameterKey": "REGIONDEST" 50 | }, 51 | { 52 | "NoEcho": false, 53 | "ParameterKey": "NAME" 54 | } 55 | ], 56 | "Capabilities": [ 57 | "CAPABILITY_IAM" 58 | ] 59 | } 60 | 61 | validating aws-s3-crr-dr.yaml 62 | { 63 | "Description": "Create a simple encrypted S3 bucket and suffix the region to the name\n", 64 | "Parameters": [ 65 | { 66 | "NoEcho": false, 67 | "ParameterKey": "NAME" 68 | } 69 | ] 70 | } 71 | 72 | creating replication bucket 73 | { 74 | "StackId": "arn:aws:cloudformation:us-west-1:xxx:stack/aws-s3-crr-dr/2667dbb0-6cf8-11e8-9a36-500cc17864e6" 75 | } 76 | checking status 77 | checking status 78 | checking status 79 | checking status 80 | checking status 81 | checking status 82 | checking status 83 | creating primary bucket 84 | { 85 | "StackId": "arn:aws:cloudformation:us-east-1:xxx:stack/aws-s3-crr-primary/3fc1b9f0-6cf8-11e8-9fe0-500c2854b635" 86 | } 87 | ``` 88 | 89 | An optional third argument can be included which will specify the aws 90 | profile to have the creates and delete applied to. 91 | This can be helpful if you need to use different IAM accounts with different privileges. 92 | The following is an example of including *dummy* as the name of a profile to use. 93 | 94 | ``` 95 | 96 | $ ./aws-s3-create-bucket-replicated.sh create testme dummy1 97 | using aws profile 3 98 | validating aws-s3-crr-primary.yaml 99 | { 100 | "CapabilitiesReason": "The following resource(s) require capabilities: [AWS::IAM::Role]", 101 | "Description": "s3 crr testing\n", 102 | "Parameters": [ 103 | { 104 | "NoEcho": false, 105 | "ParameterKey": "REGIONDEST" 106 | }, 107 | { 108 | "NoEcho": false, 109 | "ParameterKey": "NAME" 110 | } 111 | ], 112 | "Capabilities": [ 113 | "CAPABILITY_IAM" 114 | ] 115 | } 116 | 117 | running cfn-lint 118 | 119 | validating aws-s3-crr-dr.yaml 120 | { 121 | "Description": "Create a simple encrypted S3 bucket and suffix the region to the name\n", 122 | "Parameters": [ 123 | { 124 | "NoEcho": false, 125 | "ParameterKey": "NAME" 126 | } 127 | ] 128 | } 129 | 130 | running cfn-lint 131 | 132 | creating replication bucket 133 | { 134 | "StackId": "arn:aws:cloudformation:us-west-1:xxx:stack/aws-s3-crr-dr/2ac4fa70-4142-11e9-b917-06cca9edf6c0" 135 | } 136 | checking status 137 | checking status 138 | checking status 139 | checking status 140 | checking status 141 | checking status 142 | checking status 143 | creating primary bucket 144 | { 145 | "StackId": "arn:aws:cloudformation:us-east-1:xxx:stack/aws-s3-crr-primary/44585f40-4142-11e9-a295-0e639cbb91d4" 146 | } 147 | 148 | ``` 149 | 150 | After the CloudFormation stacks are successfully created any files copied to the source region bucket should automatically appear in the destination region bucket. 151 | 152 | The script can also be run with a delete argument and will delete both stacks created which will cause the buckets created to be deleted as well. 153 | 154 | Note, before trying to delete the CloudFormation stacks the bucket contents in both regions must be deleted. 155 | This script does not do it itself so it must be done manually. 156 | 157 | ``` 158 | $ ./aws-s3-create-bucket-replicated.sh delete my-unique-bucket-01 159 | validating aws-s3-crr-primary.yaml 160 | { 161 | "CapabilitiesReason": "The following resource(s) require capabilities: [AWS::IAM::Role]", 162 | "Description": "s3 crr testing\n", 163 | "Parameters": [ 164 | { 165 | "NoEcho": false, 166 | "ParameterKey": "REGIONDEST" 167 | }, 168 | { 169 | "NoEcho": false, 170 | "ParameterKey": "NAME" 171 | } 172 | ], 173 | "Capabilities": [ 174 | "CAPABILITY_IAM" 175 | ] 176 | } 177 | 178 | validating aws-s3-crr-dr.yaml 179 | { 180 | "Description": "Create a simple encrypted S3 bucket and suffix the region to the name\n", 181 | "Parameters": [ 182 | { 183 | "NoEcho": false, 184 | "ParameterKey": "NAME" 185 | } 186 | ] 187 | } 188 | 189 | deleting stack aws-s3-crr-primary 190 | deleting stack aws-s3-crr-dr 191 | ``` 192 | --------------------------------------------------------------------------------