├── .github ├── solutionid_validator.sh └── workflows │ └── maintainer_workflows.yml ├── .gitignore ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── assets ├── ChunkFileProcessor.png ├── Dashboard.png ├── MRBlogs-Failover.png ├── MRBlogs-Primary Region Sequence.png ├── MRBlogs-Primary-Region-Sequence.png ├── MainOrchestrator.png ├── load-test.sh └── testfile.csv ├── deployment ├── Makefile ├── cleanup.sh ├── dashboard-template.yml ├── fisTemplate.yml ├── globalResources.yml ├── globalRouting.yml ├── regionalVpc.yml ├── samTemplate.yaml └── samconfig.toml └── source ├── custom-resource ├── app.py ├── cfnresponse.py ├── requirements.txt └── testfile_financial_data.csv ├── failover ├── __init__.py ├── app.py └── requirements.txt ├── get-data ├── __init__.py ├── app.py ├── requirements.txt └── schemas.py ├── merge-s3-files ├── __init__.py ├── app.py └── requirements.txt ├── read-file ├── __init__.py ├── app.py └── requirements.txt ├── reconciliation ├── __init__.py ├── app.py └── requirements.txt ├── s3-lambda-notification ├── __init__.py ├── app.py └── requirements.txt ├── send-email ├── __init__.py ├── app.py └── requirements.txt ├── split-ip-file ├── __init__.py ├── app.py └── requirements.txt ├── statemachine ├── blog-sfn-main-orchestrator.json └── blog-sfn-process-chunk.json ├── validate-data ├── __init__.py ├── app.py ├── requirements.txt └── schemas.py └── write-output-chunk ├── __init__.py ├── app.py └── requirements.txt /.github/solutionid_validator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #set -e 3 | 4 | echo "checking solution id $1" 5 | echo "grep -nr --exclude-dir='.github' "$1" ./.." 6 | result=$(grep -nr --exclude-dir='.github' "$1" ./..) 7 | if [ $? -eq 0 ] 8 | then 9 | echo "Solution ID $1 found\n" 10 | echo "$result" 11 | exit 0 12 | else 13 | echo "Solution ID $1 not found" 14 | exit 1 15 | fi 16 | 17 | export result 18 | -------------------------------------------------------------------------------- /.github/workflows/maintainer_workflows.yml: -------------------------------------------------------------------------------- 1 | # Workflows managed by aws-solutions-library-samples maintainers 2 | name: Maintainer Workflows 3 | on: 4 | # Triggers the workflow on push or pull request events but only for the "main" branch 5 | push: 6 | branches: [ "main" ] 7 | pull_request: 8 | branches: [ "main" ] 9 | types: [opened, reopened, edited] 10 | 11 | jobs: 12 | CheckSolutionId: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Run solutionid validator 17 | run: | 18 | chmod u+x ./.github/solutionid_validator.sh 19 | ./.github/solutionid_validator.sh ${{ vars.SOLUTIONID }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | .idea 3 | .DS_Store 4 | .aws-sam 5 | test/* 6 | /deployment/cfn_nag_scan_results.txt 7 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | CODEOWNERS @aws-solutions-library-samples/maintainers 2 | /.github/workflows/maintainer_workflows.yml @aws-solutions-library-samples/maintainers 3 | /.github/solutionid_validator.sh @aws-solutions-library-samples/maintainers 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT No Attribution 2 | 3 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 13 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 15 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 16 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Guidance for Multi-Region Serverless Batch Applications on AWS 2 | 3 | ## Getting started 4 | 5 | This guidance helps customers design and operate a multi-Region, serverless batch solution on AWS using services like 6 | Step Functions, DynamoDB Global Tables, S3 Multi-Region Access Points, Route53 Application Recovery Controller and 7 | Lambda functions. The solution is deployed across two Regions that can failover and failback from one Region to another 8 | in an automated fashion. It leverages Amazon Route 53 Application Recovery Controller to help with the regional failover 9 | using AWS Systems Manager documents, S3 Multi-Region Access Points to provide a unified endpoint for S3 access that 10 | spans multiple Region, and DynamoDB Global Tables for persisting batch state and tracking. 11 | 12 | The step function based batch processing solution follows the architecture outlined in the AWS Blog article ["Creating AWS Serverless Batch Processing Architectures"](https://aws.amazon.com/blogs/compute/creating-aws-serverless-batch-processing-architectures/). For more details about the inner workings of the processing logic within the step functions, refer to this article. 13 | 14 | ## Architecture 15 | 16 | ### 1. Operating Batch in Primary Region 17 | ![Application Running in Primary Region](assets/MRBlogs-Primary-Region-Sequence.png) 18 | 1. A file is put to S3 bucket via Multi-region Access Point. MRAP routes the file to one of the S3 buckets. Each bucket will replicate the object to the other bucket. 19 | 2. The lambda function is invoked via S3 putObject event in both regions. 20 | 3. The function will resolve the TXT record in the Route53 private hosted zone to determine if it is the active region. If it is, execution will continue. If it is not, the function will exit and no further actions will be taken. The function in the active region writes metadata on the file to the DynamoDB Batch State table including that the processing has started and starts the first Step Function. 21 | 4. The first Step Function (Main Orchestrator) orchestrates the processing of the file. ![4 - StepFunction](assets/MainOrchestrator.png) 22 | 1. The first task state Split Input File into chunks calls a Lambda function. It splits the main file into multiple chunks based on the number of records and stores each chunk into an S3 bucket. 23 | 2. The next state is a map state called Call Step Functions for each chunk. It uses the Step Functions service integration to trigger the Chunk Processor workflow for each chunk of the file. It also passes the S3 bucket path of the split file chunk as a parameter to the Chunk Processor workflow. Then the Main batch orchestrator waits for all the child workflow executions to complete. 24 | 3. Once all the child workflows are processed successfully, the next task state is Merge all Files. This combines all the processed chunks into a single file and then stores the file back to the S3 bucket. 25 | 4. The next task state Email the file takes the output file. It generates an S3 presigned URL for the file using the MRAP endpoint, and sends an email with the S3 MRAP presigned URL. 26 | 5. The next Step Function (Chunk File Processor) is responsible for processing each row from the chunk file that was passed. ![5 - StepFunction](assets/ChunkFileProcessor.png) 27 | 1. The first task state Read reads the chunked file from S3 and converts it to an array of JSON objects. Each JSON object represents a row in the chunk file. 28 | 2. The next state is a map state called Process messages. It runs a set of steps for each element of an input array. The input to the map state is an array of JSON objects passed by the previous task. 29 | 3. Within the map state, Validate Data is the first state. It invokes a Lambda function that validates each JSON object using the rules that you have created. 30 | 4. Records that fail validation are stored in an Amazon DynamoDB table. 31 | 5. The next state Get Financial Data invokes Amazon API Gateway endpoints to enrich the data in the file with data from a DynamoDB table. 32 | 6. When the map state iterations are complete, the Write output file state triggers a task. It calls a Lambda function, which converts the JSON data back to CSV and writes the output object to S3. 33 | 6. The merged file is written to S3 and bucket replication replicates it to the standby region's bucket. 34 | 7. A pre-signed URL is generated using the multi-region access point (MRAP) so that the file can be retrieved from either bucket (closest to the user) and the routing logic is abstracted from the client. 35 | 8. The pre-signed URL is mailed to the recipients so that they can retrieve the file from one of the S3 buckets via the multi-region access point. 36 | 37 | ### 2. Cross Region Failover and Failback 38 | ![Cross-Region Failover and Failback](assets/MRBlogs-Failover.png) 39 | 1. Systems Manager runbook is executed to initiate failover to standby region 40 | 2. The runbook invokes a Lambda function that connects to the Route53 Application Recovery Controller (ARC) cluster to toggle the TXT record in Route53 private hosted zone. 41 | 3. The runbook waits for 15 minute for S3 replication (since this solution enables the S3 Replication Time Control which has a SLA of 15 mins for replication) to finish and then invokes a second reconciliation Lambda function that reads the Batch State DynamoDB global table to determine the names of the objects to start processing but not complete. The function then re-copies those objects within the standby bucket into the `input` directory. It also logs any objects that were unfinished according to the DynamoDB table status but were not present in the S3 bucket in the standby region. 42 | 4. This creates the S3 putObject event and invokes the lambda function. 43 | 5. The function will resolve the TXT recored in the Route53 private hosted zone to determine if it is the active region. Since the failover function in step 2 altered the TXT record, execution will continue. The function writes metadata on the file to the DynamoDB Batch State table including that the processing has started and starts the first Step Function. 44 | 6. The first Step Function (Main Orchestrator) orchestrates the processing of the file. 45 | 1. The first task state Split Input File into chunks calls a Lambda function. It splits the main file into multiple chunks based on the number of records and stores each chunk into an S3 bucket. 46 | 2. The next state is a map state called Call Step Functions for each chunk. It uses the Step Functions service integration to trigger the Chunk Processor workflow for each chunk of the file. It also passes the S3 bucket path of the split file chunk as a parameter to the Chunk Processor workflow. Then the Main batch orchestrator waits for all the child workflow executions to complete. 47 | 3. Once all the child workflows are processed successfully, the next task state is Merge all Files. This combines all the processed chunks into a single file and then stores the file back to the S3 bucket. 48 | 4. The next task state Email the file takes the output file. It generates an S3 presigned URL for the file using the MRAP endpoint, and sends an email with the S3 MRAP presigned URL. 49 | 7. The next Step Function (Chunk File Processor) is responsible for processing each row from the chunk file that was passed. 50 | 1. The first task state Read reads the chunked file from S3 and converts it to an array of JSON objects. Each JSON object represents a row in the chunk file. 51 | 2. The next state is a map state called Process messages. It runs a set of steps for each element of an input array. The input to the map state is an array of JSON objects passed by the previous task. 52 | 3. Within the map state, Validate Data is the first state. It invokes a Lambda function that validates each JSON object using the rules that you have created. 53 | 4. Records that fail validation are stored in an Amazon DynamoDB table. 54 | 5. The next state Get Financial Data invokes Amazon API Gateway endpoints to enrich the data in the file with data from a DynamoDB table. 55 | 6. When the map state iterations are complete, the Write output file state triggers a task. It calls a Lambda function, which converts the JSON data back to CSV and writes the output object to S3. 56 | 8. The merged file is written to S3 and bucket replication replicates it to the standby region's bucket. 57 | 9. A pre-signed URL is generated using the multi-region access point (MRAP) so that the file can be retrieved from either bucket (closest to the user) and the routing logic is abstracted from the client. 58 | 10. The pre-signed URL is mailed to the recipients so that they can retrieve the file from one of the S3 buckets via the multi-region access point. 59 | 60 | ## Pre-requisites 61 | 62 | * To deploy this example guidance, you need an AWS account (We suggest using a temporary or a development account to 63 | test this guidance), and a user identity with access to the following services: 64 | * AWS CloudFormation 65 | * Amazon API Gateway 66 | * Amazon CloudWatch 67 | * Amazon DynamoDB 68 | * AWS Step Functions 69 | * AWS Lambda 70 | * Amazon Simple Storage Service 71 | * Amazon Route 53 72 | * Amazon Virtual Private Cloud (VPC) 73 | * AWS Identity and Access Management (IAM) 74 | * AWS Secrets Manager 75 | * AWS Systems Manager 76 | * Amazon Simple Email Service 77 | * Install the latest version of AWS CLI v2 on your machine, including configuring the CLI for a specific account and region 78 | profile. Please follow the [AWS CLI setup instructions](https://github.com/aws/aws-cli). Make sure you have a 79 | default profile set up; you may need to run `aws configure` if you have never set up the CLI before. 80 | * Install Python version 3.9 on your machine. Please follow the [Download and Install Python](https://www.python.org/downloads/) instructions. 81 | * Install the latest version of AWS SAM CLI on your machine. Please follow the [Install AWS SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.html) 82 | * Install `make` for your OS if it is not already there. 83 | 84 | ### Regions 85 | 86 | This demonstration by default uses `us-east-1` as the primary region and `us-west-2` as the backup region. These can be changed in the Makefile 87 | 88 | ## Deployment 89 | 90 | For the purposes of this workshop, we deploy the CloudFormation Templates and SAM Templates via a Makefile. For a production 91 | workload, you'd want to have an automated deployment pipeline. As discussed in this 92 | [article](https://aws.amazon.com/builders-library/automating-safe-hands-off-deployments/?did=ba_card&trk=ba_card), a 93 | multi-region pipeline should follow a staggered deployment schedule to reduce the blast radius of a bad deployment. 94 | Take particular care with changes that introduce possibly backwards-incompatible changes like schema modifications, 95 | and make use of schema versioning. 96 | 97 | ## Configuration 98 | Before starting deployment process please update the following variables in the `deployment/Makefile`: 99 | 100 | **ENV** - It is the unique variable that indicates the environment name. Global resources created, such as S3 buckets, use this name. (ex: -dev) 101 | 102 | **SES_IDENTITY** - The email id used to create a verified SES identity for sending emails. (ex: joejane@xyz.com). The solution creates a 103 | new identity in the Amazon Simple Email Service which requires a verification before it can be used by the solution. You can also provide an email address with the `+` symbol (for example; joejane+batch@xyz.com). 104 | 105 | **PRIMARY_REGION** - The AWS region that will serve as primary for the workload 106 | 107 | **SECONDARY_REGION** - The AWS region that will serve as standby or failover for the workload 108 | 109 | ## Deployment Steps 110 | 111 | We use make file to automate the deployment commands. The make file is optimized for Mac. If you plan to deploy the solution from another OS, you may have to update few commands. 112 | 113 | 1. Deploy the full solution from the `deployment` folder 114 | ```shell 115 | make deploy 116 | ``` 117 | 118 | 2. Check the Inbox of the SES Identity user email, for two emails asking you the click the verification links contained within. Please click on those links. 119 | 120 | Once the above steps have completed, we should have all the key components of the workload deployed in the primary and standby region as specified in the architecture. Expect the total deployment to take several minutes as it is orchestrating a deployment in two AWS regions in the proper order. 121 | 122 | ## Verify the deployment 123 | 124 | 1. Find the S3 Multi-Region Access Point(MRAP) ARN using the AWS Console or follow the below steps to construct the ARN. 125 | 126 | a. Export the AWS_ACCOUNT and the Multi Region Access Point name as created by the CloudFormation stack. The default MRAP name as configured, 127 | follows the convention `source-bucket-mrap${ENV}` where the value of ENV is the same as was set in the Makefile. 128 | 129 | ```shell 130 | export AWS_ACCOUNT=xxxx 131 | export MRAP_NAME=source-bucket-mrap${ENV} 132 | ``` 133 | b. To get the S3 MRAP Alias you can use the below command 134 | ```shell 135 | export MRAP_ALIAS=$(aws s3control get-multi-region-access-point --account-id ${AWS_ACCOUNT} --name ${MRAP_NAME} --region us-west-2 --output=text --query='{Alias: AccessPoint.Alias}') 136 | ``` 137 | 138 | 2. From the `assets` folder, upload the [testfile.csv](assets/testfile.csv) to the Access Point identified in Step 1. 139 | ```shell 140 | aws s3 cp testfile.csv s3://arn:aws:s3::${AWS_ACCOUNT}:accesspoint/${MRAP_ALIAS}/input/testfile.csv 141 | ``` 142 | 3. Depending on the Region that is primary, the file would be processed in that Region and you should receive an email in the inbox of the user belonging to the `SES_IDENTITY` that was provided initially. The email should contain a PreSigned URL to the processed output of the batch. Since this URL was generated using the S3 Multi-region access point, the file will be retrieved from either available bucket. 143 | 144 | ## Load Testing 145 | 146 | To simulate load by uploading a large number of files to S3 you can execute a simple shell script like the example shown in [assets/load-test.sh](assets/load-test.sh) 147 | The script is automated to upload 100 objects to the MRAP endpoint with a 1 second delay in between consecutive uploads. 148 | 149 | Execute the shell script as follows. To make the shell script as an executable you can use the command `chmod +x load_test.sh` 150 | ```shell 151 | export MRAP_ARN="arn:aws:s3::${AWS_ACCOUNT}:accesspoint/${MRAP_ALIAS}" 152 | ./load-test.sh -a $MRAP_ARN -r -w 153 | ``` 154 | 155 | ## Observability 156 | 157 | The deployment also provisions a Cloudwatch dashboard by the name of `MultiRegionBatchDashboard${ENV}` (where ENV is the same value that was set before the deployment), 158 | which display the following graphs 159 | 160 | ![Dashboard](assets/Dashboard.png) 161 | 162 | __For each Region (Primary and Standby)__ 163 | * Number of input files split for processing 164 | * Number of Processed files 165 | * Main Orchestrator Step Function Metrics 166 | * Executions succeeded 167 | * Executions started 168 | * Executions failed 169 | * Chunk File Processor Step Function Metrics 170 | * Executions succeeded 171 | * Executions started 172 | * Executions failed 173 | * Number of files reconciled 174 | 175 | ## Injecting Chaos to simulate failures 176 | 177 | To induce failures into your environment, you can use the [fisTemplate.yml](deployment/fisTemplate.yml) and perform Chaos experiments. 178 | This cloudformation template uses AWS Fault Injection Simulator to simulate failures like blocking network access to S3 Buckets/DynamoDB from the subnets for a specific duration. 179 | Running this experiment will allow you to perform a Regional failover and observe the reconciliation process. 180 | 181 | To deploy the Chaos testing resources in both the Regions - Primary and Standby, you can execute the below command. 182 | ```shell 183 | make chaos 184 | ``` 185 | 186 | The following sequence of steps can be used to test this solution. 187 | 1. The first step is to get the experimentTemplateId to use in our experiment; use the below command for that and make a note of the `id` value 188 | ```shell 189 | export templateId=$(aws fis list-experiment-templates --query='experimentTemplates[?tags.Name==`MultiRegionBatchChaosTemplate`].{id: id}' --output text) 190 | ``` 191 | 2. Execute the experiment in the primary Region using the following command using the templateId from the previous step. 192 | ```shell 193 | aws fis start-experiment --experiment-template-id $templateId 194 | ``` 195 | For more details on the specific FIS network action used in this template, read this [documentation](https://docs.aws.amazon.com/fis/latest/userguide/fis-actions-reference.html#disrupt-connectivity) 196 | 197 | ## Executing SSM Failover Automation in Standby Region 198 | 199 | To trigger a Regional failover of the batch process, execute the SSM Automation from the standby Region as follows. 200 | 201 | ```shell 202 | export SECONDARY_REGION=us-west-2 203 | aws ssm start-automation-execution --document-name "AutomationFailoverRunbook" --document-version "\$DEFAULT" --region $SECONDARY_REGION 204 | ``` 205 | 206 | ## Cleanup 207 | 208 | 1. Delete all the cloudformation stacks and associated resources from both the Regions, by running the following command from the `deployment` folder 209 | ```shell 210 | make destroy-all 211 | ``` 212 | 213 | 2. if you have deployed the Chaos testing stack, you can delete the cloudformation stacks using the following command 214 | ```shell 215 | make destroy-chaos 216 | ``` 217 | 218 | ## Security 219 | See [CONTRIBUTING](CONTRIBUTING.md) for more information. 220 | 221 | ## License 222 | This library is licensed under the MIT-0 License. See the [LICENSE](LICENSE) file. 223 | -------------------------------------------------------------------------------- /assets/ChunkFileProcessor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/assets/ChunkFileProcessor.png -------------------------------------------------------------------------------- /assets/Dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/assets/Dashboard.png -------------------------------------------------------------------------------- /assets/MRBlogs-Failover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/assets/MRBlogs-Failover.png -------------------------------------------------------------------------------- /assets/MRBlogs-Primary Region Sequence.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/assets/MRBlogs-Primary Region Sequence.png -------------------------------------------------------------------------------- /assets/MRBlogs-Primary-Region-Sequence.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/assets/MRBlogs-Primary-Region-Sequence.png -------------------------------------------------------------------------------- /assets/MainOrchestrator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/assets/MainOrchestrator.png -------------------------------------------------------------------------------- /assets/load-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -xv 3 | 4 | usage() { echo "Usage: $0 [-a ] [-r ] [-w ]" 1>&2; exit 1; } 5 | 6 | while getopts ":a:r:w:" opt; do 7 | case $opt in 8 | a) 9 | S3_MRAP_ARN=$OPTARG 10 | ;; 11 | r) 12 | RUNS=$OPTARG 13 | ;; 14 | w) 15 | INTERVAL=$OPTARG 16 | ;; 17 | esac 18 | done 19 | shift $((OPTIND-1)) 20 | 21 | if [ -z "$S3_MRAP_ARN" ] || [ -z "$RUNS" ] || [ -z "$INTERVAL" ]; then 22 | usage 23 | fi 24 | 25 | echo "S3 MRAP ARN: $S3_MRAP_ARN" 26 | echo "Number of batch file runs: $RUNS" 27 | echo "Wait between uploads: $INTERVAL" 28 | echo "Starting Load Test..." 29 | 30 | for i in $(seq 1 $RUNS); 31 | do 32 | fileName=testfile_$i.csv 33 | aws s3 cp testfile.csv s3://$S3_MRAP_ARN/input/$fileName 34 | echo "Uploaded File: $fileName" 35 | sleep $INTERVAL 36 | done -------------------------------------------------------------------------------- /deployment/Makefile: -------------------------------------------------------------------------------- 1 | # (SO9169) 2 | # Before starting deployment process please update the following variables. 3 | # ENV - unique variable that indicates the environment name. Global resources created, such as S3 buckets, use this name. (ex: devx) 4 | # PRIMARY_REGION - AWS region that will serve as primary for the workload 5 | # SECONDARY_REGION - AWS region that will serve as the failover region for the workload 6 | # DOMAIN_NAME - the Route53 private hosted zone domain name that should be created for this workload 7 | 8 | ENV="-dev" 9 | PRIMARY_REGION=us-east-1 10 | SECONDARY_REGION=us-west-2 11 | SES_IDENTITY=sender@example.com 12 | EMPTY_LIST='{ "Objects": null }' 13 | .DEFAULT_GOAL := test-creds 14 | MAKE=/usr/bin/make 15 | 16 | global: 17 | @echo "Installing global infrastructure Cloudformation stack..." 18 | $(eval result:=$(shell aws cloudformation create-stack --stack-name global-base$(ENV) --region $(PRIMARY_REGION) --output text --template-body file://globalResources.yml --capabilities CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND --parameters ParameterKey=PrimaryRegion,ParameterValue=$(PRIMARY_REGION) ParameterKey=SecondaryRegion,ParameterValue=$(SECONDARY_REGION) ParameterKey=Env,ParameterValue=$(ENV))) 19 | @aws cloudformation wait stack-create-complete --stack-name global-base$(ENV) --region $(PRIMARY_REGION) 20 | @echo "Finished global infrastructure stack" 21 | 22 | primary-vpc: 23 | @echo "Installing resources into primary region..." 24 | @echo "Installing VPC infrastructure Cloudformation stack..." 25 | $(eval result:=$(shell aws cloudformation create-stack --stack-name regionalVPC$(ENV) --capabilities CAPABILITY_IAM --region $(PRIMARY_REGION) --output text --template-body file://regionalVpc.yml --parameters ParameterKey=Env,ParameterValue=$(ENV))) 26 | @aws cloudformation wait stack-create-complete --stack-name regionalVPC$(ENV) --region $(PRIMARY_REGION) 27 | 28 | primary-app: 29 | @echo "Building and Deploying Application..." 30 | sam build -t ./samTemplate.yaml 31 | sam deploy --resolve-s3 --stack-name batch$(ENV) --capabilities CAPABILITY_IAM --region $(PRIMARY_REGION) --no-confirm-changeset --parameter-overrides Env=$(ENV) SESSender=$(SES_IDENTITY) SESRecipient=$(SES_IDENTITY) SESIdentityName=$(SES_IDENTITY) SourceBucket=$(shell aws cloudformation list-stack-resources --region $(PRIMARY_REGION) --stack-name regionalVPC$(ENV) --query "StackResourceSummaries[?LogicalResourceId=='SourceBucket'].{PhysicalResourceId: PhysicalResourceId}" --output text) --disable-rollback 32 | 33 | secondary-app: 34 | @echo "Building and Deploying Application..." 35 | sam deploy --resolve-s3 --stack-name batch$(ENV) --capabilities CAPABILITY_IAM --region $(SECONDARY_REGION) --no-confirm-changeset --parameter-overrides Env=$(ENV) SESSender=$(SES_IDENTITY) SESRecipient=$(SES_IDENTITY) SESIdentityName=$(SES_IDENTITY) SourceBucket=$(shell aws cloudformation list-stack-resources --region $(SECONDARY_REGION) --stack-name regionalVPC$(ENV) --query "StackResourceSummaries[?LogicalResourceId=='SourceBucket'].{PhysicalResourceId: PhysicalResourceId}" --output text) --disable-rollback 36 | 37 | secondary-vpc: 38 | @echo "Installing resources into secondary region..." 39 | @echo "Installing VPC infrastructure Cloudformation stack..." 40 | $(eval result:=$(shell aws cloudformation create-stack --stack-name regionalVPC$(ENV) --capabilities CAPABILITY_IAM --region $(SECONDARY_REGION) --output text --template-body file://regionalVpc.yml --parameters ParameterKey=Env,ParameterValue=$(ENV))) 41 | @aws cloudformation wait stack-create-complete --stack-name regionalVPC$(ENV) --region $(SECONDARY_REGION) 42 | 43 | global-routing: 44 | @echo "Creating global routing resources..." 45 | $(eval result:=$(shell aws cloudformation create-stack --stack-name global-routing$(ENV) --region $(PRIMARY_REGION) --output text --template-body file://globalRouting.yml --capabilities CAPABILITY_IAM --parameters ParameterKey=Env,ParameterValue=$(ENV) ParameterKey=PrimaryRegion,ParameterValue=$(PRIMARY_REGION) ParameterKey=SecondaryRegion,ParameterValue=$(SECONDARY_REGION) ParameterKey=PrimaryRegionBucketSecretName,ParameterValue=SourceBucket-$(PRIMARY_REGION)$(ENV) ParameterKey=SecondaryRegionBucketSecretName,ParameterValue=SourceBucket-$(SECONDARY_REGION)$(ENV) )) 46 | @aws cloudformation wait stack-create-complete --stack-name global-routing$(ENV) --region $(PRIMARY_REGION) 47 | 48 | dashboard: 49 | @echo "Creating dashboard..." 50 | $(eval result:=$(shell aws cloudformation create-stack --stack-name dashboard$(ENV) --region $(PRIMARY_REGION) --output text --template-body file://dashboard-template.yml --capabilities CAPABILITY_IAM --parameters ParameterKey=Env,ParameterValue=$(ENV) ParameterKey=PrimaryRegion,ParameterValue=$(PRIMARY_REGION) ParameterKey=SecondaryRegion,ParameterValue=$(SECONDARY_REGION) )) 51 | @aws cloudformation wait stack-create-complete --stack-name dashboard$(ENV) --region $(PRIMARY_REGION) 52 | @echo "Deploy Complete!" 53 | 54 | chaos-primary: 55 | @echo "Creating Chaos Testing resources in Primary Region..." 56 | $(eval result:=$(shell aws cloudformation create-stack --stack-name chaos$(ENV) --region $(PRIMARY_REGION) --output text --template-body file://fisTemplate.yml --capabilities CAPABILITY_IAM)) 57 | @aws cloudformation wait stack-create-complete --stack-name chaos$(ENV) --region $(PRIMARY_REGION) 58 | @echo "Deploy Complete in Primary Region!" 59 | 60 | chaos-secondary: 61 | @echo "Creating Chaos Testing resources in Secondary Region..." 62 | $(eval result:=$(shell aws cloudformation create-stack --stack-name chaos$(ENV) --region $(SECONDARY_REGION) --output text --template-body file://fisTemplate.yml --capabilities CAPABILITY_IAM)) 63 | @aws cloudformation wait stack-create-complete --stack-name chaos$(ENV) --region $(SECONDARY_REGION) 64 | @echo "Deploy Complete in Secondary Region!" 65 | 66 | deploy: test-creds global primary-vpc secondary-vpc global-routing primary-app secondary-app dashboard 67 | 68 | chaos: test-creds chaos-primary chaos-secondary 69 | 70 | destroy-chaos: 71 | @echo "Removing Chaos Resources..." 72 | @aws cloudformation delete-stack --stack-name chaos$(ENV) --region $(PRIMARY_REGION) 73 | @aws cloudformation wait stack-delete-complete --stack-name chaos$(ENV) --region $(PRIMARY_REGION) 74 | @echo "Destroyed Chaos Resources in Primary Region!" 75 | 76 | @aws cloudformation delete-stack --stack-name chaos$(ENV) --region $(SECONDARY_REGION) 77 | @aws cloudformation wait stack-delete-complete --stack-name chaos$(ENV) --region $(SECONDARY_REGION) 78 | @echo "Destroyed Chaos Resources in Secondary Region!" 79 | 80 | destroy-all: 81 | @echo "Removing all cloudformation stacks!!" 82 | @aws cloudformation delete-stack --stack-name dashboard$(ENV) --region $(PRIMARY_REGION) 83 | @aws cloudformation wait stack-delete-complete --stack-name dashboard$(ENV) --region $(PRIMARY_REGION) 84 | 85 | @echo "Removing application from secondary region" 86 | @-sam delete --stack-name batch$(ENV) --region $(SECONDARY_REGION) --no-prompts 87 | 88 | @echo "Removing application from primary region" 89 | @-sam delete --stack-name batch$(ENV) --region $(PRIMARY_REGION) --no-prompts 90 | 91 | @echo "Removing global routing resources" 92 | @aws cloudformation delete-stack --stack-name global-routing$(ENV) --region $(PRIMARY_REGION) 93 | @aws cloudformation wait stack-delete-complete --stack-name global-routing$(ENV) --region $(PRIMARY_REGION) 94 | 95 | @echo "Cleaning up S3 Buckets..." 96 | $(eval result:=$(shell aws cloudformation list-stack-resources --region $(SECONDARY_REGION) --stack-name regionalVPC$(ENV) --query "StackResourceSummaries[?LogicalResourceId=='LoggingBucket'].{PhysicalResourceId: PhysicalResourceId}" --output text |cut -d: -f6 )) 97 | ./cleanup.sh $(result) 98 | 99 | $(eval result:=$(shell aws cloudformation list-stack-resources --region $(SECONDARY_REGION) --stack-name regionalVPC$(ENV) --query "StackResourceSummaries[?LogicalResourceId=='SourceBucket'].{PhysicalResourceId: PhysicalResourceId}" --output text |cut -d: -f6 )) 100 | ./cleanup.sh $(result) 101 | 102 | $(eval result:=$(shell aws cloudformation list-stack-resources --region $(PRIMARY_REGION) --stack-name regionalVPC$(ENV) --query "StackResourceSummaries[?LogicalResourceId=='LoggingBucket'].{PhysicalResourceId: PhysicalResourceId}" --output text |cut -d: -f6 )) 103 | ./cleanup.sh $(result) 104 | 105 | $(eval result:=$(shell aws cloudformation list-stack-resources --region $(PRIMARY_REGION) --stack-name regionalVPC$(ENV) --query "StackResourceSummaries[?LogicalResourceId=='SourceBucket'].{PhysicalResourceId: PhysicalResourceId}" --output text |cut -d: -f6 )) 106 | ./cleanup.sh $(result) 107 | @echo "S3 Buckets emptied..." 108 | 109 | @echo "Remove primary regional stacks..." 110 | @aws cloudformation delete-stack --stack-name regionalVPC$(ENV) --region $(PRIMARY_REGION) 111 | @aws cloudformation wait stack-delete-complete --stack-name regionalVPC$(ENV) --region $(PRIMARY_REGION) 112 | 113 | @echo "Remove secondary regional stacks..." 114 | @aws cloudformation delete-stack --stack-name regionalVPC$(ENV) --region $(SECONDARY_REGION) 115 | @aws cloudformation wait stack-delete-complete --stack-name regionalVPC$(ENV) --region $(SECONDARY_REGION) 116 | 117 | @echo "Remove global stack..." 118 | @aws cloudformation delete-stack --stack-name global-base$(ENV) --region $(PRIMARY_REGION) 119 | @aws cloudformation wait stack-delete-complete --stack-name global-base$(ENV) --region $(PRIMARY_REGION) 120 | 121 | clean: 122 | @echo "To remove all stacks deployed by this solution, run 'make destroy-all'" 123 | 124 | test-creds: 125 | @echo "Current AWS session:" 126 | @aws sts get-caller-identity -------------------------------------------------------------------------------- /deployment/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #set -xv 3 | BUCKET=$1 4 | # test if an s3 bucket exists and delete all objects and delete markers from it 5 | # if it does not exist, just print a message 6 | bucketExists=`aws s3api list-buckets --query "Buckets[].Name" | grep -w "$BUCKET" | wc -l` 7 | 8 | if [ "$bucketExists" -eq 1 ] 9 | then 10 | objects=$(aws s3api list-object-versions --bucket "$BUCKET" --no-paginate --output=json --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}') 11 | count=$(awk '/"Objects": null/ {print}' <<< "$objects" | wc -l) 12 | while [ "$count" -ne 1 ]; 13 | do 14 | echo 'Deleting Object Versions...' 15 | aws s3api delete-objects --bucket "$BUCKET" --delete "$objects" --no-cli-pager 16 | objects=$(aws s3api list-object-versions --bucket "$BUCKET" --no-paginate --output=json --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}') 17 | count=$(awk '/"Objects": null/ {print}' <<< "$objects" | wc -l) 18 | done 19 | 20 | deleteMarkers=$(aws s3api list-object-versions --bucket "$BUCKET" --no-paginate --output=json --query='{Objects: DeleteMarkers[].{Key:Key,VersionId:VersionId}}') 21 | deleteMarkersCount=$(awk '/"Objects": null/ {print}' <<< "$deleteMarkers" | wc -l) 22 | while [ "$deleteMarkersCount" -ne 1 ]; 23 | do 24 | echo 'Deleting DeleteMarkers...' 25 | aws s3api delete-objects --bucket "$BUCKET" --delete "$deleteMarkers" --no-cli-pager 26 | deleteMarkers=$(aws s3api list-object-versions --bucket "$BUCKET" --no-paginate --output=json --query='{Objects: DeleteMarkers[].{Key:Key,VersionId:VersionId}}') 27 | deleteMarkersCount=$(awk '/"Objects": null/ {print}' <<< "$deleteMarkers" | wc -l) 28 | done 29 | else 30 | echo "Bucket $BUCKET does not exist..." 31 | fi -------------------------------------------------------------------------------- /deployment/dashboard-template.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | Description: '(SO9169) Cloudwatch Dashboard Stack' 3 | Parameters: 4 | Env: 5 | Type: String 6 | Default: '' 7 | Description: String to enable multiple deployments per AWS region 8 | PrimaryRegion: 9 | Type: String 10 | Description: Enter the Primary Region 11 | Default: "us-east-1" 12 | SecondaryRegion: 13 | Type: String 14 | Description: Enter the Secondary Region 15 | Default: "us-west-2" 16 | 17 | Resources: 18 | MultiRegionBatchDashboard: 19 | Type: AWS::CloudWatch::Dashboard 20 | Properties: 21 | DashboardName: !Sub MultiRegionBatchDashboard${Env} 22 | DashboardBody: !Sub '{ 23 | "widgets": [ 24 | { 25 | "height": 6, 26 | "width": 6, 27 | "y": 0, 28 | "x": 0, 29 | "type": "metric", 30 | "properties": { 31 | "view": "timeSeries", 32 | "stacked": false, 33 | "metrics": [ 34 | [ "MultiRegionBatch${Env}", "InputFilesSplit", "service", "SplitInputFileFunction${Env}" ] 35 | ], 36 | "region": "${PrimaryRegion}", 37 | "title": "${PrimaryRegion} - Number of Input Files Split for Processing", 38 | "period": 1, 39 | "stat": "Sum" 40 | } 41 | }, 42 | { 43 | "height": 6, 44 | "width": 6, 45 | "y": 0, 46 | "x": 6, 47 | "type": "metric", 48 | "properties": { 49 | "view": "timeSeries", 50 | "stacked": false, 51 | "metrics": [ 52 | [ "MultiRegionBatch${Env}", "EmailSent", "service", "SendEmailFunction${Env}" ] 53 | ], 54 | "region": "${PrimaryRegion}", 55 | "title": "${PrimaryRegion} - Number of Processed Files", 56 | "period": 1, 57 | "stat": "Sum" 58 | } 59 | }, 60 | { 61 | "height": 6, 62 | "width": 6, 63 | "y": 0, 64 | "x": 12, 65 | "type": "metric", 66 | "properties": { 67 | "metrics": [ 68 | [ "AWS/States", "ExecutionsSucceeded", "StateMachineArn", "arn:aws:states:${PrimaryRegion}:${AWS::AccountId}:stateMachine:BlogBatchMainOrchestrator${Env}" ], 69 | [ ".", "ExecutionsFailed", ".", "." ], 70 | [ ".", "ExecutionsStarted", ".", "." ] 71 | ], 72 | "view": "timeSeries", 73 | "stacked": false, 74 | "region": "${PrimaryRegion}", 75 | "period": 1, 76 | "stat": "Sum", 77 | "title": "${PrimaryRegion} - Main Orchestrator Step Function Metrics" 78 | } 79 | }, 80 | { 81 | "height": 6, 82 | "width": 6, 83 | "y": 0, 84 | "x": 18, 85 | "type": "metric", 86 | "properties": { 87 | "view": "timeSeries", 88 | "stacked": false, 89 | "metrics": [ 90 | [ "AWS/States", "ExecutionsSucceeded", "StateMachineArn", "arn:aws:states:${PrimaryRegion}:${AWS::AccountId}:stateMachine:BlogBatchProcessChunk${Env}" ], 91 | [ ".", "ExecutionsFailed", ".", "." ], 92 | [ ".", "ExecutionsStarted", ".", "." ] 93 | ], 94 | "region": "${PrimaryRegion}", 95 | "title": "${PrimaryRegion} - Chunk File Processor Step Function Metrics", 96 | "period": 1, 97 | "stat": "Sum" 98 | } 99 | }, 100 | { 101 | "height": 6, 102 | "width": 6, 103 | "y": 6, 104 | "x": 0, 105 | "type": "metric", 106 | "properties": { 107 | "view": "timeSeries", 108 | "stacked": false, 109 | "metrics": [ 110 | [ "MultiRegionBatch${Env}", "InputFilesSplit", "service", "SplitInputFileFunction${Env}" ] 111 | ], 112 | "region": "${SecondaryRegion}", 113 | "title": "${SecondaryRegion} - Number of Input Files Split for Processing", 114 | "period": 1, 115 | "stat": "Sum" 116 | } 117 | }, 118 | { 119 | "type": "metric", 120 | "x": 6, 121 | "y": 6, 122 | "width": 6, 123 | "height": 6, 124 | "properties": { 125 | "view": "timeSeries", 126 | "stacked": false, 127 | "metrics": [ 128 | [ "MultiRegionBatch${Env}", "EmailSent", "service", "SendEmailFunction${Env}" ] 129 | ], 130 | "region": "${SecondaryRegion}", 131 | "title": "${SecondaryRegion} - Number of Processed Files", 132 | "period": 1, 133 | "stat": "Sum" 134 | } 135 | }, 136 | { 137 | "type": "metric", 138 | "x": 12, 139 | "y": 6, 140 | "width": 6, 141 | "height": 6, 142 | "properties": { 143 | "view": "timeSeries", 144 | "stacked": false, 145 | "metrics": [ 146 | [ "AWS/States", "ExecutionsStarted", "StateMachineArn", "arn:aws:states:${SecondaryRegion}:${AWS::AccountId}:stateMachine:BlogBatchMainOrchestrator${Env}" ], 147 | [ ".", "ExecutionsSucceeded", ".", "." ], 148 | [ ".", "ExecutionsFailed", ".", "." ] 149 | ], 150 | "region": "${SecondaryRegion}", 151 | "title": "${SecondaryRegion} - Main Orchestrator Step Function Metrics", 152 | "period": 1, 153 | "stat": "Sum" 154 | } 155 | }, 156 | { 157 | "type": "metric", 158 | "x": 18, 159 | "y": 6, 160 | "width": 6, 161 | "height": 6, 162 | "properties": { 163 | "view": "timeSeries", 164 | "stacked": false, 165 | "metrics": [ 166 | [ "AWS/States", "ExecutionsSucceeded", "StateMachineArn", "arn:aws:states:${SecondaryRegion}:${AWS::AccountId}:stateMachine:BlogBatchProcessChunk${Env}" ], 167 | [ ".", "ExecutionsStarted", ".", "." ], 168 | [ ".", "ExecutionsFailed", ".", "." ] 169 | ], 170 | "region": "${SecondaryRegion}", 171 | "title": "${SecondaryRegion} - Chunk File Processor Step Function Metrics", 172 | "period": 1, 173 | "stat": "Sum" 174 | } 175 | }, 176 | { 177 | "type": "metric", 178 | "x": 6, 179 | "y": 12, 180 | "width": 6, 181 | "height": 6, 182 | "properties": { 183 | "view": "timeSeries", 184 | "stacked": false, 185 | "metrics": [ 186 | [ "MultiRegionBatch${Env}", "ReconciledFiles", "service", "AutomationReconciliationFunction${Env}" ] 187 | ], 188 | "region": "${SecondaryRegion}", 189 | "title": "${SecondaryRegion} - Number of files reconciled", 190 | "period": 1, 191 | "stat": "Sum" 192 | } 193 | }, 194 | { 195 | "type": "metric", 196 | "x": 0, 197 | "y": 12, 198 | "width": 6, 199 | "height": 6, 200 | "properties": { 201 | "view": "timeSeries", 202 | "stacked": false, 203 | "metrics": [ 204 | [ "MultiRegionBatch${Env}", "ReconciledFiles", "service", "AutomationReconciliationFunction${Env}" ] 205 | ], 206 | "region": "${PrimaryRegion}", 207 | "title": "${PrimaryRegion} - Number of files reconciled", 208 | "period": 1, 209 | "stat": "Sum" 210 | } 211 | } 212 | ] 213 | }' -------------------------------------------------------------------------------- /deployment/fisTemplate.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | Description: '(SO9169) This template will perform network distruption and creates DENY for various scopes of AWS structured boundaries/services. ' 3 | Parameters: 4 | FISExperimentSubnetTag: 5 | Description: 'VPC Subnet that will be blocked for specific network or service traffic. It will use Network ACL(NACL) DENY rules to DENY. Please enter the value of the Name tag' 6 | Type: String 7 | Default: mr-batch-Private 8 | PrefixListIdentifier: 9 | Description: 'The identifier of the Managed Prefix List (either ARN, ID or Name) to block at the start of the experiment. This is ONLY required if prefix-list is the SCOPE ' 10 | Type: String 11 | Default: '' 12 | Scope: 13 | Description: 'The source traffic to block (inbound and outbound )for the subnet. Possible options are: all, availability-zone, prefix-list, s3, dynamodb, vpc' 14 | Type: String 15 | Default: s3 16 | AllowedValues: 17 | - all 18 | - availability-zone 19 | - s3 20 | - dynamodb 21 | - vpc 22 | ExperimentDuration: 23 | Description: 'The time duration to execute the experiment. This is in the format of PT1M (1 minute )' 24 | Type: String 25 | Default: PT20M 26 | ExperimentTemplateName: 27 | Description: 'Name of the experiment template. This should be unique for each experiment' 28 | Type: String 29 | Default: MultiRegionBatchChaosTemplate 30 | 31 | Resources: 32 | FISLogGroupKey: 33 | Type: AWS::KMS::Key 34 | Properties: 35 | Enabled: true 36 | EnableKeyRotation: true 37 | KeyPolicy: 38 | Version: 2012-10-17 39 | Id: key-loggroup 40 | Statement: 41 | - Sid: Enable IAM User Permissions 42 | Effect: Allow 43 | Principal: 44 | AWS: !Join 45 | - '' 46 | - - !Sub 'arn:${AWS::Partition}:iam::' 47 | - !Ref 'AWS::AccountId' 48 | - ':root' 49 | Action: 'kms:*' 50 | Resource: '*' 51 | - Sid: Enable Cloudwatch access 52 | Effect: Allow 53 | Principal: 54 | Service: !Sub "logs.${AWS::Region}.amazonaws.com" 55 | Action: 56 | - kms:Encrypt* 57 | - kms:Decrypt* 58 | - kms:ReEncrypt* 59 | - kms:GenerateDataKey* 60 | - kms:Describe* 61 | Resource: '*' 62 | 63 | FISExperimentLogGroup: 64 | Type: 'AWS::Logs::LogGroup' 65 | Properties: 66 | KmsKeyId: !GetAtt FISLogGroupKey.Arn 67 | LogGroupName: 68 | !Join 69 | - '' 70 | - - !Ref ExperimentTemplateName 71 | - 'logs' 72 | RetentionInDays: 7 73 | 74 | FISBlockingRole: 75 | Type: 'AWS::IAM::Role' 76 | Properties: 77 | AssumeRolePolicyDocument: 78 | Version: 2012-10-17 79 | Statement: 80 | - Effect: Allow 81 | Principal: 82 | Service: [ fis.amazonaws.com ] 83 | Action: [ "sts:AssumeRole" ] 84 | Path: / 85 | Policies: 86 | - PolicyName: FISExperimentTemplate-BlockNetwork 87 | PolicyDocument: 88 | Version: "2012-10-17" 89 | Statement: 90 | - Effect: Allow 91 | Action: 92 | - "logs:CreateLogGroup" 93 | - "logs:CreateLogStream" 94 | - "logs:PutLogEvents" 95 | - "logs:DescribeLogGroups" 96 | - "logs:DescribeResourcePolicies" 97 | - "logs:PutResourcePolicy" 98 | - "logs:CreateLogDelivery" 99 | Resource: '*' 100 | Condition: 101 | StringEquals: 102 | "aws:PrincipalAccount": !Sub "${AWS::AccountId}" 103 | "aws:ResourceAccount": !Sub "${AWS::AccountId}" 104 | MaxSessionDuration: 3600 105 | ManagedPolicyArns: 106 | - 'arn:aws:iam::aws:policy/service-role/AWSFaultInjectionSimulatorNetworkAccess' 107 | 108 | FISBlockNetwork: 109 | Type: 'AWS::FIS::ExperimentTemplate' 110 | Properties: 111 | Description: 'This experiment is created to block various services within (DENY inbound and outbound) a Subnet' 112 | Targets: 113 | Subnets-deny: 114 | ResourceType: 'aws:ec2:subnet' 115 | ResourceTags: 116 | Name: !Ref FISExperimentSubnetTag 117 | SelectionMode: 'ALL' 118 | Actions: 119 | DenyNetwork: 120 | ActionId: 'aws:network:disrupt-connectivity' 121 | Parameters: 122 | duration: !Ref ExperimentDuration 123 | prefixListIdentifier: !Ref PrefixListIdentifier 124 | scope: !Ref Scope 125 | Targets: 126 | Subnets: 'Subnets-deny' 127 | StopConditions: 128 | - Source: 'none' 129 | LogConfiguration: 130 | CloudWatchLogsConfiguration: 131 | LogGroupArn: !GetAtt FISExperimentLogGroup.Arn 132 | LogSchemaVersion: 1 133 | RoleArn: !GetAtt FISBlockingRole.Arn 134 | Tags: 135 | Name: !Ref ExperimentTemplateName -------------------------------------------------------------------------------- /deployment/globalResources.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | Description: '(SO9169) Global Resources Stack' 3 | Transform: 4 | - AWS::LanguageExtensions 5 | Parameters: 6 | Env: 7 | Type: String 8 | Default: '' 9 | Description: String to enable multiple deployments per AWS region 10 | PrimaryRegion: 11 | Type: String 12 | Description: Enter the Primary Region 13 | Default: "us-east-1" 14 | SecondaryRegion: 15 | Type: String 16 | Description: Enter the Secondary Region 17 | Default: "us-west-2" 18 | 19 | Resources: 20 | ##### Application Recovery Controller####### 21 | ArcCluster: 22 | Type: AWS::Route53RecoveryControl::Cluster 23 | Properties: 24 | Name: !Sub BatchFailoverArcCluster${Env} 25 | ArcControlPanel: 26 | Type: AWS::Route53RecoveryControl::ControlPanel 27 | Properties: 28 | ClusterArn: !GetAtt ArcCluster.ClusterArn 29 | Name: !Sub BatchFailoverArcControlPanel${Env} 30 | ArcRoutingControl: 31 | Type: AWS::Route53RecoveryControl::RoutingControl 32 | Properties: 33 | ClusterArn: !GetAtt ArcCluster.ClusterArn 34 | ControlPanelArn: !GetAtt ArcControlPanel.ControlPanelArn 35 | Name: !Sub BatchFailoverArcRoutingControl${Env} 36 | ArcHealthCheck: 37 | Type: AWS::Route53::HealthCheck 38 | Properties: 39 | HealthCheckConfig: 40 | Type: RECOVERY_CONTROL 41 | RoutingControlArn: !GetAtt ArcRoutingControl.RoutingControlArn 42 | HealthCheckTags: 43 | - Key: Name 44 | Value: !Sub BatchFailoverArcHealthCheck${Env} 45 | ArcClusterSecret: 46 | Type: AWS::SecretsManager::Secret 47 | Properties: 48 | Name: !Sub ArcClusterSecret${Env} 49 | Description: "ARC Cluster ARN" 50 | KmsKeyId: "alias/aws/secretsmanager" 51 | SecretString: !Ref ArcCluster 52 | ReplicaRegions: 53 | - Region: !Ref SecondaryRegion 54 | ArcRoutingControlSecret: 55 | Type: AWS::SecretsManager::Secret 56 | Properties: 57 | Name: !Sub ArcRoutingControlSecret${Env} 58 | Description: "ARC Routing Control ARN" 59 | KmsKeyId: "alias/aws/secretsmanager" 60 | SecretString: !Ref ArcRoutingControl 61 | ReplicaRegions: 62 | - Region: !Ref SecondaryRegion 63 | ArcHealthCheckId: 64 | Type: AWS::SSM::Parameter 65 | Properties: 66 | Type: String 67 | Name: !Sub ArcHealthCheckId${Env} 68 | Value: !Ref ArcHealthCheck 69 | LambdaExecutionRole: 70 | Properties: 71 | AssumeRolePolicyDocument: 72 | Statement: 73 | - 74 | Action: 75 | - "sts:AssumeRole" 76 | Effect: Allow 77 | Principal: 78 | Service: 79 | - lambda.amazonaws.com 80 | Version: "2012-10-17" 81 | Path: / 82 | Policies: 83 | - 84 | PolicyDocument: 85 | Statement: 86 | - Effect: Allow 87 | Action: 88 | - "logs:CreateLogGroup" 89 | - "logs:CreateLogStream" 90 | - "logs:PutLogEvents" 91 | Resource: !Sub 'arn:aws:logs:*:${AWS::AccountId}:*' 92 | - Effect: Allow 93 | Action: 94 | - "route53-recovery-cluster:GetRoutingControlState" 95 | - "route53-recovery-cluster:ListRoutingControls" 96 | - "route53-recovery-cluster:UpdateRoutingControlState" 97 | - "route53-recovery-cluster:UpdateRoutingControlStates" 98 | - "route53-recovery-control-config:DescribeCluster" 99 | - "route53-recovery-control-config:DescribeControlPanel" 100 | - "route53-recovery-control-config:DescribeRoutingControl" 101 | - "route53-recovery-control-config:UpdateControlPanel" 102 | - "route53-recovery-control-config:UpdateRoutingControl" 103 | Resource: 104 | - !Sub 'arn:aws:route53-recovery-control::${AWS::AccountId}:cluster/*' 105 | - !Sub 'arn:aws:route53-recovery-control::${AWS::AccountId}:controlpanel/*' 106 | - Effect: Allow 107 | Action: 108 | - "secretsmanager:GetSecretValue" 109 | - "secretsmanager:PutSecretValue" 110 | - "secretsmanager:CreateSecret" 111 | - "secretsmanager:UpdateSecret" 112 | - "secretsmanager:DeleteSecret" 113 | - "secretsmanager:RemoveRegionsFromReplication" 114 | - "secretsmanager:ReplicateSecretToRegions" 115 | - "secretsmanager:ListSecrets" 116 | Resource: !Sub 'arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:*' 117 | Version: "2012-10-17" 118 | PolicyName: root 119 | Type: "AWS::IAM::Role" 120 | CustomBackedLambda: 121 | Type: AWS::Lambda::Function 122 | Properties: 123 | FunctionName: !Sub ArcClusterManagementFunction${Env} 124 | Runtime: python3.9 125 | Role: !GetAtt LambdaExecutionRole.Arn 126 | Handler: index.lambda_handler 127 | Timeout: 90 128 | Environment: 129 | Variables: 130 | ARC_CLUSTER_ARN: !Ref ArcCluster 131 | ARC_ROUTING_CONTROL_ARN: !Ref ArcRoutingControl 132 | ARC_CLUSTER_ENDPOINTS_SECRET: !Sub ArcClusterEndpoints${Env} 133 | REPLICA_REGION: !Ref SecondaryRegion 134 | Code: 135 | ZipFile: | 136 | import cfnresponse 137 | import logging 138 | import boto3 139 | import os 140 | import json 141 | # Init of the logging module 142 | logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') 143 | 144 | 145 | def lambda_handler(event, context): 146 | print(event) 147 | responseData = {} 148 | try: 149 | client = boto3.client('route53-recovery-control-config', region_name='us-west-2') 150 | cluster = client.describe_cluster(ClusterArn=os.environ['ARC_CLUSTER_ARN']) 151 | endpoints = cluster['Cluster']['ClusterEndpoints'] 152 | regions = ["us-east-1", "us-west-2", "eu-west-1", "ap-northeast-1", "ap-southeast-2"] 153 | sorted_endpoints = {} 154 | for region in regions: 155 | for endpoint in endpoints: 156 | if endpoint["Region"] == region: 157 | sorted_endpoints[region] = endpoint["Endpoint"] 158 | responseData['cluster_endpoints'] = json.dumps(sorted_endpoints) 159 | client = boto3.client('secretsmanager', region_name=os.environ['AWS_REGION']) 160 | try: 161 | if (event['RequestType'] == 'Update'): 162 | client.describe_secret(SecretId=os.environ['ARC_CLUSTER_ENDPOINTS_SECRET']) 163 | responseData['put_secret_value_response'] = client.put_secret_value( 164 | SecretId=os.environ['ARC_CLUSTER_ENDPOINTS_SECRET'], 165 | SecretString=json.dumps(sorted_endpoints), 166 | ) 167 | logging.info('Cluster Endpoints secret updated') 168 | elif (event['RequestType'] == 'Create'): 169 | responseData['create_secret_response'] = client.create_secret( 170 | Description='ARC Cluster Endpoints', 171 | Name=os.environ['ARC_CLUSTER_ENDPOINTS_SECRET'], 172 | SecretString=json.dumps(sorted_endpoints), 173 | AddReplicaRegions=[{'Region': os.environ['REPLICA_REGION']}] 174 | ) 175 | logging.info('Cluster Endpoints secret created') 176 | elif (event['RequestType'] == 'Delete'): 177 | responseData['remove_replica_region_response'] = client.remove_regions_from_replication( 178 | SecretId=os.environ['ARC_CLUSTER_ENDPOINTS_SECRET'], 179 | RemoveReplicaRegions=[ 180 | os.environ['REPLICA_REGION'], 181 | ] 182 | ) 183 | responseData['delete_secret_response'] = client.delete_secret( 184 | SecretId=os.environ['ARC_CLUSTER_ENDPOINTS_SECRET'], 185 | ForceDeleteWithoutRecovery=True 186 | ) 187 | logging.info('Cluster Endpoints secret deleted') 188 | else: 189 | logging.error('Unsupported Stack Operation') 190 | except Exception as err: 191 | logging.error(err) 192 | responseData['secret_operation_response'] = 'Failed' 193 | updated_routing_control_state = "NotUpdated" 194 | done = False 195 | for region in regions: 196 | for endpoint in endpoints: 197 | if endpoint["Region"] == region: 198 | 199 | try: 200 | logging.info("route 53 recovery cluster endpoint: " + endpoint["Endpoint"]) 201 | client = boto3.client('route53-recovery-cluster', region_name=region, endpoint_url=endpoint["Endpoint"]) 202 | 203 | logging.info("toggling routing control") 204 | routing_control_state = client.get_routing_control_state(RoutingControlArn=os.environ['ARC_ROUTING_CONTROL_ARN']) 205 | logging.info("Current Routing Control State: " + routing_control_state["RoutingControlState"]) 206 | if routing_control_state["RoutingControlState"] == "On": 207 | logging.info("Routing Control State is ON") 208 | done = True 209 | break 210 | else: 211 | client.update_routing_control_state(RoutingControlArn=os.environ['ARC_ROUTING_CONTROL_ARN'], RoutingControlState="On") 212 | routing_control_state = client.get_routing_control_state(RoutingControlArn=os.environ['ARC_ROUTING_CONTROL_ARN']) 213 | updated_routing_control_state = routing_control_state["RoutingControlState"] 214 | logging.info("Updated routing Control State is " + updated_routing_control_state) 215 | done = True 216 | break 217 | except Exception as e: 218 | logging.error(e) 219 | if done: 220 | break 221 | responseData['routing_control_state'] = updated_routing_control_state 222 | responseData['message'] = 'Success' 223 | logging.info('Sending %s to cloudformation', responseData['message']) 224 | cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) 225 | except Exception as err: 226 | responseData['message'] = 'Failed' 227 | logging.error(err) 228 | logging.info('Sending %s to cloudformation', responseData['message']) 229 | cfnresponse.send(event, context, cfnresponse.FAILED, responseData) 230 | Description: Function to setup ARC Cluster endpoint secrets and rotate arc controls 231 | 232 | InvokeCustomLambda: 233 | DependsOn: CustomBackedLambda 234 | Type: Custom::InvokeCustomLambda 235 | Properties: 236 | ServiceToken: !GetAtt CustomBackedLambda.Arn 237 | ##### DynamoDB Global Table for Batch State ####### 238 | BatchStateTable: 239 | Type: AWS::DynamoDB::GlobalTable 240 | Properties: 241 | BillingMode: PAY_PER_REQUEST 242 | AttributeDefinitions: 243 | - AttributeName: "fileName" 244 | AttributeType: "S" 245 | - AttributeName: "status" 246 | AttributeType: "S" 247 | KeySchema: 248 | - AttributeName: "fileName" 249 | KeyType: "HASH" 250 | Replicas: 251 | - Region: !Ref PrimaryRegion 252 | PointInTimeRecoverySpecification: 253 | PointInTimeRecoveryEnabled: true 254 | - Region: !Ref SecondaryRegion 255 | PointInTimeRecoverySpecification: 256 | PointInTimeRecoveryEnabled: true 257 | StreamSpecification: 258 | StreamViewType: NEW_AND_OLD_IMAGES 259 | SSESpecification: 260 | SSEEnabled: true 261 | GlobalSecondaryIndexes: 262 | - IndexName: status-index 263 | KeySchema: 264 | - AttributeName: status 265 | KeyType: HASH 266 | Projection: 267 | ProjectionType: ALL 268 | BatchStateTableSecret: 269 | Type: AWS::SecretsManager::Secret 270 | Properties: 271 | Name: !Sub BatchStateTableNameSecret${Env} 272 | Description: "DDB Batch State Table" 273 | KmsKeyId: "alias/aws/secretsmanager" 274 | SecretString: !Ref BatchStateTable 275 | ReplicaRegions: 276 | - Region: !Ref SecondaryRegion 277 | ##### SMTP Credentials for SES Service ####### 278 | SmtpIamUserGroup: 279 | Type: AWS::IAM::Group 280 | Properties: 281 | ManagedPolicyArns: 282 | - "arn:aws:iam::aws:policy/AmazonSESFullAccess" 283 | Path: '/' 284 | SmtpIamUser: 285 | Type: AWS::IAM::User 286 | Properties: 287 | Path: '/' 288 | SmtpAddUserToGroup: 289 | Type: AWS::IAM::UserToGroupAddition 290 | Properties: 291 | GroupName: !Ref SmtpIamUserGroup 292 | Users: 293 | - !Ref SmtpIamUser 294 | SmtpIamUserCredentials: 295 | Type: AWS::IAM::AccessKey 296 | Properties: 297 | UserName: !Ref SmtpIamUser 298 | 299 | SmtpUserCredentials: 300 | Type: AWS::SecretsManager::Secret 301 | Properties: 302 | Name: !Sub SmtpCredentialsSecret${Env} 303 | Description: "SMTP User Access Keys" 304 | KmsKeyId: "alias/aws/secretsmanager" 305 | ReplicaRegions: 306 | - Region: !Ref SecondaryRegion 307 | SecretString: 308 | Fn::ToJsonString: 309 | AccessKey: !Ref SmtpIamUserCredentials 310 | SecretAccessKey: !GetAtt SmtpIamUserCredentials.SecretAccessKey 311 | ##### Regional Failover Automation Role ####### 312 | AutomationServiceRole: 313 | Type: AWS::IAM::Role 314 | Properties: 315 | AssumeRolePolicyDocument: 316 | Version: '2012-10-17' 317 | Statement: 318 | - Effect: Allow 319 | Principal: 320 | Service: 321 | - ssm.amazonaws.com 322 | Action: sts:AssumeRole 323 | Condition: 324 | StringEquals: 325 | aws:SourceAccount: !Sub ${AWS::AccountId} 326 | ArnLike: 327 | aws:SourceArn: !Sub arn:aws:ssm:*:${AWS::AccountId}:automation-execution/* 328 | ManagedPolicyArns: 329 | - arn:aws:iam::aws:policy/service-role/AmazonSSMAutomationRole 330 | Path: "/" 331 | RoleName: !Sub AutomationServiceRole${Env} 332 | Outputs: 333 | BatchStateTable: 334 | Description: "Batch State Table" 335 | Value: !Ref BatchStateTable 336 | BatchStateTableSecret: 337 | Description: "Replicated AWS Secret storing the Batch State DynamoDB Global Table's name" 338 | Value: !Ref BatchStateTableSecret 339 | ArcHealthCheck: 340 | Description: "Route53 Health Check to associate with TXT record in HostedZone stack" 341 | Value: !Ref ArcHealthCheck 342 | ArcRoutingControl: 343 | Description: "ARC Routing Control" 344 | Value: !Ref ArcRoutingControl 345 | ArcControlPanel: 346 | Description: "ARC Control Panel" 347 | Value: !Ref ArcControlPanel 348 | SmtpCredentialsSecret: 349 | Description: "SMTP User Credentials" 350 | Value: !Ref SmtpUserCredentials -------------------------------------------------------------------------------- /deployment/globalRouting.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | Description: '(SO9169) Routing Controls' 3 | 4 | Parameters: 5 | Env: 6 | Type: String 7 | Default: '' 8 | Description: String to enable multiple deployments per AWS region 9 | PrimaryRegion: 10 | Type: String 11 | Description: Enter the Primary Region 12 | Default: "us-east-1" 13 | SecondaryRegion: 14 | Type: String 15 | Description: Enter the Secondary Region 16 | Default: "us-west-2" 17 | MRAPName: 18 | Type: String 19 | Description: Enter the MRAP Name 20 | Default: "source-bucket-mrap" 21 | ReplicationRuleSetupCustomLambda: 22 | Type: String 23 | Description: Enter the MRAP Name 24 | Default: "ReplicationRuleSetupCustomLambda" 25 | PrimaryRegionBucketSecretName: 26 | Type: String 27 | Description: Enter the Secret that contains the source bucket details in the primary Region. 28 | Default: "SourceBucket-us-east-1" 29 | SecondaryRegionBucketSecretName: 30 | Type: String 31 | Description: Enter the Secret that contains the source bucket details in the secondary Region. 32 | Default: "SourceBucket-us-west-2" 33 | DomainName: 34 | Type: String 35 | Description: The name of the domain that you want created as a private hosted zone in Route53 36 | Default: demo.io 37 | 38 | Resources: 39 | HostedZone: 40 | Type: AWS::Route53::HostedZone 41 | Properties: 42 | Name: !Sub ${DomainName} 43 | VPCs: 44 | - VPCId: !Sub '{{resolve:secretsmanager:VpcId-${PrimaryRegion}${Env}}}' 45 | VPCRegion: !Ref PrimaryRegion 46 | - VPCId: !Sub '{{resolve:secretsmanager:VpcId-${SecondaryRegion}${Env}}}' 47 | VPCRegion: !Ref SecondaryRegion 48 | PrimaryRegionRecord: 49 | Type: AWS::Route53::RecordSet 50 | Properties: 51 | HostedZoneId: !Ref HostedZone 52 | Name: !Sub active.${DomainName} 53 | Failover: PRIMARY 54 | HealthCheckId: !Sub '{{resolve:ssm:ArcHealthCheckId${Env}}}' 55 | Type: TXT 56 | SetIdentifier: PrimaryRegion 57 | TTL: 60 58 | ResourceRecords: 59 | - !Sub '"${PrimaryRegion}"' 60 | SecondaryRegionRecord: 61 | Type: AWS::Route53::RecordSet 62 | Properties: 63 | HostedZoneId: !Ref HostedZone 64 | Name: !Sub active.${DomainName} 65 | Failover: SECONDARY 66 | Type: TXT 67 | SetIdentifier: SecondaryRegion 68 | TTL: 60 69 | ResourceRecords: 70 | - !Sub '"${SecondaryRegion}"' 71 | DNSRecordSecret: 72 | Type: AWS::SecretsManager::Secret 73 | Properties: 74 | Name: !Sub DNSRecordSecret${Env} 75 | Description: "Route53 DNS" 76 | KmsKeyId: "alias/aws/secretsmanager" 77 | SecretString: !Ref PrimaryRegionRecord 78 | ReplicaRegions: 79 | - Region: !Ref SecondaryRegion 80 | SourceBucketMRAP: 81 | Type: AWS::S3::MultiRegionAccessPoint 82 | Properties: 83 | Name: !Sub ${MRAPName}${Env} 84 | Regions: 85 | - Bucket: !Sub '{{resolve:secretsmanager:${PrimaryRegionBucketSecretName}:SecretString:SourceBucket}}' 86 | - Bucket: !Sub '{{resolve:secretsmanager:${SecondaryRegionBucketSecretName}:SecretString:SourceBucket}}' 87 | SourceBucketMRAPSecret: 88 | Type: AWS::SecretsManager::Secret 89 | Properties: 90 | Name: !Sub SourceBucketMRAPSecret${Env} 91 | Description: "S3 MRAP Alias" 92 | KmsKeyId: "alias/aws/secretsmanager" 93 | SecretString: !GetAtt SourceBucketMRAP.Alias 94 | ReplicaRegions: 95 | - Region: !Ref SecondaryRegion 96 | LambdaExecutionRole: 97 | Properties: 98 | AssumeRolePolicyDocument: 99 | Statement: 100 | - Action: 101 | - "sts:AssumeRole" 102 | Effect: Allow 103 | Principal: 104 | Service: 105 | - lambda.amazonaws.com 106 | Version: "2012-10-17" 107 | Path: / 108 | Policies: 109 | - PolicyDocument: 110 | Statement: 111 | - Action: 112 | - "logs:CreateLogGroup" 113 | - "logs:CreateLogStream" 114 | - "logs:PutLogEvents" 115 | Effect: Allow 116 | Resource: "arn:aws:logs:*:*:*" 117 | - Action: 118 | - "s3:PutReplicationConfiguration" 119 | - "s3:PutBucketVersioning" 120 | Effect: Allow 121 | Resource: "arn:aws:s3:::*" 122 | - Action: 123 | - "iam:PassRole" 124 | Effect: Allow 125 | Resource: !Sub "arn:aws:iam::${AWS::AccountId}:role/*" 126 | Version: "2012-10-17" 127 | PolicyName: root 128 | Type: "AWS::IAM::Role" 129 | ReplicationRole: 130 | Type: 'AWS::IAM::Role' 131 | Properties: 132 | AssumeRolePolicyDocument: 133 | Version: "2012-10-17" 134 | Statement: 135 | - Effect: Allow 136 | Principal: 137 | Service: 138 | - s3.amazonaws.com 139 | Action: 140 | - 'sts:AssumeRole' 141 | Path: / 142 | Policies: 143 | - PolicyName: ReplicationPolicy 144 | PolicyDocument: 145 | Version: "2012-10-17" 146 | Statement: 147 | - Effect: Allow 148 | Action: 149 | - "s3:GetObjectVersionForReplication" 150 | - "s3:GetObjectVersionAcl" 151 | - "s3:GetObjectVersionTagging" 152 | - "s3:ListBucket" 153 | - "s3:GetReplicationConfiguration" 154 | - "s3:ReplicateObject" 155 | - "s3:ReplicateDelete" 156 | - "s3:ReplicateTags" 157 | Resource: "arn:aws:s3:::*" 158 | CustomBackedLambda: 159 | Type: AWS::Lambda::Function 160 | Properties: 161 | FunctionName: !Sub ${ReplicationRuleSetupCustomLambda}${Env} 162 | Runtime: python3.9 163 | Role: !GetAtt LambdaExecutionRole.Arn 164 | Handler: index.lambda_handler 165 | Timeout: 90 166 | Environment: 167 | Variables: 168 | Bucket1Name: !Sub '{{resolve:secretsmanager:${PrimaryRegionBucketSecretName}:SecretString:SourceBucket}}' 169 | Bucket1Arn: !Sub '{{resolve:secretsmanager:${PrimaryRegionBucketSecretName}:SecretString:SourceBucketArn}}' 170 | Bucket2Name: !Sub '{{resolve:secretsmanager:${SecondaryRegionBucketSecretName}:SecretString:SourceBucket}}' 171 | Bucket2Arn: !Sub '{{resolve:secretsmanager:${SecondaryRegionBucketSecretName}:SecretString:SourceBucketArn}}' 172 | ReplicationRole: !GetAtt ReplicationRole.Arn 173 | Code: 174 | ZipFile: | 175 | import cfnresponse 176 | import logging 177 | import boto3 178 | import os 179 | # Init of the logging module 180 | logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') 181 | 182 | 183 | def lambda_handler(event, context): 184 | try: 185 | s3 = boto3.client('s3') 186 | logging.info(event['ResourceProperties']) 187 | if event.get('RequestType') == 'Create': 188 | responseData = {} 189 | responseData['put_bucket_1_versioning_response'] = s3.put_bucket_versioning( 190 | Bucket=os.environ['Bucket1Name'], 191 | VersioningConfiguration={ 192 | 'MFADelete': 'Disabled', 193 | 'Status': 'Enabled', 194 | } 195 | ) 196 | responseData['put_bucket_1_replication_response'] = s3.put_bucket_replication( 197 | Bucket=os.environ['Bucket1Name'], 198 | ReplicationConfiguration={ 199 | 'Role': os.environ['ReplicationRole'], 200 | 'Rules': [ 201 | { 202 | 'Status': 'Enabled', 203 | 'Priority': 1, 204 | 'DeleteMarkerReplication': { 205 | 'Status': 'Enabled' 206 | }, 207 | 'Filter': { 208 | 'Prefix': '' 209 | }, 210 | 'Destination': { 211 | 'Bucket': os.environ['Bucket2Arn'], 212 | 'ReplicationTime': { 213 | 'Status': 'Enabled', 214 | 'Time': { 215 | 'Minutes': 15 216 | } 217 | }, 218 | 'Metrics': { 219 | 'Status': 'Enabled', 220 | 'EventThreshold': { 221 | 'Minutes': 15 222 | } 223 | } 224 | } 225 | } 226 | ] 227 | } 228 | ) 229 | responseData['put_bucket_2_versioning_response'] = s3.put_bucket_versioning( 230 | Bucket=os.environ['Bucket2Name'], 231 | VersioningConfiguration={ 232 | 'MFADelete': 'Disabled', 233 | 'Status': 'Enabled', 234 | } 235 | ) 236 | responseData['put_bucket_2_replication_response'] = s3.put_bucket_replication( 237 | Bucket=os.environ['Bucket2Name'], 238 | ReplicationConfiguration={ 239 | 'Role': os.environ['ReplicationRole'], 240 | 'Rules': [ 241 | { 242 | 'Status': 'Enabled', 243 | 'Priority': 1, 244 | 'DeleteMarkerReplication': { 245 | 'Status': 'Enabled' 246 | }, 247 | 'Filter': { 248 | 'Prefix': '' 249 | }, 250 | 'Destination': { 251 | 'Bucket': os.environ['Bucket1Arn'], 252 | 'ReplicationTime': { 253 | 'Status': 'Enabled', 254 | 'Time': { 255 | 'Minutes': 15 256 | } 257 | }, 258 | 'Metrics': { 259 | 'Status': 'Enabled', 260 | 'EventThreshold': { 261 | 'Minutes': 15 262 | } 263 | } 264 | } 265 | } 266 | ] 267 | } 268 | ) 269 | responseData['message'] = 'Success' 270 | logging.info('Sending %s to cloudformation', responseData['message']) 271 | cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) 272 | elif event.get('RequestType') == 'Delete': 273 | responseData = {} 274 | responseData['delete_bucket_1_repl_response'] = s3.delete_bucket_replication( 275 | Bucket=os.environ['Bucket1Name'] 276 | ) 277 | responseData['delete_bucket_2_repl_response'] = s3.delete_bucket_replication( 278 | Bucket=os.environ['Bucket2Name'] 279 | ) 280 | responseData['message'] = "Success" 281 | logging.info('Sending %s to cloudformation', responseData['message']) 282 | cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) 283 | else: 284 | logging.error('Unknown operation: %s', event.get('RequestType')) 285 | responseData = {} 286 | responseData['message'] = "Invalid operation" 287 | cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) 288 | except Exception as err: 289 | responseData = {} 290 | responseData['message'] = 'Failed' 291 | logging.error(err) 292 | cfnresponse.send(event, context, cfnresponse.FAILED, responseData) 293 | Description: FUnction to enable Cross Region replication between buckets 294 | 295 | InvokeCustomLambda: 296 | DependsOn: CustomBackedLambda 297 | Type: Custom::InvokeCustomLambda 298 | Properties: 299 | ServiceToken: !GetAtt CustomBackedLambda.Arn 300 | Outputs: 301 | SourceBucketMRAPEndpoint: 302 | Description: "SourceBucketMRAP" 303 | Value: !GetAtt SourceBucketMRAP.Alias 304 | CustomLambdaOutput: 305 | Description: Message from custom lambda 306 | Value: !GetAtt InvokeCustomLambda.message -------------------------------------------------------------------------------- /deployment/regionalVpc.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: 2010-09-09 2 | Description: '(SO9169) AWS CloudFormation Sample Template for a VPC with private subnets' 3 | 4 | Parameters: 5 | #General: 6 | Env: 7 | Type: String 8 | Default: '' 9 | Description: String to enable multiple deployments per AWS region 10 | NamingPrefix: 11 | Type: String 12 | Description: The prefix to be used for resources created by this template. 13 | Default: mr-batch 14 | PrimaryRegion: 15 | Type: String 16 | Description: Enter the Primary Region 17 | Default: "us-east-1" 18 | SecondaryRegion: 19 | Type: String 20 | Description: Enter the Secondary Region 21 | Default: "us-west-2" 22 | 23 | ######################################################################## 24 | 25 | Mappings: 26 | RegionMap: 27 | us-east-1: 28 | "VPCCidrBlock": 10.1.0.0/16 29 | "PrivateCidrBlock1": 10.1.0.0/20 30 | "PrivateCidrBlock2": 10.1.16.0/20 31 | "PrivateCidrBlock3": 10.1.32.0/20 32 | "AvailabilityZoneId1": use1-az1 33 | "AvailabilityZoneId2": use1-az4 34 | "AvailabilityZoneId3": use1-az6 35 | us-west-2: 36 | "VPCCidrBlock": 10.2.0.0/16 37 | "PrivateCidrBlock1": 10.2.0.0/20 38 | "PrivateCidrBlock2": 10.2.16.0/20 39 | "PrivateCidrBlock3": 10.2.32.0/20 40 | "AvailabilityZoneId1": usw2-az1 41 | "AvailabilityZoneId2": usw2-az2 42 | "AvailabilityZoneId3": usw2-az3 43 | 44 | ######################################################################## 45 | 46 | Conditions: 47 | isPrimary: !Equals 48 | - !Ref AWS::Region 49 | - us-east-1 50 | 51 | ######################################################################## 52 | 53 | Resources: 54 | ########### 55 | # VPC 56 | ########### 57 | 58 | #VPC 59 | MultiRegionBatchVPC: 60 | Type: AWS::EC2::VPC 61 | Properties: 62 | #https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-vpc.html 63 | CidrBlock: !FindInMap [RegionMap, !Ref "AWS::Region", "VPCCidrBlock"] 64 | EnableDnsSupport: true 65 | EnableDnsHostnames: true 66 | Tags: 67 | - Key: Name 68 | Value: !Join 69 | - "-" 70 | - - !Ref NamingPrefix 71 | - "VPC" 72 | FlowLogRole: 73 | Properties: 74 | AssumeRolePolicyDocument: 75 | Statement: 76 | - Action: 77 | - "sts:AssumeRole" 78 | Effect: Allow 79 | Principal: 80 | Service: 81 | - vpc-flow-logs.amazonaws.com 82 | Version: "2012-10-17" 83 | Path: / 84 | Policies: 85 | - PolicyDocument: 86 | Statement: 87 | - Effect: Allow 88 | Action: 89 | - "logs:CreateLogGroup" 90 | - "logs:CreateLogStream" 91 | - "logs:PutLogEvents" 92 | - "logs:DescribeLogGroups" 93 | - "logs:DescribeLogStreams" 94 | Resource: !Sub 'arn:aws:logs:*:${AWS::AccountId}:*' 95 | Version: "2012-10-17" 96 | PolicyName: flowlogspolicy 97 | Type: "AWS::IAM::Role" 98 | VpcFlowLogs: 99 | Type: AWS::EC2::FlowLog 100 | Properties: 101 | DeliverLogsPermissionArn: !GetAtt FlowLogRole.Arn 102 | LogGroupName: MultiRegionBatchVPCFlowLogs 103 | ResourceId: !Ref MultiRegionBatchVPC 104 | ResourceType: VPC 105 | TrafficType: ALL 106 | PrivateSubnet1: 107 | Type: AWS::EC2::Subnet 108 | #https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-subnet.html 109 | Properties: 110 | VpcId: !Ref MultiRegionBatchVPC 111 | AvailabilityZoneId: !FindInMap [RegionMap, !Ref "AWS::Region", AvailabilityZoneId1] 112 | CidrBlock: !FindInMap [RegionMap, !Ref "AWS::Region", PrivateCidrBlock1] 113 | MapPublicIpOnLaunch: false 114 | Tags: 115 | - Key: Name 116 | Value: !Join 117 | - "-" 118 | - - !Ref NamingPrefix 119 | - Private 120 | 121 | PrivateRouteTableAssociation1: 122 | Type: AWS::EC2::SubnetRouteTableAssociation 123 | Properties: 124 | RouteTableId: !Ref PrivateRouteTable 125 | SubnetId: !Ref PrivateSubnet1 126 | 127 | PrivateSubnet2: 128 | Type: AWS::EC2::Subnet 129 | #https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-subnet.html 130 | Properties: 131 | VpcId: !Ref MultiRegionBatchVPC 132 | AvailabilityZoneId: !FindInMap [RegionMap, !Ref "AWS::Region", AvailabilityZoneId2] 133 | CidrBlock: !FindInMap [RegionMap, !Ref "AWS::Region", PrivateCidrBlock2] 134 | MapPublicIpOnLaunch: false 135 | Tags: 136 | - Key: Name 137 | Value: !Join 138 | - "-" 139 | - - !Ref NamingPrefix 140 | - Private 141 | 142 | PrivateRouteTableAssociation2: 143 | Type: AWS::EC2::SubnetRouteTableAssociation 144 | Properties: 145 | RouteTableId: !Ref PrivateRouteTable 146 | SubnetId: !Ref PrivateSubnet2 147 | 148 | PrivateSubnet3: 149 | Type: AWS::EC2::Subnet 150 | #https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-subnet.html 151 | Properties: 152 | VpcId: !Ref MultiRegionBatchVPC 153 | AvailabilityZoneId: !FindInMap [RegionMap, !Ref "AWS::Region", AvailabilityZoneId3] 154 | CidrBlock: !FindInMap [RegionMap, !Ref "AWS::Region", PrivateCidrBlock3] 155 | MapPublicIpOnLaunch: false 156 | Tags: 157 | - Key: Name 158 | Value: !Join 159 | - "-" 160 | - - !Ref NamingPrefix 161 | - Private 162 | 163 | PrivateRouteTableAssociation3: 164 | Type: AWS::EC2::SubnetRouteTableAssociation 165 | Properties: 166 | RouteTableId: !Ref PrivateRouteTable 167 | SubnetId: !Ref PrivateSubnet3 168 | 169 | PrivateRouteTable: 170 | Type: AWS::EC2::RouteTable 171 | Properties: 172 | VpcId: !Ref MultiRegionBatchVPC 173 | 174 | VPCEndpointFoS3: 175 | Type: AWS::EC2::VPCEndpoint 176 | Properties: 177 | RouteTableIds: 178 | - !Ref PrivateRouteTable 179 | ServiceName: !Sub 'com.amazonaws.${AWS::Region}.s3' 180 | VpcEndpointType: Gateway 181 | VpcId: !Ref MultiRegionBatchVPC 182 | 183 | VPCEndpointForDynamoDB: 184 | Type: AWS::EC2::VPCEndpoint 185 | Properties: 186 | RouteTableIds: 187 | - !Ref PrivateRouteTable 188 | ServiceName: !Sub 'com.amazonaws.${AWS::Region}.dynamodb' 189 | VpcEndpointType: Gateway 190 | VpcId: !Ref MultiRegionBatchVPC 191 | 192 | VPCEndpointForStepFunctions: 193 | Type: AWS::EC2::VPCEndpoint 194 | Properties: 195 | ServiceName: !Sub 'com.amazonaws.${AWS::Region}.states' 196 | VpcEndpointType: Interface 197 | VpcId: !Ref MultiRegionBatchVPC 198 | PrivateDnsEnabled: true 199 | SubnetIds: 200 | - !Ref PrivateSubnet1 201 | - !Ref PrivateSubnet2 202 | - !Ref PrivateSubnet3 203 | SecurityGroupIds: 204 | - !Ref LambdaSecurityGroup 205 | 206 | VPCEndpointForSMTP: 207 | Type: AWS::EC2::VPCEndpoint 208 | Properties: 209 | ServiceName: !Sub 'com.amazonaws.${AWS::Region}.email-smtp' 210 | VpcEndpointType: Interface 211 | VpcId: !Ref MultiRegionBatchVPC 212 | PrivateDnsEnabled: true 213 | SubnetIds: 214 | - !Ref PrivateSubnet1 215 | - !Ref PrivateSubnet2 216 | - !Ref PrivateSubnet3 217 | SecurityGroupIds: 218 | - !Ref LambdaSecurityGroup 219 | 220 | VPCEndpointForSecretsManager: 221 | Type: AWS::EC2::VPCEndpoint 222 | Properties: 223 | ServiceName: !Sub 'com.amazonaws.${AWS::Region}.secretsmanager' 224 | VpcEndpointType: Interface 225 | VpcId: !Ref MultiRegionBatchVPC 226 | PrivateDnsEnabled: true 227 | SubnetIds: 228 | - !Ref PrivateSubnet1 229 | - !Ref PrivateSubnet2 230 | - !Ref PrivateSubnet3 231 | SecurityGroupIds: 232 | - !Ref LambdaSecurityGroup 233 | 234 | VPCEndpointForAPIGW: 235 | Type: AWS::EC2::VPCEndpoint 236 | Properties: 237 | ServiceName: !Sub 'com.amazonaws.${AWS::Region}.execute-api' 238 | VpcEndpointType: Interface 239 | VpcId: !Ref MultiRegionBatchVPC 240 | PrivateDnsEnabled: true 241 | SubnetIds: 242 | - !Ref PrivateSubnet1 243 | - !Ref PrivateSubnet2 244 | - !Ref PrivateSubnet3 245 | SecurityGroupIds: 246 | - !Ref LambdaSecurityGroup 247 | 248 | ############# 249 | # SSM Param to store API VPC EndpointId 250 | ############# 251 | APIGWVPCEndpointId: 252 | Type: AWS::SSM::Parameter 253 | Properties: 254 | Type: String 255 | Name: !Sub VPCEndpointForAPIGW${Env} 256 | Value: !Ref VPCEndpointForAPIGW 257 | 258 | ############ 259 | # Secret to store the VpcId of the created VPC so that the R53 Hosted Zone can be associated with it 260 | ############ 261 | VpcId: 262 | Type: AWS::SecretsManager::Secret 263 | Properties: 264 | Name: !Sub VpcId-${AWS::Region}${Env} 265 | Description: "VPC Id" 266 | KmsKeyId: "alias/aws/secretsmanager" 267 | SecretString: !Ref MultiRegionBatchVPC 268 | ReplicaRegions: 269 | - Region: !If [isPrimary, us-west-2, us-east-1] 270 | 271 | ############ 272 | # Store subnet values in Parameter Store 273 | ############ 274 | Subnet1: 275 | Type: AWS::SSM::Parameter 276 | Properties: 277 | Type: String 278 | Name: !Sub Subnet1${Env} 279 | Value: !Ref PrivateSubnet1 280 | Subnet2: 281 | Type: AWS::SSM::Parameter 282 | Properties: 283 | Type: String 284 | Name: !Sub Subnet2${Env} 285 | Value: !Ref PrivateSubnet2 286 | Subnet3: 287 | Type: AWS::SSM::Parameter 288 | Properties: 289 | Type: String 290 | Name: !Sub Subnet3${Env} 291 | Value: !Ref PrivateSubnet3 292 | 293 | ################################### 294 | # Security Group 295 | ################################### 296 | 297 | LambdaSecurityGroup: 298 | Type: AWS::EC2::SecurityGroup 299 | Properties: 300 | VpcId: !Ref MultiRegionBatchVPC 301 | GroupDescription: "Security Group for Multi Region Batch resources" 302 | SecurityGroupIngress: 303 | - FromPort: 443 304 | ToPort: 443 305 | IpProtocol: tcp 306 | CidrIp: !GetAtt MultiRegionBatchVPC.CidrBlock 307 | Description: "Allows inbound traffic for HTTPS" 308 | - FromPort: 25 309 | ToPort: 25 310 | IpProtocol: tcp 311 | CidrIp: !GetAtt MultiRegionBatchVPC.CidrBlock 312 | Description: "Allows inbound traffic for SMTP" 313 | 314 | ################ 315 | # Buckets 316 | ################ 317 | 318 | PrivateSecurityGroup: 319 | Type: AWS::SSM::Parameter 320 | Properties: 321 | Type: String 322 | Name: !Sub PrivateSG${Env} 323 | Value: !Ref LambdaSecurityGroup 324 | 325 | SourceBucket: 326 | Type: AWS::S3::Bucket 327 | Properties: 328 | BucketEncryption: 329 | ServerSideEncryptionConfiguration: 330 | - ServerSideEncryptionByDefault: 331 | SSEAlgorithm: AES256 332 | LoggingConfiguration: 333 | DestinationBucketName: !Ref LoggingBucket 334 | VersioningConfiguration: 335 | Status: Enabled 336 | 337 | SourceBucketSecret: 338 | Type: AWS::SecretsManager::Secret 339 | Properties: 340 | Name: !Sub SourceBucket-${AWS::Region}${Env} 341 | Description: "Source Bucket ARN" 342 | KmsKeyId: "alias/aws/secretsmanager" 343 | SecretString: !Sub '{"SourceBucket":"${SourceBucket}","SourceBucketArn":"${SourceBucket.Arn}"}' 344 | ReplicaRegions: 345 | - Region: !If [isPrimary, !Ref SecondaryRegion, !Ref PrimaryRegion] 346 | 347 | LoggingBucket: 348 | Type: 'AWS::S3::Bucket' 349 | Properties: 350 | BucketEncryption: 351 | ServerSideEncryptionConfiguration: 352 | - ServerSideEncryptionByDefault: 353 | SSEAlgorithm: AES256 354 | VersioningConfiguration: 355 | Status: Enabled 356 | 357 | LoggingBucketPolicy: 358 | Type: 'AWS::S3::BucketPolicy' 359 | Properties: 360 | Bucket: !Ref LoggingBucket 361 | PolicyDocument: 362 | Version: '2012-10-17' 363 | Statement: 364 | - Sid: 'AllowLogDelivery' 365 | Action: 's3:PutObject' 366 | Effect: 'Allow' 367 | Resource: 368 | - !Sub '${LoggingBucket.Arn}' 369 | - !Sub '${LoggingBucket.Arn}/*' 370 | Principal: 371 | Service: "logging.s3.amazonaws.com" 372 | Condition: 373 | ArnLike: 374 | 'aws:SourceArn': !Sub '${SourceBucket.Arn}' 375 | StringEquals: 376 | 'aws:SourceAccount': !Sub '${AWS::AccountId}' 377 | 378 | 379 | ######################################################################## 380 | Outputs: 381 | MultiRegionBatchVPCId: 382 | Value: !Ref MultiRegionBatchVPC 383 | SourceBucket: 384 | Value: !Ref SourceBucket 385 | LogingBucket: 386 | Value: !Ref LoggingBucket 387 | ######################################################################## -------------------------------------------------------------------------------- /deployment/samTemplate.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Transform: 3 | - AWS::Serverless-2016-10-31 4 | - AWS::LanguageExtensions 5 | Description: > 6 | (SO9169) SAM Template for AWS Step Functions batch processing 7 | 8 | Globals: 9 | Function: 10 | Timeout: 900 11 | 12 | Parameters: 13 | Env: 14 | Type: String 15 | Default: '' 16 | Description: String to enable multiple deployments per AWS region 17 | SESSender: 18 | Type: String 19 | Default: "sender@example.com" 20 | Description: Specify the sender email address. 21 | SESRecipient: 22 | Type: String 23 | Default: "recipient@example.com" 24 | Description: Specify the recipient email address. 25 | SESIdentityName: 26 | Type: String 27 | Default: "sender@example.com" 28 | Description: An email address or domain that Amazon SES users use to send email. It is a best practice to authorize only specific email addresses such as in this case sender@example.com to send emails. If your SES Accounts are in sandbox you have to specify both the sender and recipient emails, in that case modify the template.yaml to add the permissions for recipient email address. 29 | InputArchiveFolder: 30 | Type: String 31 | Default: "input_archive" 32 | Description: Amazon S3 prefix in the SourceBucket where the input file will be archived after processing. 33 | FileChunkSize: 34 | Type: String 35 | Default: 600 36 | Description: Size of each of the chunks, which is split from the input file. 37 | FileDelimiter: 38 | Type: String 39 | Default: "," 40 | Description: Delimiter of the CSV file (for example, a comma). 41 | PrimaryRegion: 42 | Type: String 43 | Description: Enter the Primary Region 44 | Default: "us-east-1" 45 | SourceBucket: 46 | Type: String 47 | Description: The name of the in-region S3 bucket where file to be processed are uploaded 48 | PowerToolsLambdaLayerAccountId: 49 | Type: String 50 | Description: The AWS AccountId where the PowerTools Python Lambda Layer is hosted 51 | Default: 017000801446 52 | 53 | Conditions: 54 | isPrimaryRegion: !Equals 55 | - !Ref "AWS::Region" 56 | - !Ref PrimaryRegion 57 | 58 | Resources: 59 | SESIdentity: 60 | Type: AWS::SES::EmailIdentity 61 | Properties: 62 | EmailIdentity: !Ref SESSender 63 | 64 | BlogBatchProcessChunk: 65 | Type: AWS::Serverless::StateMachine 66 | Properties: 67 | Name: !Sub BlogBatchProcessChunk${Env} 68 | Tracing: 69 | Enabled: true 70 | DefinitionUri: ../source/statemachine/blog-sfn-process-chunk.json 71 | DefinitionSubstitutions: 72 | ReadFileFunctionArn: !GetAtt ReadFileFunction.Arn 73 | WriteOutputChunkFunctionArn: !GetAtt WriteOutputChunkFunction.Arn 74 | ValidateDataFunctionArn: !GetAtt ValidateDataFunction.Arn 75 | ApiEndpoint: !Sub "${Api}.execute-api.${AWS::Region}.amazonaws.com" 76 | ErrorTableName: !Ref ErrorTable 77 | Policies: 78 | - LambdaInvokePolicy: 79 | FunctionName: !Ref GetDataFunction 80 | - LambdaInvokePolicy: 81 | FunctionName: !Ref ReadFileFunction 82 | - LambdaInvokePolicy: 83 | FunctionName: !Ref WriteOutputChunkFunction 84 | - LambdaInvokePolicy: 85 | FunctionName: !Ref ValidateDataFunction 86 | - DynamoDBWritePolicy: 87 | TableName: !Ref ErrorTable 88 | - Statement: 89 | - Sid: AllowApiGatewayInvoke 90 | Effect: Allow 91 | Action: 92 | - execute-api:Invoke 93 | Resource: !Sub "arn:${AWS::Partition}:execute-api:${AWS::Region}:${AWS::AccountId}:${Api}/*/GET/financials/*" 94 | 95 | BlogBatchMainOrchestrator: 96 | Type: AWS::Serverless::StateMachine 97 | Properties: 98 | Name: !Sub BlogBatchMainOrchestrator${Env} 99 | Tracing: 100 | Enabled: true 101 | DefinitionUri: ../source/statemachine/blog-sfn-main-orchestrator.json 102 | DefinitionSubstitutions: 103 | SplitInputFileFunctionArn: !GetAtt SplitInputFileFunction.Arn 104 | MergeS3FilesFunctionArn: !GetAtt MergeS3FilesFunction.Arn 105 | SendEmailFunctionArn: !GetAtt SendEmailFunction.Arn 106 | SNSArn: !Ref SNSTopic 107 | SESSender: !Ref SESSender 108 | SESRecipient: !Ref SESRecipient 109 | BlogBatchProcessChunkArn: !GetAtt BlogBatchProcessChunk.Arn 110 | Policies: 111 | - LambdaInvokePolicy: 112 | FunctionName: !Ref SplitInputFileFunction 113 | - LambdaInvokePolicy: 114 | FunctionName: !Ref MergeS3FilesFunction 115 | - LambdaInvokePolicy: 116 | FunctionName: !Ref SendEmailFunction 117 | - SNSCrudPolicy: 118 | TopicName: !GetAtt SNSTopic.TopicName 119 | - StepFunctionsExecutionPolicy: 120 | StateMachineName: !GetAtt BlogBatchProcessChunk.Name 121 | - Statement: 122 | - Sid: AllowPutTargets 123 | Effect: Allow 124 | Action: 125 | - events:PutTargets 126 | - events:PutRule 127 | - events:DescribeRule 128 | Resource: !Sub "arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule" 129 | - Sid: AllowStatesDescribeStop 130 | Effect: Allow 131 | Action: 132 | - states:DescribeExecution 133 | - states:StopExecution 134 | Resource: !Sub "arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${BlogBatchProcessChunk.Name}:*" 135 | 136 | SplitInputFileFunction: 137 | Type: AWS::Serverless::Function 138 | Properties: 139 | Layers: 140 | - !Sub arn:aws:lambda:${AWS::Region}:${PowerToolsLambdaLayerAccountId}:layer:AWSLambdaPowertoolsPythonV2:20 141 | CodeUri: ../source/split-ip-file/ 142 | Handler: app.lambda_handler 143 | Runtime: python3.9 144 | Tracing: Active 145 | VpcConfig: 146 | SubnetIds: 147 | - !Sub '{{resolve:ssm:Subnet1${Env}}}' 148 | - !Sub '{{resolve:ssm:Subnet2${Env}}}' 149 | - !Sub '{{resolve:ssm:Subnet3${Env}}}' 150 | SecurityGroupIds: 151 | - !Sub '{{resolve:ssm:PrivateSG${Env}}}' 152 | Policies: 153 | - S3CrudPolicy: 154 | BucketName: !Sub '{{resolve:secretsmanager:SourceBucket-${AWS::Region}${Env}:SecretString:SourceBucket}}' 155 | Environment: 156 | Variables: 157 | POWERTOOLS_SERVICE_NAME: !Sub 'SplitInputFileFunction${Env}' 158 | POWERTOOLS_METRICS_NAMESPACE: !Sub 'MultiRegionBatch${Env}' 159 | LOG_LEVEL: INFO 160 | 161 | SplitInputFileFunctionLogGroup: 162 | DependsOn: SplitInputFileFunction 163 | Type: AWS::Logs::LogGroup 164 | Properties: 165 | KmsKeyId: !GetAtt LogGroupKey.Arn 166 | LogGroupName: !Sub /aws/lambda/${SplitInputFileFunction} 167 | RetentionInDays: 7 168 | 169 | AutomationReconciliationFunction: 170 | Type: AWS::Serverless::Function 171 | Properties: 172 | FunctionName: !Sub AutomationReconciliationFunction${Env} 173 | CodeUri: ../source/reconciliation/ 174 | Layers: 175 | - !Sub arn:aws:lambda:${AWS::Region}:${PowerToolsLambdaLayerAccountId}:layer:AWSLambdaPowertoolsPythonV2:20 176 | Tracing: Active 177 | Handler: app.lambda_handler 178 | Runtime: python3.9 179 | VpcConfig: 180 | SubnetIds: 181 | - !Sub '{{resolve:ssm:Subnet1${Env}}}' 182 | - !Sub '{{resolve:ssm:Subnet2${Env}}}' 183 | - !Sub '{{resolve:ssm:Subnet3${Env}}}' 184 | SecurityGroupIds: 185 | - !Sub '{{resolve:ssm:PrivateSG${Env}}}' 186 | Policies: 187 | - DynamoDBCrudPolicy: 188 | TableName: !Sub '{{resolve:secretsmanager:BatchStateTableNameSecret${Env}}}' 189 | - S3CrudPolicy: 190 | BucketName: !Ref SourceBucket 191 | Environment: 192 | Variables: 193 | BATCH_STATE_DDB: !Sub '{{resolve:secretsmanager:BatchStateTableNameSecret${Env}}}' 194 | SECONDARY_REGION_BUCKET: !Sub '{{resolve:secretsmanager:SourceBucket-${AWS::Region}${Env}:SecretString:SourceBucket}}' 195 | POWERTOOLS_SERVICE_NAME: !Sub 'AutomationReconciliationFunction${Env}' 196 | POWERTOOLS_METRICS_NAMESPACE: !Sub 'MultiRegionBatch${Env}' 197 | LOG_LEVEL: INFO 198 | 199 | 200 | AutomationReconciliationFunctionLogGroup: 201 | DependsOn: AutomationReconciliationFunction 202 | Type: AWS::Logs::LogGroup 203 | Properties: 204 | KmsKeyId: !GetAtt LogGroupKey.Arn 205 | LogGroupName: !Sub /aws/lambda/${AutomationReconciliationFunction} 206 | RetentionInDays: 7 207 | 208 | MergeS3FilesFunction: 209 | Type: AWS::Serverless::Function 210 | Properties: 211 | Layers: 212 | - !Sub arn:aws:lambda:${AWS::Region}:${PowerToolsLambdaLayerAccountId}:layer:AWSLambdaPowertoolsPythonV2:20 213 | Tracing: Active 214 | CodeUri: ../source/merge-s3-files/ 215 | Handler: app.lambda_handler 216 | Runtime: python3.9 217 | VpcConfig: 218 | SubnetIds: 219 | - !Sub '{{resolve:ssm:Subnet1${Env}}}' 220 | - !Sub '{{resolve:ssm:Subnet2${Env}}}' 221 | - !Sub '{{resolve:ssm:Subnet3${Env}}}' 222 | SecurityGroupIds: 223 | - !Sub '{{resolve:ssm:PrivateSG${Env}}}' 224 | Policies: 225 | - S3ReadPolicy: 226 | BucketName: !Sub '{{resolve:secretsmanager:SourceBucket-${AWS::Region}${Env}:SecretString:SourceBucket}}' 227 | - S3WritePolicy: 228 | BucketName: !Sub '{{resolve:secretsmanager:SourceBucket-${AWS::Region}${Env}:SecretString:SourceBucket}}' 229 | Environment: 230 | Variables: 231 | POWERTOOLS_SERVICE_NAME: !Sub 'MergeS3FilesFunction${Env}' 232 | POWERTOOLS_METRICS_NAMESPACE: !Sub 'MultiRegionBatch${Env}' 233 | LOG_LEVEL: INFO 234 | 235 | 236 | MergeS3FilesFunctionLogGroup: 237 | DependsOn: MergeS3FilesFunction 238 | Type: AWS::Logs::LogGroup 239 | Properties: 240 | KmsKeyId: !GetAtt LogGroupKey.Arn 241 | LogGroupName: !Sub /aws/lambda/${MergeS3FilesFunction} 242 | RetentionInDays: 7 243 | 244 | SendEmailFunction: 245 | Type: AWS::Serverless::Function 246 | Properties: 247 | Layers: 248 | - !Sub arn:aws:lambda:${AWS::Region}:${PowerToolsLambdaLayerAccountId}:layer:AWSLambdaPowertoolsPythonV2:20 249 | Tracing: Active 250 | CodeUri: ../source/send-email/ 251 | Handler: app.lambda_handler 252 | Runtime: python3.9 253 | VpcConfig: 254 | SubnetIds: 255 | - !Sub '{{resolve:ssm:Subnet1${Env}}}' 256 | - !Sub '{{resolve:ssm:Subnet2${Env}}}' 257 | - !Sub '{{resolve:ssm:Subnet3${Env}}}' 258 | SecurityGroupIds: 259 | - !Sub '{{resolve:ssm:PrivateSG${Env}}}' 260 | Policies: 261 | - SESCrudPolicy: 262 | IdentityName: !Ref SESIdentityName 263 | - Version: '2012-10-17' 264 | Statement: 265 | - Effect: Allow 266 | Action: 267 | - "s3:GetObject" 268 | - "s3:ListBucket" 269 | - "s3:GetBucketLocation" 270 | - "s3:GetObjectVersion" 271 | - "s3:GetLifecycleConfiguration" 272 | Resource: "arn:aws:s3:::*" 273 | Condition: 274 | StringEquals: 275 | "s3:ResourceAccount": !Sub "${AWS::AccountId}" 276 | - DynamoDBWritePolicy: 277 | TableName: !Sub '{{resolve:secretsmanager:BatchStateTableNameSecret${Env}}}' 278 | - AWSSecretsManagerGetSecretValuePolicy: 279 | SecretArn: !Sub 'arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:*' 280 | - Version: '2012-10-17' 281 | Statement: 282 | - Effect: Allow 283 | Action: 284 | - "s3:GetObject" 285 | - "s3:ListBucket" 286 | - "s3:GetBucketLocation" 287 | - "s3:GetObjectVersion" 288 | - "s3:GetLifecycleConfiguration" 289 | Resource: 290 | - !Join 291 | - '' 292 | - - !Sub "arn:${AWS::Partition}:s3::${AWS::AccountId}:accesspoint/" 293 | - !Sub '{{resolve:secretsmanager:SourceBucketMRAPSecret${Env}}}' 294 | - !Join 295 | - '' 296 | - - !Sub "arn:${AWS::Partition}:s3::${AWS::AccountId}:accesspoint/" 297 | - !Sub '{{resolve:secretsmanager:SourceBucketMRAPSecret${Env}}}' 298 | - "/*" 299 | Environment: 300 | Variables: 301 | BATCH_STATE_DDB: !Sub '{{resolve:secretsmanager:BatchStateTableNameSecret${Env}}}' 302 | SMTP_CREDENTIAL_SECRET: !Sub SmtpCredentialsSecret${Env} 303 | SMTP_HOST: !Sub 'email-smtp.${AWS::Region}.amazonaws.com' 304 | MRAP_ALIAS_SECRET: !Sub SourceBucketMRAPSecret${Env} 305 | POWERTOOLS_SERVICE_NAME: !Sub 'SendEmailFunction${Env}' 306 | POWERTOOLS_METRICS_NAMESPACE: !Sub 'MultiRegionBatch${Env}' 307 | LOG_LEVEL: INFO 308 | 309 | SendEmailFunctionLogGroup: 310 | DependsOn: SendEmailFunction 311 | Type: AWS::Logs::LogGroup 312 | Properties: 313 | KmsKeyId: !GetAtt LogGroupKey.Arn 314 | LogGroupName: !Sub /aws/lambda/${SendEmailFunction} 315 | RetentionInDays: 7 316 | 317 | Api: 318 | Type: AWS::Serverless::Api 319 | DependsOn: ApiCWLRoleArn 320 | Properties: 321 | TracingEnabled: true 322 | StageName: Prod 323 | Auth: 324 | DefaultAuthorizer: AWS_IAM 325 | UsagePlan: 326 | CreateUsagePlan: PER_API 327 | UsagePlanName: "batch-api-usage-plan" 328 | Quota: 329 | Limit: 100 330 | Period: DAY 331 | Throttle: 332 | BurstLimit: 50 333 | RateLimit: 100 334 | Description: "Batch API Usage Plan" 335 | AccessLogSetting: 336 | DestinationArn: !Sub ${ApiAccessLogGroup.Arn} 337 | Format: "{ 'requestId':'$context.requestId', 'ip': '$context.identity.sourceIp', 'caller':'$context.identity.caller', 'user':'$context.identity.user','requestTime':'$context.requestTime', 'xrayTraceId':'$context.xrayTraceId', 'wafResponseCode':'$context.wafResponseCode', 'httpMethod':'$context.httpMethod','resourcePath':'$context.resourcePath', 'status':'$context.status','protocol':'$context.protocol', 'responseLength':'$context.responseLength' }" 338 | 339 | ApiAccessLogGroup: 340 | Type: AWS::Logs::LogGroup 341 | DependsOn: Api 342 | Properties: 343 | LogGroupName: !Sub /aws/apigateway/${Api} 344 | RetentionInDays: 7 345 | KmsKeyId: !GetAtt LogGroupKey.Arn 346 | 347 | LogGroupKey: 348 | Type: AWS::KMS::Key 349 | Properties: 350 | Enabled: true 351 | EnableKeyRotation: true 352 | KeyPolicy: 353 | Version: 2012-10-17 354 | Id: key-loggroup 355 | Statement: 356 | - Sid: Enable IAM User Permissions 357 | Effect: Allow 358 | Principal: 359 | AWS: !Join 360 | - '' 361 | - - !Sub 'arn:${AWS::Partition}:iam::' 362 | - !Ref 'AWS::AccountId' 363 | - ':root' 364 | Action: 'kms:*' 365 | Resource: '*' 366 | - Sid: Enable Cloudwatch access 367 | Effect: Allow 368 | Principal: 369 | Service: !Sub "logs.${AWS::Region}.amazonaws.com" 370 | Action: 371 | - kms:Encrypt* 372 | - kms:Decrypt* 373 | - kms:ReEncrypt* 374 | - kms:GenerateDataKey* 375 | - kms:Describe* 376 | Resource: '*' 377 | 378 | 379 | ApiCWLRoleArn: 380 | Type: AWS::ApiGateway::Account 381 | Properties: 382 | CloudWatchRoleArn: !GetAtt CloudWatchRole.Arn 383 | 384 | 385 | CloudWatchRole: 386 | Type: AWS::IAM::Role 387 | Properties: 388 | AssumeRolePolicyDocument: 389 | Version: '2012-10-17' 390 | Statement: 391 | Action: 'sts:AssumeRole' 392 | Effect: Allow 393 | Principal: 394 | Service: apigateway.amazonaws.com 395 | Path: / 396 | ManagedPolicyArns: 397 | - !Sub 'arn:${AWS::Partition}:iam::aws:policy/service-role/AmazonAPIGatewayPushToCloudWatchLogs' 398 | 399 | GetDataFunction: 400 | Type: AWS::Serverless::Function 401 | Properties: 402 | Layers: 403 | - !Sub arn:aws:lambda:${AWS::Region}:${PowerToolsLambdaLayerAccountId}:layer:AWSLambdaPowertoolsPythonV2:20 404 | Tracing: Active 405 | CodeUri: ../source/get-data/ 406 | Handler: app.lambda_handler 407 | Runtime: python3.9 408 | VpcConfig: 409 | SubnetIds: 410 | - !Sub '{{resolve:ssm:Subnet1${Env}}}' 411 | - !Sub '{{resolve:ssm:Subnet2${Env}}}' 412 | - !Sub '{{resolve:ssm:Subnet3${Env}}}' 413 | SecurityGroupIds: 414 | - !Sub '{{resolve:ssm:PrivateSG${Env}}}' 415 | Environment: 416 | Variables: 417 | TABLE_NAME: !Ref FinancialTable 418 | POWERTOOLS_SERVICE_NAME: !Sub 'GetDataFunction${Env}' 419 | POWERTOOLS_METRICS_NAMESPACE: !Sub 'MultiRegionBatch${Env}' 420 | LOG_LEVEL: INFO 421 | Policies: 422 | - AWSLambdaExecute 423 | - DynamoDBReadPolicy: 424 | TableName: !Ref FinancialTable 425 | Events: 426 | GetData: 427 | Type: Api 428 | Properties: 429 | RestApiId: !Ref Api 430 | Path: /financials/{uuid} 431 | Method: get 432 | 433 | GetDataFunctionLogGroup: 434 | DependsOn: GetDataFunction 435 | Type: AWS::Logs::LogGroup 436 | Properties: 437 | KmsKeyId: !GetAtt LogGroupKey.Arn 438 | LogGroupName: !Sub /aws/lambda/${GetDataFunction} 439 | RetentionInDays: 7 440 | 441 | ReadFileFunction: 442 | Type: AWS::Serverless::Function 443 | Properties: 444 | Layers: 445 | - !Sub arn:aws:lambda:${AWS::Region}:${PowerToolsLambdaLayerAccountId}:layer:AWSLambdaPowertoolsPythonV2:20 446 | Tracing: Active 447 | CodeUri: ../source/read-file/ 448 | Handler: app.lambda_handler 449 | Runtime: python3.9 450 | Environment: 451 | Variables: 452 | POWERTOOLS_SERVICE_NAME: !Sub 'ReadFileFunction${Env}' 453 | POWERTOOLS_METRICS_NAMESPACE: !Sub 'MultiRegionBatch${Env}' 454 | LOG_LEVEL: INFO 455 | VpcConfig: 456 | SubnetIds: 457 | - !Sub '{{resolve:ssm:Subnet1${Env}}}' 458 | - !Sub '{{resolve:ssm:Subnet2${Env}}}' 459 | - !Sub '{{resolve:ssm:Subnet3${Env}}}' 460 | SecurityGroupIds: 461 | - !Sub '{{resolve:ssm:PrivateSG${Env}}}' 462 | Policies: 463 | - S3ReadPolicy: 464 | BucketName: !Sub '{{resolve:secretsmanager:SourceBucket-${AWS::Region}${Env}:SecretString:SourceBucket}}' 465 | 466 | 467 | ReadFileFunctionLogGroup: 468 | DependsOn: ReadFileFunction 469 | Type: AWS::Logs::LogGroup 470 | Properties: 471 | KmsKeyId: !GetAtt LogGroupKey.Arn 472 | LogGroupName: !Sub /aws/lambda/${ReadFileFunction} 473 | RetentionInDays: 7 474 | 475 | 476 | FinancialTable: 477 | Type: AWS::DynamoDB::Table 478 | Properties: 479 | PointInTimeRecoverySpecification: 480 | PointInTimeRecoveryEnabled: true 481 | SSESpecification: 482 | SSEEnabled: true 483 | AttributeDefinitions: 484 | - AttributeName: uuid 485 | AttributeType: S 486 | KeySchema: 487 | - AttributeName: uuid 488 | KeyType: HASH 489 | BillingMode: PAY_PER_REQUEST 490 | 491 | ErrorTable: 492 | Type: AWS::DynamoDB::Table 493 | Properties: 494 | PointInTimeRecoverySpecification: 495 | PointInTimeRecoveryEnabled: true 496 | SSESpecification: 497 | SSEEnabled: true 498 | AttributeDefinitions: 499 | - AttributeName: uuid 500 | AttributeType: S 501 | KeySchema: 502 | - AttributeName: uuid 503 | KeyType: HASH 504 | BillingMode: PAY_PER_REQUEST 505 | 506 | WriteOutputChunkFunction: 507 | Type: AWS::Serverless::Function 508 | Properties: 509 | Layers: 510 | - !Sub arn:aws:lambda:${AWS::Region}:${PowerToolsLambdaLayerAccountId}:layer:AWSLambdaPowertoolsPythonV2:20 511 | Tracing: Active 512 | CodeUri: ../source/write-output-chunk/ 513 | Handler: app.lambda_handler 514 | Runtime: python3.9 515 | Environment: 516 | Variables: 517 | POWERTOOLS_SERVICE_NAME: !Sub 'WriteOutputChunkFunction${Env}' 518 | POWERTOOLS_METRICS_NAMESPACE: !Sub 'MultiRegionBatch${Env}' 519 | LOG_LEVEL: INFO 520 | VpcConfig: 521 | SubnetIds: 522 | - !Sub '{{resolve:ssm:Subnet1${Env}}}' 523 | - !Sub '{{resolve:ssm:Subnet2${Env}}}' 524 | - !Sub '{{resolve:ssm:Subnet3${Env}}}' 525 | SecurityGroupIds: 526 | - !Sub '{{resolve:ssm:PrivateSG${Env}}}' 527 | Policies: 528 | - S3WritePolicy: 529 | BucketName: !Sub '{{resolve:secretsmanager:SourceBucket-${AWS::Region}${Env}:SecretString:SourceBucket}}' 530 | 531 | 532 | WriteOutputChunkFunctionLogGroup: 533 | DependsOn: WriteOutputChunkFunction 534 | Type: AWS::Logs::LogGroup 535 | Properties: 536 | KmsKeyId: !GetAtt LogGroupKey.Arn 537 | LogGroupName: !Sub /aws/lambda/${WriteOutputChunkFunction} 538 | RetentionInDays: 7 539 | 540 | ValidateDataFunction: 541 | Type: AWS::Serverless::Function 542 | Properties: 543 | Layers: 544 | - !Sub arn:aws:lambda:${AWS::Region}:${PowerToolsLambdaLayerAccountId}:layer:AWSLambdaPowertoolsPythonV2:20 545 | Tracing: Active 546 | CodeUri: ../source/validate-data/ 547 | Handler: app.lambda_handler 548 | Runtime: python3.9 549 | Environment: 550 | Variables: 551 | POWERTOOLS_SERVICE_NAME: !Sub 'ValidateDataFunction${Env}' 552 | POWERTOOLS_METRICS_NAMESPACE: !Sub 'MultiRegionBatch${Env}' 553 | LOG_LEVEL: INFO 554 | VpcConfig: 555 | SubnetIds: 556 | - !Sub '{{resolve:ssm:Subnet1${Env}}}' 557 | - !Sub '{{resolve:ssm:Subnet2${Env}}}' 558 | - !Sub '{{resolve:ssm:Subnet3${Env}}}' 559 | SecurityGroupIds: 560 | - !Sub '{{resolve:ssm:PrivateSG${Env}}}' 561 | 562 | ValidateDataFunctionLogGroup: 563 | DependsOn: ValidateDataFunction 564 | Type: AWS::Logs::LogGroup 565 | Properties: 566 | KmsKeyId: !GetAtt LogGroupKey.Arn 567 | LogGroupName: !Sub /aws/lambda/${ValidateDataFunction} 568 | RetentionInDays: 7 569 | 570 | AutomationRegionalFailoverFunction: 571 | Type: AWS::Serverless::Function 572 | Properties: 573 | Layers: 574 | - !Sub arn:aws:lambda:${AWS::Region}:${PowerToolsLambdaLayerAccountId}:layer:AWSLambdaPowertoolsPythonV2:20 575 | Tracing: Active 576 | FunctionName: !Sub AutomationRegionalFailoverFunction${Env} 577 | CodeUri: ../source/failover/ 578 | Handler: app.lambda_handler 579 | Runtime: python3.9 580 | Policies: 581 | - Version: '2012-10-17' 582 | Statement: 583 | - Effect: Allow 584 | Action: 585 | - "route53-recovery-cluster:*" 586 | - "route53-recovery-control-config:*" 587 | Resource: 588 | - !Sub 'arn:aws:route53-recovery-control::${AWS::AccountId}:cluster/*' 589 | - !Sub 'arn:aws:route53-recovery-control::${AWS::AccountId}:controlpanel/*' 590 | - !Sub 'arn:aws:route53-recovery-control::${AWS::AccountId}:controlpanel/*/routingcontrol/*' 591 | Environment: 592 | Variables: 593 | POWERTOOLS_SERVICE_NAME: !Sub 'AutomationRegionalFailoverFunction${Env}' 594 | POWERTOOLS_METRICS_NAMESPACE: !Sub 'MultiRegionBatch${Env}' 595 | LOG_LEVEL: INFO 596 | ARC_ROUTING_CONTROL_ARN: !Sub '{{resolve:secretsmanager:ArcRoutingControlSecret${Env}}}' 597 | ARC_CLUSTER_ENDPOINTS: !Sub '{{resolve:secretsmanager:ArcClusterEndpoints${Env}}}' 598 | 599 | AutomationRegionalFailoverFunctionLogGroup: 600 | DependsOn: AutomationRegionalFailoverFunction 601 | Type: AWS::Logs::LogGroup 602 | Properties: 603 | KmsKeyId: !GetAtt LogGroupKey.Arn 604 | LogGroupName: !Sub /aws/lambda/${AutomationRegionalFailoverFunction} 605 | RetentionInDays: 7 606 | 607 | S3NotificationLambdaFunction: 608 | Type: AWS::Serverless::Function 609 | Properties: 610 | Layers: 611 | - !Sub arn:aws:lambda:${AWS::Region}:${PowerToolsLambdaLayerAccountId}:layer:AWSLambdaPowertoolsPythonV2:20 612 | Tracing: Active 613 | CodeUri: ../source/s3-lambda-notification/ 614 | Handler: app.lambda_handler 615 | Runtime: python3.9 616 | VpcConfig: 617 | SubnetIds: 618 | - !Sub '{{resolve:ssm:Subnet1${Env}}}' 619 | - !Sub '{{resolve:ssm:Subnet2${Env}}}' 620 | - !Sub '{{resolve:ssm:Subnet3${Env}}}' 621 | SecurityGroupIds: 622 | - !Sub '{{resolve:ssm:PrivateSG${Env}}}' 623 | Policies: 624 | - StepFunctionsExecutionPolicy: 625 | StateMachineName: !GetAtt BlogBatchMainOrchestrator.Name 626 | - DynamoDBWritePolicy: 627 | TableName: !Sub '{{resolve:secretsmanager:BatchStateTableNameSecret${Env}}}' 628 | - AWSSecretsManagerGetSecretValuePolicy: 629 | SecretArn: !Sub 'arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:*' 630 | Environment: 631 | Variables: 632 | STATE_MACHINE_EXECUTION_NAME: "BlogBatchMainOrchestrator" 633 | INPUT_ARCHIVE_FOLDER: !Ref InputArchiveFolder 634 | FILE_CHUNK_SIZE: !Ref FileChunkSize 635 | FILE_DELIMITER: !Ref FileDelimiter 636 | STATE_MACHINE_ARN: !GetAtt BlogBatchMainOrchestrator.Arn 637 | BATCH_STATE_DDB: !Sub '{{resolve:secretsmanager:BatchStateTableNameSecret${Env}}}' 638 | DNS_RECORD_SECRET: !Sub DNSRecordSecret${Env} 639 | POWERTOOLS_SERVICE_NAME: !Sub 'S3NotificationLambdaFunction${Env}' 640 | POWERTOOLS_METRICS_NAMESPACE: !Sub 'MultiRegionBatch${Env}' 641 | LOG_LEVEL: INFO 642 | 643 | S3NotificationLambdaFunctionLogGroup: 644 | DependsOn: S3NotificationLambdaFunction 645 | Type: AWS::Logs::LogGroup 646 | Properties: 647 | KmsKeyId: !GetAtt LogGroupKey.Arn 648 | LogGroupName: !Sub /aws/lambda/${S3NotificationLambdaFunction} 649 | RetentionInDays: 7 650 | 651 | S3BucketEventPermission: 652 | Type: AWS::Lambda::Permission 653 | Properties: 654 | Action: lambda:invokeFunction 655 | SourceAccount: !Ref 'AWS::AccountId' 656 | FunctionName: !Ref S3NotificationLambdaFunction 657 | SourceArn: !Sub '{{resolve:secretsmanager:SourceBucket-${AWS::Region}${Env}:SecretString:SourceBucketArn}}' 658 | Principal: s3.amazonaws.com 659 | 660 | PostStackProcessingFunctionRole: 661 | Type: AWS::IAM::Role 662 | Properties: 663 | AssumeRolePolicyDocument: 664 | Version: '2012-10-17' 665 | Statement: 666 | - Effect: Allow 667 | Principal: 668 | Service: lambda.amazonaws.com 669 | Action: sts:AssumeRole 670 | Path: / 671 | ManagedPolicyArns: 672 | - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 673 | Policies: 674 | - PolicyName: S3BucketNotificationDynamoDBInsertPolicy 675 | PolicyDocument: 676 | Version: '2012-10-17' 677 | Statement: 678 | - Sid: AllowBucketNotification 679 | Effect: Allow 680 | Action: s3:PutBucketNotification 681 | Resource: 682 | - !Sub 'arn:${AWS::Partition}:s3:::{{resolve:secretsmanager:SourceBucket-${AWS::Region}${Env}:SecretString:SourceBucket}}' 683 | - !Sub 'arn:${AWS::Partition}:s3:::{{resolve:secretsmanager:SourceBucket-${AWS::Region}${Env}:SecretString:SourceBucket}}/*' 684 | - Sid: DynamoDBInsert 685 | Effect: Allow 686 | Action: dynamodb:BatchWriteItem 687 | Resource: 688 | - !GetAtt FinancialTable.Arn 689 | - PolicyName: EC2NetworkInterfacesPolicy 690 | PolicyDocument: 691 | Version: '2012-10-17' 692 | Statement: 693 | - Sid: AllowNetworkInterfacePermissions 694 | Effect: Allow 695 | Action: 696 | - "ec2:DescribeNetworkInterfaces" 697 | - "ec2:CreateNetworkInterface" 698 | - "ec2:DeleteNetworkInterface" 699 | - "ec2:DescribeInstances" 700 | - "ec2:AttachNetworkInterface" 701 | Resource: "*" 702 | Condition: 703 | StringEquals: 704 | "aws:PrincipalAccount": !Sub "${AWS::AccountId}" 705 | 706 | PostStackProcessingFunction: 707 | Type: AWS::Serverless::Function 708 | Properties: 709 | Layers: 710 | - !Sub arn:aws:lambda:${AWS::Region}:${PowerToolsLambdaLayerAccountId}:layer:AWSLambdaPowertoolsPythonV2:20 711 | Tracing: Active 712 | Description: Function to apply notification to the S3 bucket 713 | CodeUri: ../source/custom-resource/ 714 | Handler: app.lambda_handler 715 | Runtime: python3.9 716 | Role: !GetAtt PostStackProcessingFunctionRole.Arn 717 | Environment: 718 | Variables: 719 | POWERTOOLS_SERVICE_NAME: !Sub 'PostStackProcessingFunction${Env}' 720 | POWERTOOLS_METRICS_NAMESPACE: !Sub 'MultiRegionBatch${Env}' 721 | LOG_LEVEL: INFO 722 | VpcConfig: 723 | SubnetIds: 724 | - !Sub '{{resolve:ssm:Subnet1${Env}}}' 725 | - !Sub '{{resolve:ssm:Subnet2${Env}}}' 726 | - !Sub '{{resolve:ssm:Subnet3${Env}}}' 727 | SecurityGroupIds: 728 | - !Sub '{{resolve:ssm:PrivateSG${Env}}}' 729 | 730 | PostStackProcessingFunctionLogGroup: 731 | DependsOn: PostStackProcessingFunction 732 | Type: AWS::Logs::LogGroup 733 | Properties: 734 | KmsKeyId: !GetAtt LogGroupKey.Arn 735 | LogGroupName: !Sub /aws/lambda/${PostStackProcessingFunction} 736 | RetentionInDays: 7 737 | 738 | PostStackProcessing: 739 | Type: Custom::PostStackProcessing 740 | Properties: 741 | ServiceToken: !GetAtt PostStackProcessingFunction.Arn 742 | S3Bucket: !Ref SourceBucket 743 | FunctionARN: !GetAtt S3NotificationLambdaFunction.Arn 744 | NotificationId: S3ObjectCreatedEvent 745 | FinancialTableName: !Ref FinancialTable 746 | 747 | SNSTopic: 748 | Type: AWS::SNS::Topic 749 | Properties: 750 | KmsMasterKeyId: alias/aws/sns 751 | 752 | AutomationFailoverRunbook: 753 | Type: AWS::SSM::Document 754 | Properties: 755 | Name: 'AutomationFailoverRunbook' 756 | DocumentType: Automation 757 | DocumentFormat: YAML 758 | Content: 759 | description: |- 760 | *Runbook for Batch Failover* 761 | 762 | --- 763 | # Runbook for Batch Failover 764 | 765 | 1. Get Current Routing Control State 766 | 2. Rotate Arc Controls 767 | 3. Wait for DNS Cache Refresh and S3 CRR 768 | 4. Trigger Reconciliation 769 | schemaVersion: '0.3' 770 | assumeRole: !Sub 'arn:aws:iam::${AWS::AccountId}:role/AutomationServiceRole${Env}' 771 | mainSteps: 772 | - name: GetRoutingControlState 773 | action: 'aws:invokeLambdaFunction' 774 | maxAttempts: 1 775 | timeoutSeconds: 120 776 | onFailure: Abort 777 | inputs: 778 | FunctionName: !Ref AutomationRegionalFailoverFunction 779 | InputPayload: 780 | FUNCTION: get_current_routing_control_state 781 | outputs: 782 | - Name: CURRENT_ROUTING_CONTROL_STATE 783 | Selector: $.Payload.routing_control_state 784 | Type: String 785 | - name: RotateArcControls 786 | action: 'aws:invokeLambdaFunction' 787 | maxAttempts: 1 788 | timeoutSeconds: 120 789 | onFailure: Abort 790 | inputs: 791 | FunctionName: !Ref AutomationRegionalFailoverFunction 792 | InputPayload: 793 | FUNCTION: rotate_arc_controls 794 | CURRENT_ROUTING_CONTROL_STATE: '{{GetRoutingControlState.CURRENT_ROUTING_CONTROL_STATE}}' 795 | outputs: 796 | - Name: UPDATED_ROUTING_CONTROL_STATE 797 | Selector: $.Payload.routing_control_state 798 | Type: String 799 | - name: WaitForDNSCacheRefresh 800 | action: aws:sleep 801 | inputs: 802 | Duration: PT15M 803 | - name: TriggerReconciliation 804 | action: 'aws:invokeLambdaFunction' 805 | maxAttempts: 1 806 | timeoutSeconds: 60 807 | onFailure: Abort 808 | inputs: 809 | FunctionName: !Ref AutomationReconciliationFunction 810 | outputs: 811 | - Name: NUMBER_OF_FILES_SUBMITTED_FOR_RECONCILIATION 812 | Selector: $.Payload.num_files_submitted_for_reconciliation 813 | Type: Integer 814 | - Name: FILE_NAMES_FOR_RECONCILIATION 815 | Selector: $.Payload.file_list 816 | Type: StringList 817 | -------------------------------------------------------------------------------- /deployment/samconfig.toml: -------------------------------------------------------------------------------- 1 | version = 0.1 2 | [default] 3 | [default.deploy] 4 | [default.deploy.parameters] 5 | parameter_overrides = "InputArchiveFolder=\"input_archive\" FileChunkSize=\"600\" FileDelimiter=\",\"" 6 | -------------------------------------------------------------------------------- /source/custom-resource/app.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | import csv 4 | import json 5 | 6 | import boto3 7 | from aws_lambda_powertools import Logger, Tracer, Metrics 8 | 9 | import cfnresponse 10 | 11 | s3Client = boto3.client('s3') 12 | dynamodb = boto3.resource('dynamodb') 13 | 14 | metrics = Metrics() 15 | tracer = Tracer() 16 | logger = Logger() 17 | 18 | 19 | @metrics.log_metrics(capture_cold_start_metric=False) 20 | @logger.inject_lambda_context(log_event=True, clear_state=True) 21 | @tracer.capture_lambda_handler 22 | def lambda_handler(event, context): 23 | logger.info('Received event: %s' % json.dumps(event)) 24 | 25 | status = cfnresponse.FAILED 26 | new_physical_id = None 27 | 28 | try: 29 | properties = event.get('ResourceProperties') 30 | physical_id = event.get('PhysicalResourceId') 31 | 32 | status, new_physical_id = { 33 | 'Create': create, 34 | 'Update': update, 35 | 'Delete': delete 36 | }.get(event['RequestType'], lambda x, y: (cfnresponse.FAILED, None))(properties, physical_id) 37 | except Exception as e: 38 | logger.exception('Exception: %s' % e) 39 | status = cfnresponse.FAILED 40 | finally: 41 | cfnresponse.send(event, context, status, {}, new_physical_id) 42 | 43 | 44 | @tracer.capture_method 45 | def add_bucket_notification(bucket_name, notification_id, function_arn): 46 | notification_response = s3Client.put_bucket_notification_configuration( 47 | Bucket=bucket_name, 48 | NotificationConfiguration={ 49 | 'LambdaFunctionConfigurations': [ 50 | { 51 | 'Id': notification_id, 52 | 'LambdaFunctionArn': function_arn, 53 | 'Events': [ 54 | 's3:ObjectCreated:*' 55 | ], 56 | 'Filter': { 57 | 'Key': { 58 | 'FilterRules': [ 59 | { 60 | 'Name': 'prefix', 61 | 'Value': 'input/' 62 | }, 63 | { 64 | 'Name': 'suffix', 65 | 'Value': 'csv' 66 | }, 67 | ] 68 | } 69 | } 70 | }, 71 | ] 72 | } 73 | ) 74 | return notification_response 75 | 76 | 77 | def load_csv_data(table_name): 78 | csv_file = "testfile_financial_data.csv" 79 | 80 | batch_size = 100 81 | batch = [] 82 | 83 | for row in csv.DictReader(open(csv_file)): 84 | if len(batch) >= batch_size: 85 | write_to_dynamo(batch, table_name) 86 | batch.clear() 87 | batch.append(row) 88 | 89 | if batch: 90 | write_to_dynamo(batch, table_name) 91 | 92 | return { 93 | 'statusCode': 200, 94 | 'body': json.dumps('CSV file loaded into the DYnamoDB table') 95 | } 96 | 97 | 98 | @tracer.capture_method 99 | def write_to_dynamo(rows, table_name): 100 | try: 101 | table = dynamodb.Table(table_name) 102 | except: 103 | logger.exception("Error loading DynamoDB table. Check if table was created correctly and environment variable.") 104 | 105 | try: 106 | with table.batch_writer() as batch: 107 | for i in range(len(rows)): 108 | batch.put_item( 109 | Item=rows[i] 110 | ) 111 | except Exception as e: 112 | logger.exception("Exception occurred") 113 | 114 | 115 | def create(properties, physical_id): 116 | bucket_name = properties['S3Bucket'] 117 | notification_id = properties['NotificationId'] 118 | function_arn = properties['FunctionARN'] 119 | table_name = properties['FinancialTableName'] 120 | response = add_bucket_notification(bucket_name, notification_id, function_arn) 121 | logger.info('AddBucketNotification response: %s' % json.dumps(response)) 122 | logger.info('Loading table: %s' % table_name) 123 | response = load_csv_data(table_name) 124 | logger.info('AddBucketNotification response: %s' % json.dumps(response)) 125 | 126 | return cfnresponse.SUCCESS, physical_id 127 | 128 | 129 | def update(properties, physical_id): 130 | return cfnresponse.SUCCESS, None 131 | 132 | 133 | def delete(properties, physical_id): 134 | return cfnresponse.SUCCESS, None 135 | -------------------------------------------------------------------------------- /source/custom-resource/cfnresponse.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | from __future__ import print_function 5 | import urllib3 6 | import json 7 | 8 | SUCCESS = "SUCCESS" 9 | FAILED = "FAILED" 10 | 11 | http = urllib3.PoolManager() 12 | 13 | 14 | def send(event, context, responseStatus, responseData, physicalResourceId=None, noEcho=False, reason=None): 15 | responseUrl = event['ResponseURL'] 16 | 17 | print(responseUrl) 18 | 19 | responseBody = { 20 | 'Status': responseStatus, 21 | 'Reason': reason or "See the details in CloudWatch Log Stream: {}".format(context.log_stream_name), 22 | 'PhysicalResourceId': physicalResourceId or context.log_stream_name, 23 | 'StackId': event['StackId'], 24 | 'RequestId': event['RequestId'], 25 | 'LogicalResourceId': event['LogicalResourceId'], 26 | 'NoEcho': noEcho, 27 | 'Data': responseData 28 | } 29 | 30 | json_responseBody = json.dumps(responseBody) 31 | 32 | print("Response body:") 33 | print(json_responseBody) 34 | 35 | headers = { 36 | 'content-type': '', 37 | 'content-length': str(len(json_responseBody)) 38 | } 39 | 40 | try: 41 | response = http.request('PUT', responseUrl, headers=headers, body=json_responseBody) 42 | print("Status code:", response.status) 43 | 44 | 45 | except Exception as e: 46 | 47 | print("send(..) failed executing http.request(..):", e) -------------------------------------------------------------------------------- /source/custom-resource/requirements.txt: -------------------------------------------------------------------------------- 1 | aws_lambda_powertools -------------------------------------------------------------------------------- /source/failover/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/source/failover/__init__.py -------------------------------------------------------------------------------- /source/failover/app.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import boto3 4 | from aws_lambda_powertools import Logger, Tracer, Metrics 5 | from aws_lambda_powertools.metrics import MetricUnit 6 | 7 | metrics = Metrics() 8 | tracer = Tracer() 9 | logger = Logger() 10 | 11 | 12 | @metrics.log_metrics(capture_cold_start_metric=False) 13 | @logger.inject_lambda_context(log_event=True, clear_state=True) 14 | @tracer.capture_lambda_handler 15 | def lambda_handler(event, context): 16 | function = event["FUNCTION"] 17 | 18 | if function == "get_current_routing_control_state": 19 | return get_current_routing_control_state(event, context) 20 | elif function == "rotate_arc_controls": 21 | return rotate_arc_controls(event, context) 22 | else: 23 | dummy(event, context) 24 | 25 | 26 | @tracer.capture_method 27 | def get_current_routing_control_state(event, context): 28 | 29 | logger.info("get_current_routing_control_state Invoked") 30 | endpoints = json.loads(os.environ['ARC_CLUSTER_ENDPOINTS']) 31 | routing_control_arn = os.environ['ARC_ROUTING_CONTROL_ARN'] 32 | 33 | for region, endpoint in endpoints.items(): 34 | try: 35 | logger.info("route 53 recover cluster endpoint: " + endpoint) 36 | client = boto3.client('route53-recovery-cluster', region_name=region, endpoint_url=endpoint) 37 | routing_control_state = client.get_routing_control_state(RoutingControlArn=routing_control_arn) 38 | 39 | logger.info("routing Control State is " + routing_control_state["RoutingControlState"]) 40 | 41 | return {'routing_control_state': routing_control_state["RoutingControlState"]} 42 | except Exception as e: 43 | logger.exception("Exception occurred while getting current routing control state") 44 | 45 | 46 | @tracer.capture_method 47 | def rotate_arc_controls(event, context): 48 | 49 | logger.info("update_arc_control Invoked") 50 | endpoints = json.loads(os.environ['ARC_CLUSTER_ENDPOINTS']) 51 | routing_control_arn = os.environ['ARC_ROUTING_CONTROL_ARN'] 52 | updated_routing_control_state = "NotUpdated" 53 | done = False 54 | for region, endpoint in endpoints.items(): 55 | try: 56 | logger.info("route 53 recovery cluster endpoint: " + endpoint) 57 | client = boto3.client('route53-recovery-cluster', region_name=region, endpoint_url=endpoint) 58 | 59 | logger.info("toggling routing control") 60 | routing_control_state = client.get_routing_control_state(RoutingControlArn=routing_control_arn) 61 | logger.info("Current Routing Control State: " + routing_control_state["RoutingControlState"]) 62 | if routing_control_state["RoutingControlState"] == "On": 63 | client.update_routing_control_state(RoutingControlArn=routing_control_arn, RoutingControlState="Off") 64 | routing_control_state = client.get_routing_control_state(RoutingControlArn=routing_control_arn) 65 | updated_routing_control_state = routing_control_state["RoutingControlState"] 66 | logger.info("Updated routing Control State is " + updated_routing_control_state) 67 | done = True 68 | break 69 | else: 70 | client.update_routing_control_state(RoutingControlArn=routing_control_arn, RoutingControlState="On") 71 | routing_control_state = client.get_routing_control_state(RoutingControlArn=routing_control_arn) 72 | updated_routing_control_state = routing_control_state["RoutingControlState"] 73 | logger.info("Updated routing Control State is " + updated_routing_control_state) 74 | done = True 75 | break 76 | except Exception as e: 77 | logger.exception("Exception occurred while toggling ARC Routing Control") 78 | if done: 79 | metrics.add_metric(name="RegionalFailover", unit=MetricUnit.Count, value=1) 80 | break 81 | return {'routing_control_state': updated_routing_control_state} 82 | 83 | 84 | def dummy(event, context): 85 | logger.info("dummy") 86 | 87 | 88 | if __name__ == "__main__": 89 | event = dict() 90 | event["FUNCTION"] = "dummy" 91 | lambda_handler(event, None) 92 | -------------------------------------------------------------------------------- /source/failover/requirements.txt: -------------------------------------------------------------------------------- 1 | aws_lambda_powertools -------------------------------------------------------------------------------- /source/get-data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/source/get-data/__init__.py -------------------------------------------------------------------------------- /source/get-data/app.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | import boto3 4 | import os 5 | import json 6 | from botocore.exceptions import ClientError 7 | from aws_lambda_powertools.utilities.validation import validate 8 | from aws_lambda_powertools.utilities.validation.exceptions import SchemaValidationError 9 | import schemas 10 | from aws_lambda_powertools import Logger, Tracer, Metrics 11 | from aws_lambda_powertools.logging import correlation_paths 12 | 13 | tracer = Tracer() 14 | logger = Logger() 15 | 16 | dynamodb = boto3.resource('dynamodb') 17 | 18 | 19 | @logger.inject_lambda_context(log_event=True, clear_state=True, correlation_id_path=correlation_paths.API_GATEWAY_REST) 20 | @tracer.capture_lambda_handler 21 | def lambda_handler(event, context): 22 | request = event.get('pathParameters') 23 | 24 | uuid = request.get('uuid') 25 | 26 | input_object = {"uuid": uuid} 27 | 28 | try: 29 | validate(event=input_object, schema=schemas.INPUT) 30 | except SchemaValidationError as e: 31 | return {"response": "failure", "error": e} 32 | 33 | table_name = os.environ['TABLE_NAME'] 34 | 35 | table = dynamodb.Table(table_name) 36 | 37 | try: 38 | response = table.get_item( 39 | Key={ 40 | 'uuid': request.get('uuid') 41 | } 42 | ) 43 | except ClientError as e: 44 | logger.exception("Exception occurred while accessing DDB Table") 45 | else: 46 | item = response['Item'] 47 | 48 | return { 49 | 'statusCode': 200, 50 | 'body': json.dumps({"item": item}) 51 | } 52 | -------------------------------------------------------------------------------- /source/get-data/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | aws-lambda-powertools 3 | fastjsonschema -------------------------------------------------------------------------------- /source/get-data/schemas.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | INPUT = { 4 | "$schema": "http://json-schema.org/draft-07/schema", 5 | "$id": "http://example.com/example.json", 6 | "type": "object", 7 | "title": "Batch processing sample schema for the use case", 8 | "description": "The root schema comprises the entire JSON document.", 9 | "required": ["uuid"], 10 | "properties": { 11 | "uuid": { 12 | "type": "string", 13 | "maxLength": 9, 14 | "pattern": "[0-9]{9}" 15 | } 16 | }, 17 | } -------------------------------------------------------------------------------- /source/merge-s3-files/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/source/merge-s3-files/__init__.py -------------------------------------------------------------------------------- /source/merge-s3-files/app.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | import boto3 4 | from aws_lambda_powertools import Logger, Tracer, Metrics 5 | metrics = Metrics() 6 | tracer = Tracer() 7 | logger = Logger() 8 | s3_client = boto3.client('s3') 9 | 10 | @metrics.log_metrics(capture_cold_start_metric=False) 11 | @logger.inject_lambda_context(log_event=True, clear_state=True) 12 | @tracer.capture_lambda_handler 13 | def lambda_handler(event, context): 14 | bucket = event['bucket'] 15 | key = event['key'] 16 | to_process_folder = event['toProcessFolder'] 17 | data = [] 18 | output_path = to_process_folder.replace("to_process", "output") 19 | 20 | output = [] 21 | 22 | header_text = [ 23 | 'uuid', 24 | 'Country', 25 | 'Item Type', 26 | 'Sales Channel', 27 | 'Order Priority', 28 | 'Order Date', 29 | 'Region', 30 | 'Ship Date', 31 | 'Units Sold', 32 | 'Unit Price', 33 | 'Unit Cost', 34 | 'Total Revenue', 35 | 'Total Cost', 36 | 'Total Profit' 37 | 38 | ] 39 | 40 | output.append(",".join(header_text) + "\n") 41 | 42 | try: 43 | for item in s3_client.list_objects_v2(Bucket=bucket, Prefix=output_path)['Contents']: 44 | if item['Key'].endswith('.csv'): 45 | resp = s3_client.select_object_content( 46 | Bucket=bucket, 47 | Key=item['Key'], 48 | ExpressionType='SQL', 49 | Expression="select * from s3object", 50 | InputSerialization={'CSV': {"FileHeaderInfo": "NONE"}, 'CompressionType': 'NONE'}, 51 | OutputSerialization={'CSV': {}}, 52 | ) 53 | 54 | for event in resp['Payload']: 55 | if 'Records' in event: 56 | records = event['Records']['Payload'].decode('utf-8') 57 | payloads = (''.join(response for response in records)) 58 | output.append(payloads) 59 | 60 | output_body = "".join(output) 61 | s3_target_key = output_path + "/" + get_output_filename(key) 62 | response = s3_client.put_object(Bucket=bucket, 63 | Key=s3_target_key, 64 | Body=output_body) 65 | 66 | line_num = 0 67 | lines = output_body.splitlines(); 68 | for line in lines: 69 | words = line.split(",") 70 | if line_num > 0: 71 | data.append(words[0]) 72 | line_num += 1 73 | 74 | logger.info("Data", input_file=key, data=data) 75 | return {"response": response, "S3OutputFileName": s3_target_key, "originalFileName": key} 76 | 77 | except Exception as e: 78 | logger.exception("Exception occurred while merging files") 79 | raise Exception(str(e)) 80 | 81 | 82 | def get_output_filename(key): 83 | last_part_pos = key.rfind("/") 84 | if last_part_pos == -1: 85 | return "" 86 | last_part_pos += 1 87 | input_file_name = key[last_part_pos:] 88 | 89 | return "completed/" + input_file_name 90 | -------------------------------------------------------------------------------- /source/merge-s3-files/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | aws_lambda_powertools -------------------------------------------------------------------------------- /source/read-file/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/source/read-file/__init__.py -------------------------------------------------------------------------------- /source/read-file/app.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | import csv 4 | import s3fs 5 | from aws_lambda_powertools import Logger, Tracer, Metrics 6 | from aws_lambda_powertools.metrics import MetricUnit 7 | 8 | metrics = Metrics() 9 | tracer = Tracer() 10 | logger = Logger() 11 | 12 | s3 = s3fs.S3FileSystem(anon=False) 13 | 14 | header = [ 15 | 'uuid', 16 | 'country', 17 | 'itemType', 18 | 'salesChannel', 19 | 'orderPriority', 20 | 'orderDate', 21 | 'region', 22 | 'shipDate' 23 | ] 24 | 25 | @metrics.log_metrics(capture_cold_start_metric=False) 26 | @logger.inject_lambda_context(log_event=True, clear_state=True) 27 | @tracer.capture_lambda_handler(capture_response=False) 28 | def lambda_handler(event, context): 29 | input_file = event['input']['FilePath'] 30 | output_data = [] 31 | skip_first = 0 32 | with s3.open(input_file, 'r', newline='', encoding='utf-8-sig') as inFile: 33 | file_reader = csv.reader(inFile) 34 | for row in file_reader: 35 | if skip_first == 0: 36 | skip_first = skip_first + 1 37 | continue 38 | new_object = {} 39 | for i in range(len(header)): 40 | new_object[header[i]] = row[i] 41 | 42 | output_data.append(new_object) 43 | 44 | return output_data 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /source/read-file/requirements.txt: -------------------------------------------------------------------------------- 1 | s3fs 2 | aws_lambda_powertools -------------------------------------------------------------------------------- /source/reconciliation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/source/reconciliation/__init__.py -------------------------------------------------------------------------------- /source/reconciliation/app.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | 5 | import boto3 6 | from boto3.dynamodb.conditions import Key 7 | from aws_lambda_powertools import Logger, Tracer, Metrics 8 | from aws_lambda_powertools.metrics import MetricUnit 9 | 10 | metrics = Metrics() 11 | tracer = Tracer() 12 | logger = Logger() 13 | ddb_client = boto3.resource('dynamodb') 14 | s3 = boto3.resource('s3') 15 | 16 | @metrics.log_metrics(capture_cold_start_metric=False) 17 | @logger.inject_lambda_context(log_event=True, clear_state=True) 18 | @tracer.capture_lambda_handler 19 | def lambda_handler(event, context): 20 | table_name = os.environ['BATCH_STATE_DDB'] 21 | # runtime_region = os.environ['AWS_REGION'] 22 | secondary_region_bucket = os.environ['SECONDARY_REGION_BUCKET'] 23 | table = ddb_client.Table(table_name) 24 | resp = table.query( 25 | # Add the name of the index you want to use in your query. 26 | IndexName="status-index", 27 | KeyConditionExpression=Key('status').eq('INITIALIZED')) 28 | logger.info({"Number of total unprocessed files:", len(resp['Items'])}) 29 | copied_files = [] 30 | for item in resp['Items']: 31 | logger.info(item) 32 | record_obj = item['s3NotificationEvent'] 33 | logger.info({'s3_event_record_obj: ', json.dumps(record_obj)}) 34 | s3_event = json.loads(record_obj) 35 | key = s3_event['Records']['s3']['object']['key'] 36 | try: 37 | copy_source = { 38 | 'Bucket': secondary_region_bucket, 39 | 'Key': key 40 | } 41 | bucket = s3.Bucket(secondary_region_bucket) 42 | response_data = bucket.copy(copy_source, key) 43 | logger.info({"----- file copied successfully: ", json.dumps(response_data)}) 44 | 45 | except Exception as err: 46 | logger.exception({"Error while copying Input File:", key}) 47 | 48 | else: 49 | copied_files.append(key) 50 | logger.info({"file submitted for processing": key, "response_data": json.dumps(response_data)}) 51 | metrics.add_metric(name="ReconciledFiles", unit=MetricUnit.Count, value=len(copied_files)) 52 | return { 53 | 'num_files_submitted_for_reconciliation': len(copied_files), 54 | 'file_list': copied_files 55 | } 56 | -------------------------------------------------------------------------------- /source/reconciliation/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | aws_lambda_powertools -------------------------------------------------------------------------------- /source/s3-lambda-notification/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/source/s3-lambda-notification/__init__.py -------------------------------------------------------------------------------- /source/s3-lambda-notification/app.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | import json 4 | import os 5 | import boto3 6 | import time 7 | import logging 8 | from datetime import date, datetime 9 | import dns.resolver 10 | from aws_lambda_powertools import Logger, Tracer, Metrics 11 | from aws_lambda_powertools.metrics import MetricUnit 12 | 13 | metrics = Metrics() 14 | tracer = Tracer() 15 | logger = Logger() 16 | 17 | state_machine_client = boto3.client('stepfunctions') 18 | dynamodb = boto3.resource('dynamodb') 19 | 20 | @tracer.capture_method 21 | def write_to_ddb(fileName, status, process_date, start_time, param): 22 | table_name = os.environ['BATCH_STATE_DDB'] 23 | table = dynamodb.Table(table_name) 24 | runtime_region = os.environ['AWS_REGION'] 25 | event_object = json.dumps(param) 26 | response = table.put_item( 27 | Item={ 28 | 'fileName': fileName, 29 | 'status': status, 30 | 'processDate': process_date, 31 | 'startTime': start_time, 32 | 'processingInitializedRegion': runtime_region, 33 | 's3NotificationEvent': event_object 34 | 35 | } 36 | ) 37 | return response 38 | 39 | @tracer.capture_method 40 | def resolve_secret_value(param): 41 | session = boto3.session.Session() 42 | client = session.client( 43 | service_name='secretsmanager', 44 | region_name=os.environ['AWS_REGION'], 45 | ) 46 | get_secret_value_response = client.get_secret_value( 47 | SecretId=param 48 | ) 49 | return get_secret_value_response['SecretString'] 50 | 51 | @metrics.log_metrics(capture_cold_start_metric=False) 52 | @logger.inject_lambda_context(log_event=True, clear_state=True) 53 | @tracer.capture_lambda_handler 54 | def lambda_handler(event, context): 55 | domain_name = resolve_secret_value(os.environ['DNS_RECORD_SECRET']) 56 | answers = dns.resolver.query(domain_name, 'TXT') 57 | primary_region = answers[0].to_text().replace('"', '') 58 | current_region = os.environ['AWS_REGION'] 59 | logger.info({"Primary Region": primary_region, "Current Region": current_region}) 60 | if current_region == primary_region: 61 | for record in event['Records']: 62 | param = { 63 | "Records": record, 64 | "inputArchiveFolder": os.environ['INPUT_ARCHIVE_FOLDER'], 65 | "fileChunkSize": int(os.environ['FILE_CHUNK_SIZE']), 66 | "fileDelimiter": os.environ['FILE_DELIMITER'] 67 | 68 | } 69 | state_machine_arn = os.environ['STATE_MACHINE_ARN'] 70 | state_machine_execution_name = os.environ['STATE_MACHINE_EXECUTION_NAME'] + str(time.time()) 71 | bucket = record['s3']['bucket']['name'] 72 | key = record['s3']['object']['key'] 73 | current_date = date.today().strftime('%m/%d/%Y') 74 | current_time = datetime.now().strftime("%H:%M:%S") 75 | responseData = {} 76 | try: 77 | responseData['step_function_response'] = state_machine_client.start_execution( 78 | stateMachineArn=state_machine_arn, 79 | name=state_machine_execution_name, 80 | input=json.dumps(param) 81 | ) 82 | write_to_ddb(key, 'INITIALIZED', current_date, current_time, param) 83 | responseData['ddb_state_table_put'] = 'SUCCESS' 84 | logging.info({"File": key, "Bucket": bucket, "Status": "Initialized", "Response Data": responseData}) 85 | except Exception as err: 86 | logger.exception({"Input File Processing Error": key}) 87 | write_to_ddb(key, 'FAILED', current_date, current_time, param) 88 | responseData['ddb_state_table_put'] = 'FAILED' 89 | logging.info({"File": key, "Bucket": bucket, "Status": "Failed", "Response Data": responseData}) 90 | else: 91 | logger.info("Current Region is not primary region, hence skipping the processing") 92 | for record in event['Records']: 93 | bucket = record['s3']['bucket']['name'] 94 | key = record['s3']['object']['key'] 95 | logging.info({"File": key, "Bucket": bucket, "Status": "Skipped"}) -------------------------------------------------------------------------------- /source/s3-lambda-notification/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | dnspython 3 | aws_lambda_powertools -------------------------------------------------------------------------------- /source/send-email/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/source/send-email/__init__.py -------------------------------------------------------------------------------- /source/send-email/app.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | import base64 4 | import hashlib 5 | import hmac 6 | import json 7 | import os 8 | import smtplib 9 | from email.message import EmailMessage 10 | 11 | import boto3 12 | from aws_lambda_powertools import Logger, Tracer, Metrics 13 | from aws_lambda_powertools.metrics import MetricUnit 14 | 15 | metrics = Metrics() 16 | tracer = Tracer() 17 | logger = Logger() 18 | s3_client = boto3.client('s3') 19 | ddb_client = boto3.resource('dynamodb') 20 | 21 | @tracer.capture_method 22 | def get_mrap_alias(mrap_alias_secret): 23 | session = boto3.session.Session() 24 | client = session.client( 25 | service_name='secretsmanager', 26 | region_name=os.environ['AWS_REGION'], 27 | ) 28 | get_secret_value_response = client.get_secret_value( 29 | SecretId=mrap_alias_secret 30 | ) 31 | return get_secret_value_response['SecretString'] 32 | 33 | @metrics.log_metrics(capture_cold_start_metric=False) 34 | @logger.inject_lambda_context(log_event=True, clear_state=True) 35 | @tracer.capture_lambda_handler 36 | def lambda_handler(event, context): 37 | sender = event['sender'] 38 | recipient = event['recipient'] 39 | 40 | s3_output_file = event['s3OutputFileName'] 41 | original_file_name = event['originalFileName'] 42 | 43 | logger.info({"original file name": original_file_name}) 44 | 45 | update_status(original_file_name) 46 | logger.info("Status updated in Batch Status Table") 47 | account_id = context.invoked_function_arn.split(":")[4] 48 | mrap_alias = get_mrap_alias(os.environ['MRAP_ALIAS_SECRET']) 49 | pre_signed_url = generate_s3_signed_url(account_id, mrap_alias, s3_output_file) 50 | logger.info("Generated Presigned URL") 51 | 52 | send_email(sender, recipient, pre_signed_url) 53 | metrics.add_metric(name="EmailSent", unit=MetricUnit.Count, value=1) 54 | 55 | return {"response": "success"} 56 | 57 | @tracer.capture_method 58 | def generate_s3_signed_url(account_id, mrap_alias, s3_target_key): 59 | return s3_client.generate_presigned_url(HttpMethod='GET', 60 | ClientMethod='get_object', 61 | Params={'Bucket': 'arn:aws:s3::' + account_id + ':accesspoint/' + mrap_alias, 62 | 'Key': s3_target_key}, 63 | ExpiresIn=3600) 64 | 65 | @tracer.capture_method 66 | def update_status(file_name): 67 | status = 'COMPLETED' 68 | table_name = os.environ['BATCH_STATE_DDB'] 69 | runtime_region = os.environ['AWS_REGION'] 70 | logger.info({"current_region", runtime_region}) 71 | table = ddb_client.Table(table_name) 72 | response = table.update_item( 73 | Key={'fileName': file_name}, 74 | UpdateExpression="set #status = :s, #completedRegion = :cRegion", 75 | ExpressionAttributeValues={ 76 | ':s': status, 77 | ':cRegion': runtime_region 78 | }, 79 | ExpressionAttributeNames={ 80 | '#status': 'status', 81 | '#completedRegion': 'processingCompletedRegion' 82 | }, 83 | ReturnValues="UPDATED_NEW") 84 | 85 | return response 86 | 87 | @tracer.capture_method 88 | def send_email(sender, recipient, pre_signed_url): 89 | # The subject line for the email. 90 | subject = "Batch Processing complete: Output file information" 91 | 92 | # The HTML body of the email. 93 | body_html = """ 94 | 95 | 96 |

The file has been processed successfully

97 |

Click the pre-signed S3 URL to access the output file: 98 | Output File

99 |

The link will expire in 60 minutes.

100 | 101 | """.format(url=pre_signed_url) 102 | 103 | # construct email 104 | email = EmailMessage() 105 | email['Subject'] = subject 106 | email['From'] = sender 107 | email['To'] = recipient 108 | email.set_content(body_html, subtype='html') 109 | # Try to send the email. 110 | try: 111 | smtp_credentials = get_smtp_credentials() 112 | response = transmit_email(email, smtp_credentials) 113 | except Exception as e: 114 | logger.exception('Unable to send email') 115 | else: 116 | logger.info("EMail sent!") 117 | 118 | @tracer.capture_method 119 | def transmit_email(email, smtp_credentials): 120 | username = smtp_credentials['AccessKey'] 121 | password = calculate_key(smtp_credentials['SecretAccessKey'], os.environ['AWS_REGION']) 122 | server = smtplib.SMTP(os.environ['SMTP_HOST'], 25) 123 | server.starttls() 124 | server.login(username, password) 125 | response = server.send_message(email) 126 | server.close() 127 | return response 128 | 129 | @tracer.capture_method 130 | def get_smtp_credentials(): 131 | session = boto3.session.Session() 132 | client = session.client( 133 | service_name='secretsmanager', 134 | region_name=os.environ['AWS_REGION'], 135 | ) 136 | get_secret_value_response = client.get_secret_value( 137 | SecretId=os.environ['SMTP_CREDENTIAL_SECRET'] 138 | ) 139 | json_secret_value = json.loads(get_secret_value_response['SecretString']) 140 | return json_secret_value 141 | 142 | 143 | # These values are required to calculate the signature. Do not change them. 144 | DATE = "11111111" 145 | SERVICE = "ses" 146 | MESSAGE = "SendRawEmail" 147 | TERMINAL = "aws4_request" 148 | VERSION = 0x04 149 | 150 | 151 | def sign(key, msg): 152 | return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest() 153 | 154 | 155 | def calculate_key(secret_access_key, region): 156 | signature = sign(("AWS4" + secret_access_key).encode('utf-8'), DATE) 157 | signature = sign(signature, region) 158 | signature = sign(signature, SERVICE) 159 | signature = sign(signature, TERMINAL) 160 | signature = sign(signature, MESSAGE) 161 | signature_and_version = bytes([VERSION]) + signature 162 | smtp_password = base64.b64encode(signature_and_version) 163 | return smtp_password.decode('utf-8') 164 | -------------------------------------------------------------------------------- /source/send-email/requirements.txt: -------------------------------------------------------------------------------- 1 | aiobotocore==2.4.2 2 | aiohttp==3.8.3 3 | aioitertools==0.11.0 4 | aiosignal==1.3.1 5 | async-timeout==4.0.2 6 | attrs==22.2.0 7 | awscrt==0.14.0 8 | boto3==1.24.59 9 | botocore==1.27.59 10 | build==0.10.0 11 | charset-normalizer==2.1.1 12 | click==8.1.3 13 | frozenlist==1.3.3 14 | fsspec==2022.11.0 15 | idna==3.4 16 | jmespath==1.0.1 17 | multidict==6.0.4 18 | packaging==23.0 19 | pip-tools==6.12.1 20 | pipdeptree==2.3.3 21 | pyproject_hooks==1.0.0 22 | python-dateutil==2.8.2 23 | PyYAML==6.0 24 | s3fs==2022.11.0 25 | s3transfer==0.6.0 26 | six==1.16.0 27 | tomli==2.0.1 28 | urllib3==1.26.14 29 | wrapt==1.14.1 30 | yarl==1.8.2 31 | aws_lambda_powertools -------------------------------------------------------------------------------- /source/split-ip-file/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/source/split-ip-file/__init__.py -------------------------------------------------------------------------------- /source/split-ip-file/app.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | import os 4 | import uuid 5 | 6 | import s3fs 7 | from aws_lambda_powertools import Logger, Tracer, Metrics 8 | from aws_lambda_powertools.metrics import MetricUnit 9 | 10 | metrics = Metrics() 11 | tracer = Tracer() 12 | logger = Logger() 13 | # S3 bucket info 14 | s3 = s3fs.S3FileSystem(anon=False) 15 | 16 | 17 | @metrics.log_metrics(capture_cold_start_metric=False) 18 | @logger.inject_lambda_context(log_event=True, clear_state=True) 19 | @tracer.capture_lambda_handler 20 | def lambda_handler(event, context): 21 | input_archive_folder = event['inputArchiveFolder'] 22 | to_process_folder = str(uuid.uuid4()) + "/" + "to_process" 23 | file_row_limit = event['fileChunkSize'] 24 | file_delimiter = event['fileDelimiter'] 25 | output_path = to_process_folder.replace("to_process", "output") 26 | 27 | record = event['Records'] 28 | 29 | bucket = record['s3']['bucket']['name'] 30 | key = record['s3']['object']['key'] 31 | logger.append_keys(s3_object_key=key) 32 | create_start_indicator(bucket, output_path) 33 | input_file = os.path.join(bucket, key) 34 | archive_path = os.path.join(bucket, input_archive_folder, os.path.basename(key)) 35 | folder = os.path.split(key)[0] 36 | s3_url = os.path.join(bucket, folder) 37 | output_file_template = os.path.splitext(os.path.basename(key))[0] + "__part" 38 | output_path = os.path.join(bucket, to_process_folder) 39 | 40 | # Number of files to be created 41 | num_files = file_count(s3.open(input_file, 'r'), file_delimiter, file_row_limit) 42 | # Split the input file into several files, each with the number of records mentioned in the fileChunkSize parameter. 43 | splitFileNames = split(input_file, 44 | s3.open(input_file, 'r'), 45 | file_delimiter, 46 | file_row_limit, 47 | output_file_template, 48 | output_path, True, 49 | num_files) 50 | # Archive the input file. 51 | archive(input_file, archive_path) 52 | 53 | response = {"bucket": bucket, "key": key, "splitFileNames": splitFileNames, 54 | "toProcessFolder": to_process_folder} 55 | metrics.add_metric(name="InputFilesSplit", unit=MetricUnit.Count, value=1) 56 | logger.info(response) 57 | return response 58 | 59 | 60 | # Determine the number of files that this Lambda function will create. 61 | def file_count(file_handler, delimiter, row_limit): 62 | import csv 63 | reader = csv.reader(file_handler, delimiter=delimiter) 64 | # Figure out the number of files this function will generate. 65 | row_count = sum(1 for row in reader) - 1 66 | # If there's a remainder, always round up. 67 | file_count = int(row_count // row_limit) + (row_count % row_limit > 0) 68 | return file_count 69 | 70 | 71 | # Split the input into several smaller files. 72 | def split(input_file, filehandler, delimiter, row_limit, output_name_template, output_path, keep_headers, num_files): 73 | import csv 74 | reader = csv.reader(filehandler, delimiter=delimiter) 75 | split_file_path = [] 76 | data = [] 77 | current_piece = 1 78 | current_out_path = os.path.join( 79 | output_path, 80 | output_name_template + str(current_piece) + "__of" + str(num_files) + ".csv" 81 | ) 82 | split_file_path.append(current_out_path) 83 | current_out_writer = csv.writer(s3.open(current_out_path, 'w'), delimiter=delimiter, quoting=csv.QUOTE_ALL) 84 | current_limit = row_limit 85 | if keep_headers: 86 | headers = next(reader) 87 | current_out_writer.writerow(headers) 88 | for i, row in enumerate(reader): 89 | if i + 1 > current_limit: 90 | current_piece += 1 91 | current_limit = row_limit * current_piece 92 | current_out_path = os.path.join( 93 | output_path, 94 | output_name_template + str(current_piece) + "__of" + str(num_files) + ".csv" 95 | ) 96 | split_file_path.append(current_out_path) 97 | current_out_writer = csv.writer(s3.open(current_out_path, 'w'), delimiter=delimiter, quoting=csv.QUOTE_ALL) 98 | if keep_headers: 99 | current_out_writer.writerow(headers) 100 | current_out_writer.writerow(row) 101 | data.append(row[0]) 102 | logger.info("Data", input_file=input_file, data=data) 103 | return split_file_path 104 | 105 | 106 | @tracer.capture_method 107 | # Move the original input file into an archive folder. 108 | def archive(input_file, archive_path): 109 | s3.copy(input_file, archive_path) 110 | s3.rm(input_file) 111 | 112 | 113 | def create_start_indicator(bucket, folder_name): 114 | response = s3.touch(bucket + "/" + folder_name + "/_started") 115 | -------------------------------------------------------------------------------- /source/split-ip-file/requirements.txt: -------------------------------------------------------------------------------- 1 | s3fs 2 | aws_lambda_powertools -------------------------------------------------------------------------------- /source/statemachine/blog-sfn-main-orchestrator.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "State machine for batch processing", 3 | "StartAt": "Split Input File into chunks", 4 | "States": { 5 | "Split Input File into chunks": { 6 | "Type": "Task", 7 | "ResultPath": "$.splitOutput", 8 | "Resource": "${SplitInputFileFunctionArn}", 9 | "Next": "Call Step function for each chunk", 10 | "Retry": [ 11 | { 12 | "ErrorEquals": [ 13 | "Lambda.TooManyRequestsException" 14 | ], 15 | "IntervalSeconds": 5, 16 | "MaxAttempts": 5, 17 | "BackoffRate": 2 18 | } 19 | ] 20 | }, 21 | "Call Step function for each chunk": { 22 | "Type": "Map", 23 | "Next": "Merge all Files", 24 | "ItemsPath": "$.splitOutput.splitFileNames", 25 | "ResultPath": null, 26 | "Parameters": { 27 | "FilePath.$": "$$.Map.Item.Value", 28 | "FileIndex.$": "$$.Map.Item.Index" 29 | }, 30 | "Iterator": { 31 | "StartAt": "Call Chunk Processor Workflow", 32 | "States": { 33 | "Call Chunk Processor Workflow": { 34 | "Type": "Task", 35 | "Resource": "arn:aws:states:::states:startExecution.sync:2", 36 | "Parameters": { 37 | "Input": { 38 | "input": { 39 | "FilePath.$": "$.FilePath" 40 | } 41 | }, 42 | "StateMachineArn": "${BlogBatchProcessChunkArn}" 43 | }, 44 | "End": true 45 | } 46 | } 47 | } 48 | }, 49 | "Merge all Files": { 50 | "Type": "Task", 51 | "Resource": "${MergeS3FilesFunctionArn}", 52 | "Parameters": { 53 | "toProcessFolder.$": "$.splitOutput.toProcessFolder", 54 | "bucket.$": "$.splitOutput.bucket", 55 | "key.$": "$.splitOutput.key" 56 | }, 57 | "ResultPath": "$.mergeResponse", 58 | "Next": "Email the file", 59 | "Retry": [ 60 | { 61 | "ErrorEquals": [ 62 | "States.ALL" 63 | ], 64 | "IntervalSeconds": 3, 65 | "MaxAttempts": 5, 66 | "BackoffRate": 2 67 | } 68 | ] 69 | }, 70 | "Email the file": { 71 | "Type": "Task", 72 | "Resource": "${SendEmailFunctionArn}", 73 | "Parameters": { 74 | "sender": "${SESSender}", 75 | "recipient": "${SESRecipient}", 76 | "bucket.$": "$.splitOutput.bucket", 77 | "s3OutputFileName.$": "$.mergeResponse.S3OutputFileName", 78 | "originalFileName.$": "$.mergeResponse.originalFileName" 79 | }, 80 | "Retry": [ 81 | { 82 | "ErrorEquals": [ 83 | "States.ALL" 84 | ], 85 | "IntervalSeconds": 3, 86 | "MaxAttempts": 5, 87 | "BackoffRate": 2 88 | } 89 | ], 90 | "End": true 91 | } 92 | } 93 | } -------------------------------------------------------------------------------- /source/statemachine/blog-sfn-process-chunk.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "AWS Step Functions example for batch processing", 3 | "StartAt": "Read File", 4 | "States": { 5 | "Read File": { 6 | "Type": "Task", 7 | "ResultPath": "$.fileContents", 8 | "Resource": "${ReadFileFunctionArn}", 9 | "Next": "Process messages", 10 | "Retry": [ 11 | { 12 | "ErrorEquals": [ 13 | "Lambda.TooManyRequestsException" 14 | ], 15 | "IntervalSeconds": 10, 16 | "MaxAttempts": 5, 17 | "BackoffRate": 2 18 | } 19 | ] 20 | }, 21 | "Process messages": { 22 | "Type": "Map", 23 | "Next": "Write output file", 24 | "ItemsPath": "$.fileContents", 25 | "ResultPath": "$.input.enrichedData", 26 | "OutputPath": "$.input", 27 | "Parameters": { 28 | "MessageNumber.$": "$$.Map.Item.Index", 29 | "MessageDetails.$": "$$.Map.Item.Value" 30 | }, 31 | "Iterator": { 32 | "StartAt": "Validate Data", 33 | "States": { 34 | "Validate Data": { 35 | "Type": "Task", 36 | "Resource": "${ValidateDataFunctionArn}", 37 | "InputPath": "$.MessageDetails", 38 | "ResultPath": "$.MessageDetails.validatedresult", 39 | "Next": "Get Financial Data", 40 | "Catch": [ 41 | { 42 | "ErrorEquals": [ 43 | "States.ALL" 44 | ], 45 | "ResultPath": "$.MessageDetails.error-info", 46 | "Next": "Store Error Record" 47 | } 48 | ] 49 | }, 50 | "Store Error Record": { 51 | "Type": "Task", 52 | "Resource": "arn:aws:states:::dynamodb:putItem", 53 | "InputPath": "$.MessageDetails", 54 | "OutputPath": "$.MessageDetails", 55 | "ResultPath": null, 56 | "Parameters": { 57 | "TableName": "${ErrorTableName}", 58 | "Item": { 59 | "uuid": { 60 | "S.$": "$.uuid" 61 | }, 62 | "country": { 63 | "S.$": "$.country" 64 | }, 65 | "itemType": { 66 | "S.$": "$.itemType" 67 | }, 68 | "salesChannel": { 69 | "S.$": "$.salesChannel" 70 | }, 71 | "orderPriority": { 72 | "S.$": "$.orderPriority" 73 | }, 74 | "orderDate": { 75 | "S.$": "$.orderDate" 76 | }, 77 | "region": { 78 | "S.$": "$.region" 79 | }, 80 | "shipDate": { 81 | "S.$": "$.shipDate" 82 | }, 83 | "error": { 84 | "S.$": "$.error-info.Error" 85 | }, 86 | "cause": { 87 | "S.$": "$.error-info.Cause" 88 | } 89 | } 90 | }, 91 | "Retry": [ 92 | { 93 | "ErrorEquals": [ 94 | "States.TaskFailed" 95 | ], 96 | "IntervalSeconds": 20, 97 | "MaxAttempts": 5, 98 | "BackoffRate": 10 99 | } 100 | ], 101 | "End": true 102 | }, 103 | "Get Financial Data": { 104 | "Type": "Task", 105 | "Resource": "arn:aws:states:::apigateway:invoke", 106 | "ResultPath": "$.MessageDetails.financialdata", 107 | "OutputPath": "$.MessageDetails", 108 | "ResultSelector": { 109 | "item.$": "$.ResponseBody.item" 110 | }, 111 | "Parameters": { 112 | "ApiEndpoint": "${ApiEndpoint}", 113 | "Method": "GET", 114 | "Stage": "Prod", 115 | "Path.$": "States.Format('financials/{}', $.MessageDetails.uuid)", 116 | "AuthType": "IAM_ROLE" 117 | }, 118 | "Retry": [ 119 | { 120 | "ErrorEquals": [ 121 | "States.TaskFailed" 122 | ], 123 | "IntervalSeconds": 3, 124 | "MaxAttempts": 5, 125 | "BackoffRate": 1 126 | } 127 | ], 128 | "End": true 129 | } 130 | } 131 | } 132 | }, 133 | "Write output file": { 134 | "Type": "Task", 135 | "Resource": "${WriteOutputChunkFunctionArn}", 136 | "ResultPath": "$.writeOutputFileResponse", 137 | "End": true, 138 | "Retry": [ 139 | { 140 | "ErrorEquals": [ 141 | "States.ALL" 142 | ], 143 | "IntervalSeconds": 3, 144 | "MaxAttempts": 5, 145 | "BackoffRate": 2 146 | } 147 | ] 148 | } 149 | } 150 | } -------------------------------------------------------------------------------- /source/validate-data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/source/validate-data/__init__.py -------------------------------------------------------------------------------- /source/validate-data/app.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | from aws_lambda_powertools.utilities.validation import validate 4 | from aws_lambda_powertools.utilities.validation.exceptions import SchemaValidationError 5 | import schemas 6 | 7 | from aws_lambda_powertools import Logger, Tracer, Metrics 8 | from aws_lambda_powertools.metrics import MetricUnit 9 | 10 | metrics = Metrics() 11 | tracer = Tracer() 12 | logger = Logger() 13 | 14 | @metrics.log_metrics(capture_cold_start_metric=False) 15 | @logger.inject_lambda_context(log_event=True, clear_state=True) 16 | @tracer.capture_lambda_handler 17 | def lambda_handler(event, context): 18 | try: 19 | validate(event=event, schema=schemas.INPUT) 20 | except SchemaValidationError as e: 21 | return {"response": "failure", "error": e} 22 | 23 | return {"response": "success"} 24 | -------------------------------------------------------------------------------- /source/validate-data/requirements.txt: -------------------------------------------------------------------------------- 1 | aws-lambda-powertools 2 | fastjsonschema -------------------------------------------------------------------------------- /source/validate-data/schemas.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | INPUT = { 4 | "$schema": "http://json-schema.org/draft-07/schema", 5 | "$id": "http://example.com/example.json", 6 | "type": "object", 7 | "title": "Batch processing sample schema for the use case", 8 | "description": "The root schema comprises the entire JSON document.", 9 | "required": ["uuid", "country", "itemType", "salesChannel", "orderPriority", "orderDate", "region", "shipDate"], 10 | "properties": { 11 | "uuid": { 12 | "type": "string", 13 | "maxLength": 9, 14 | }, 15 | "country": { 16 | "type": "string", 17 | "maxLength": 50, 18 | }, 19 | "itemType": { 20 | "type": "string", 21 | "maxLength": 30, 22 | }, 23 | "salesChannel": { 24 | "type": "string", 25 | "maxLength": 10, 26 | }, 27 | "orderPriority": { 28 | "type": "string", 29 | "maxLength": 5, 30 | }, 31 | "orderDate": { 32 | "type": "string", 33 | "maxLength": 10, 34 | }, 35 | "region": { 36 | "type": "string", 37 | "maxLength": 100, 38 | }, 39 | "shipDate": { 40 | "type": "string", 41 | "maxLength": 10, 42 | } 43 | }, 44 | } -------------------------------------------------------------------------------- /source/write-output-chunk/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-multi-region-serverless-batch-applications-on-aws/f97050ff3701f140bd954490205025700898e9eb/source/write-output-chunk/__init__.py -------------------------------------------------------------------------------- /source/write-output-chunk/app.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | import csv 4 | import boto3 5 | 6 | import json 7 | from io import StringIO 8 | from aws_lambda_powertools import Logger, Tracer, Metrics 9 | from aws_lambda_powertools.metrics import MetricUnit 10 | 11 | metrics = Metrics() 12 | tracer = Tracer() 13 | logger = Logger() 14 | s3_client = boto3.client('s3') 15 | 16 | header = [ 17 | 'uuid', 18 | 'country', 19 | 'itemType', 20 | 'salesChannel', 21 | 'orderPriority', 22 | 'orderDate', 23 | 'region', 24 | 'shipDate', 25 | 'unitsSold', 26 | 'unitPrice', 27 | 'unitCost', 28 | 'totalRevenue', 29 | 'totalCost', 30 | 'totalProfit' 31 | 32 | ] 33 | 34 | @metrics.log_metrics(capture_cold_start_metric=False) 35 | @logger.inject_lambda_context(log_event=True, clear_state=True) 36 | @tracer.capture_lambda_handler 37 | def lambda_handler(event, context): 38 | dataset = event['enrichedData'] 39 | input_file_key = event['FilePath'] 40 | output_file_key = input_file_key.replace("to_process", "output") 41 | bucket_info = get_bucket_info(output_file_key) 42 | logger.info(bucket_info) 43 | 44 | out_file = StringIO() 45 | file_writer = csv.writer(out_file, quoting=csv.QUOTE_ALL) 46 | 47 | for data in dataset: 48 | if 'error-info' in data: 49 | continue 50 | data_list = convert_to_list(data) 51 | file_writer.writerow(data_list) 52 | 53 | response = s3_client.put_object(Bucket=bucket_info['bucket'], 54 | Key=bucket_info['key'], 55 | Body=out_file.getvalue()) 56 | 57 | if response['ResponseMetadata']['HTTPStatusCode'] != 200: 58 | message = 'Writing chunk to S3 failed' + json.dumps(response, indent=2) 59 | logger.exception(message) 60 | raise Exception(message) 61 | 62 | return {"response": "success"} 63 | 64 | 65 | def convert_to_list(data): 66 | data_list = [data['uuid'], data['country'], data['itemType'], data['salesChannel'], data['orderPriority'], 67 | data['orderDate'], data['region'], data['shipDate'], 68 | data['financialdata']['item']['unitsSold'], 69 | data['financialdata']['item']['unitPrice'], 70 | data['financialdata']['item']['unitCost'], 71 | data['financialdata']['item']['totalRevenue'], 72 | data['financialdata']['item']['totalCost'], 73 | data['financialdata']['item']['totalProfit']] 74 | 75 | return data_list 76 | 77 | def get_bucket_info(filename): 78 | first_part_pos = filename.find("/") 79 | if first_part_pos == -1: 80 | return "" 81 | bucket_name = filename[:first_part_pos] 82 | file_prefix = filename[(first_part_pos + 1):] 83 | 84 | return {"bucket": bucket_name, "key": file_prefix} 85 | 86 | -------------------------------------------------------------------------------- /source/write-output-chunk/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | aws_lambda_powertools --------------------------------------------------------------------------------