├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── SECURITY.md ├── deployment ├── aws-waf-security-automations-firehose-athena.template ├── aws-waf-security-automations-webacl.template ├── aws-waf-security-automations.template ├── build-s3-dist.sh └── run-unit-tests.sh └── source ├── access_handler ├── .coveragerc ├── __init__.py ├── access_handler.py ├── poetry.lock ├── pyproject.toml └── test │ ├── __init__.py │ ├── conftest.py │ └── test_access_handler.py ├── custom_resource ├── .coveragerc ├── __init__.py ├── custom_resource.py ├── log_group_retention.py ├── operations │ ├── __init__.py │ ├── add_athena_partitions.py │ ├── config_app_access_log_bucket.py │ ├── config_aws_waf_logs.py │ ├── config_waf_log_bucket.py │ ├── config_web_acl.py │ ├── generate_app_log_parser_conf.py │ ├── generate_waf_log_parser_conf.py │ ├── operation_types.py │ └── set_log_group_retention.py ├── poetry.lock ├── pyproject.toml ├── resource_manager.py └── test │ ├── __init__.py │ ├── conftest.py │ ├── test_custom_resource.py │ ├── test_log_group_retention.py │ └── test_resource_manager.py ├── helper ├── .coveragerc ├── __init__.py ├── helper.py ├── poetry.lock ├── pyproject.toml ├── stack_requirements.py └── test │ ├── __init__.py │ ├── conftest.py │ ├── test_helper.py │ └── test_stack_requirements.py ├── image └── architecture_diagram.png ├── ip_retention_handler ├── .coveragerc ├── __init__.py ├── poetry.lock ├── pyproject.toml ├── remove_expired_ip.py ├── set_ip_retention.py └── test │ ├── __init__.py │ ├── conftest.py │ ├── test_remove_expired_ip.py │ └── test_set_ip_retention.py ├── lib ├── boto3_util.py ├── cfn_response.py ├── cw_metrics_util.py ├── dynamodb_util.py ├── logging_util.py ├── s3_util.py ├── sns_util.py ├── solution_metrics.py └── waflibv2.py ├── log_parser ├── .coveragerc ├── __init__.py ├── add_athena_partitions.py ├── athena_log_parser.py ├── build_athena_queries.py ├── lambda_log_parser.py ├── log_parser.py ├── partition_s3_logs.py ├── poetry.lock ├── pyproject.toml └── test │ ├── __init__.py │ ├── conftest.py │ ├── test_add_athena_partitions.py │ ├── test_build_athena_queries.py │ ├── test_data │ ├── E3HXCM7PFRG6HT.2023-04-24-21.d740d76bCloudFront.gz │ ├── XXXXXXXXXXXX_elasticloadbalancing_us-east-1_app.ApplicationLoadBalancer.fa87e1db7badc175_20230424T2110Z_X.X.X.X_4c8scnzy.log.gz │ ├── alb_logs_query.txt │ ├── athena_partitions_query.txt │ ├── cf-access-log-sample.gz │ ├── cloudfront_logs_query.txt │ ├── test_athena_query_result.csv │ ├── test_waf_log.gz │ ├── waf-stack-app_log_out.json │ ├── waf_logs_query_1.txt │ ├── waf_logs_query_2.txt │ ├── waf_logs_query_3.txt │ ├── waf_logs_query_4.txt │ ├── waf_logs_query_5.txt │ ├── waf_logs_query_6.txt │ ├── waf_stack-app_log_conf.json │ ├── waf_stack-waf_log_conf.json │ └── waf_stack-waf_log_out.json │ ├── test_log_parser.py │ ├── test_partition_s3_logs.py │ └── test_solution_metrics.py ├── reputation_lists_parser ├── .coveragerc ├── __init__.py ├── poetry.lock ├── pyproject.toml ├── reputation_lists.py └── test │ ├── __init__.py │ ├── conftest.py │ ├── test_data │ ├── __init__.py │ └── test_data.txt │ └── test_reputation_lists_parser.py └── timer ├── .coveragerc ├── __init__.py ├── poetry.lock ├── pyproject.toml ├── test ├── __init__.py ├── conftest.py └── test_timer.py └── timer.py /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior. 15 | 16 | **Expected behavior** 17 | A clear and concise description of what you expected to happen. 18 | 19 | **Please complete the following information about the solution:** 20 | - [ ] Version: [e.g. v3.1] 21 | 22 | To get the version of the solution, you can look at the description of the created CloudFormation stack. For example, "Security Automations for AWS WAF v3.1: This AWS CloudFormation template helps you provision the Security Automations for AWS WAF stack without worrying about creating and configuring the underlying AWS infrastructure". If the description does not contain the version information, you can look at the mappings section of the template: 23 | 24 | ```yaml 25 | Mappings: 26 | SourceCode: 27 | General: 28 | TemplateBucket: 'solutions-reference' 29 | SourceBucket: 'solutions' 30 | KeyPrefix: 'waf-security-automation/v3.1' 31 | ``` 32 | 33 | - [ ] Region: [e.g. us-east-1] 34 | - [ ] Was the solution modified from the version published on this repository? 35 | - [ ] If the answer to the previous question was yes, are the changes available on GitHub? 36 | - [ ] Have you checked your [service quotas](https://docs.aws.amazon.com/general/latest/gr/aws_service_limits.html) for the services this solution uses? 37 | - [ ] Were there any errors in the CloudWatch Logs? 38 | 39 | **Screenshots** 40 | If applicable, add screenshots to help explain your problem (please **DO NOT include sensitive information**). 41 | 42 | **Additional context** 43 | Add any other context about the problem here. 44 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this solution 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the feature you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/dist 2 | **/.zip 3 | **/.DS_Store 4 | **/node_modules 5 | **/package 6 | **/package-lock.json 7 | **/.pyc 8 | /deployment/open-source/ 9 | source/tests/__pycache__/ 10 | source/log_parser/__pycache__/ 11 | deployment/global-s3-assets/ 12 | deployment/regional-s3-assets/ 13 | source/**/idna** 14 | source/**/certifi** 15 | source/**/urllib** 16 | source/**/requests** 17 | source/**/backoff** 18 | source/**/charset** 19 | source/**/bin 20 | source/**/__pycache__ 21 | source/**/.venv** 22 | source/**/test/__pycache__ 23 | source/**/test/.pytest** 24 | 25 | # IDE specific config files 26 | .idea/ 27 | 28 | 29 | 30 | # Unit test / coverage reports 31 | **/coverage 32 | *coverage 33 | source/test/coverage-reports/ 34 | **/.venv-test 35 | 36 | # linting, scanning configurations, sonarqube 37 | .scannerwork/ 38 | 39 | # Third-party dependencies 40 | backoff* 41 | bin 42 | boto3* 43 | botocore* 44 | certifi* 45 | charset* 46 | dateutil* 47 | idna* 48 | jmespath* 49 | python_* 50 | requests* 51 | s3transfer* 52 | six* 53 | urllib* 54 | 55 | # Ignore lib folder within each lambada folder. Only include lib folder at upper level 56 | /source/**/lib 57 | !/source/lib 58 | 59 | # Build script output from 'poetry export' 60 | requirements.txt -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [4.0.6] - 2024-12-17 8 | 9 | ### Changed 10 | 11 | - Update the lambda to python 3.12 12 | 13 | ### Fixed 14 | 15 | - Added a check for payload for logging before sanitizing and logging [Github issue 274](https://github.com/aws-solutions/aws-waf-security-automations/issues/274) 16 | 17 | ## [4.0.5] - 2024-10-24 18 | 19 | ### Changed 20 | 21 | - Add poetry.lock to pin dependency versions for Python code 22 | - Adapt build scripts to use Poetry for dependency management 23 | - Replace native Python logger with aws_lambda_powertools logger 24 | 25 | ## [4.0.4] - 2024-09-23 26 | 27 | ### Fixed 28 | 29 | - Patched dependency version of `requests` to `2.32.3` to mitigate [CVE-2024-3651](https://nvd.nist.gov/vuln/detail/CVE-2024-3651) 30 | - Pinned all dependencies to specific versions for reproducable builds and enable security scanning 31 | - Allow to install latest version of `urllib3` as transitive dependency 32 | 33 | ## [4.0.3] - 2023-10-25 34 | 35 | ### Fixed 36 | 37 | - Patched urllib3 vulnerability as it is possible for a user to specify a Cookie header and unknowingly leak information via HTTP redirects to a different origin if that user doesn't disable redirects explicitly. For more details: [CVE-2023-43804](https://nvd.nist.gov/vuln/detail/CVE-2023-43804) 38 | 39 | ## [4.0.2] - 2023-09-11 40 | 41 | ### Fixed 42 | 43 | - Update trademarked name. From aws-waf-security-automations.zip to security-automations-for-aws-waf.zip 44 | - Refactor to reduce code complexity 45 | - Patched requests package vulnerability leaking Proxy-Authorization headers to destination servers when redirected to an HTTPS endpoint. For more details: [CVE-2023-32681](https://nvd.nist.gov/vuln/detail/CVE-2023-32681) [Github issue 248](https://github.com/aws-solutions/aws-waf-security-automations/issues/248) 46 | 47 | ## [4.0.1] - 2023-05-19 48 | 49 | ### Fixed 50 | 51 | - Updated gitignore files to resolve the issue for missing files [Github issue 244](https://github.com/aws-solutions/aws-waf-security-automations/issues/244) [Github issue 243](https://github.com/aws-solutions/aws-waf-security-automations/issues/243) [Github issue 245](https://github.com/aws-solutions/aws-waf-security-automations/issues) 52 | 53 | ## [4.0.0] - 2023-05-11 54 | 55 | ### Added 56 | 57 | - Added support for 10 new AWS Managed Rules rule groups (AMR) 58 | - Added support for country and URI configurations in HTTP Flood Athena log parser 59 | - Added support for user-defined S3 prefix for application access log bucket 60 | - Added support for CloudWatch log retention period configuration 61 | - Added support for multiple solution deployments in the same account and region 62 | - Added support for exporting CloudFormation stack output values 63 | - Replaced the hard coded amazonaws.com with {AWS::URLSuffix} in BadBotHoneypot API endpoint 64 | 65 | ### Fixed 66 | 67 | - Avoid account-wide API Gateway logging setting change by deleting the solution stack [GitHub issue 213](https://github.com/aws-solutions/aws-waf-security-automations/issues/213) 68 | - Avoid creating a new logging bucket for an existing app access log bucket that already has logging enabled 69 | 70 | ## [3.2.5] - 2023-04-18 71 | 72 | ### Patched 73 | 74 | - Patch s3 logging bucket settings 75 | - Updated the timeout for requests 76 | 77 | ## [3.2.4] - 2023-02-06 78 | 79 | ### Changed 80 | 81 | - Upgraded pytest to mitigate CVE-2022-42969 82 | - Upgraded requests and subsequently certifi to mitigate CVE-2022-23491 83 | 84 | ## [3.2.3] - 2022-12-13 85 | 86 | ### Changed 87 | 88 | - Add region as prefix to application attribute group name to avoid conflict with name starting with AWS. 89 | 90 | ## [3.2.2] - 2022-12-05 91 | 92 | ### Added 93 | 94 | - Added AppRegistry integration 95 | 96 | ## [3.2.1] - 2022-08-30 97 | 98 | ### Added 99 | 100 | - Added support for configuring oversize handling for requests components 101 | - Added support for configuring sensitivity level for SQL injection rule 102 | 103 | ## [3.2.0] - 2021-09-22 104 | 105 | ### Added 106 | 107 | - Added IP retention support on Allowed and Denied IP Sets 108 | 109 | ### Changed 110 | 111 | - Bug fixes 112 | 113 | ## [3.1.0] - 2020-10-22 114 | 115 | ### Changed 116 | 117 | - Replaced s3 path-style with virtual-hosted style 118 | - Added partition variable to all ARNs 119 | - Updated bug report 120 | 121 | ## [3.0.0] - 2020-07-08 122 | 123 | ### Added 124 | 125 | - Added an option to deploy AWS Managed Rules for WebACL on installation 126 | 127 | ### Changed 128 | 129 | - Upgraded from WAF classic to WAFV2 API 130 | - Eliminated dependency on NodeJS and use Python as the standardized programming language 131 | 132 | ## [2.3.3] - 2020-06-15 133 | 134 | ### Added 135 | 136 | - Implemented Athena optimization: added partitioning for CloudFront, ALB and WAF logs and Athena queries 137 | 138 | ### Changed 139 | 140 | - Fixed potential DoS vector within Bad Bots X-Forward-For header 141 | 142 | ## [2.3.2] - 2020-02-05 143 | 144 | ### Added 145 | 146 | ### Changed 147 | 148 | - Fixed README file to accurately reflect script params 149 | - Upgraded from Python 3.7 to 3.8 150 | - Changed RequestThreshold min limit from 2000 to 100 151 | 152 | ## [2.3.1] - 2019-10-30 153 | 154 | ### Added 155 | 156 | ### Changed 157 | 158 | - Fixed error handling of intermittent issue: (WAFStaleDataException) when calling the UpdateWebACL 159 | - Upgrade from Node 8 to Node 10 for Lambda function 160 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check [existing open](https://github.com/aws-solutions/aws-waf-security-automations/issues), or [recently closed](https://github.com/aws-solutions/aws-waf-security-automations/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/aws-solutions/aws-waf-security-automations/labels/help%20wanted) issues is a great place to start. 45 | 46 | ## Code of Conduct 47 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 48 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 49 | opensource-codeofconduct@amazon.com with any additional questions or comments. 50 | 51 | 52 | ## Security issue notifications 53 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 54 | 55 | 56 | ## Licensing 57 | 58 | See the [LICENSE](https://github.com/aws-solutions/aws-waf-security-automations/blob/master/LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 59 | 60 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 61 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Security Automations for AWS WAF 2 | 3 | Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except 5 | in compliance with the License. A copy of the License is located at http://www.apache.org/licenses/ 6 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, 7 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the 8 | specific language governing permissions and limitations under the License. 9 | 10 | ********************** 11 | THIRD PARTY COMPONENTS 12 | ********************** 13 | 14 | This software includes third party software subject to the following copyrights: 15 | 16 | aws-lambda-powertools under the MIT license. 17 | backoff under the MIT license. 18 | boto3 under the Apache-2.0 license. 19 | botocore under the Apache-2.0 license. 20 | certifi under the MPL-2.0 license. 21 | cffi under the MIT license. 22 | charset-normalizer under the MIT license. 23 | colorama under the 0BSD license. 24 | coverage under the Apache-2.0 license. 25 | cryptography under the Apache-2.0 license. 26 | idna under the 0BSD license. 27 | iniconfig under the MIT license. 28 | jinja2 under the 0BSD license. 29 | jmespath under the MIT license. 30 | markupsafe under the 0BSD license. 31 | moto under the Apache-2.0 license. 32 | packaging under the Apache-2.0 license. 33 | pluggy under the MIT license. 34 | pycparser under the 0BSD license. 35 | pytest under the MIT license. 36 | pytest-cov under the MIT license. 37 | pytest-env under the MIT license. 38 | pytest-mock under the MIT license. 39 | pytest-runner under the MIT license. 40 | python-dateutil under the Apache-2.0 license. 41 | pyyaml under the MIT license. 42 | requests under the Apache-2.0 license. 43 | responses under the Apache-2.0 license. 44 | s3transfer under the Apache-2.0 license. 45 | six under the MIT license. 46 | typing-extensions under the PSF-2.0 license. 47 | urllib3 under the MIT license. 48 | werkzeug under the 0BSD license. 49 | xmltodict under the MIT license. 50 | freezegun under the Apache-2.0 license. 51 | pyparsing under the MIT license. 52 | 53 | ******************** 54 | OPEN SOURCE LICENSES 55 | ******************** 56 | 57 | 0BSD - https://spdx.org/licenses/0BSD.html 58 | Apache-2.0 - https://spdx.org/licenses/Apache-2.0.html 59 | MPL-2.0 - https://spdx.org/licenses/MPL-2.0.html 60 | PSF-2.0 - https://spdx.org/licenses/PSF-2.0.html -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **[🚀 Solution Landing Page](https://aws.amazon.com/solutions/implementations/security-automations-for-aws-waf)** | **[🚧 Feature request](https://github.com/aws-solutions/aws-waf-security-automations/issues/new?assignees=&labels=feature-request%2C+enhancement&template=feature_request.md&title=)** | **[🐛 Bug Report](https://github.com/aws-solutions/aws-waf-security-automations/issues/new?assignees=&labels=bug%2C+triage&template=bug_report.md&title=)** 2 | 3 | **Note:** If you want to use the solution without building from source, navigate to Solution Landing Page. 4 | 5 | ## Table of contents 6 | 7 | - [Solution Overview](#solution-overview) 8 | - [Architecture Diagram](#architecture-diagram) 9 | - [Customizing the Solution](#customizing-the-solution) 10 | - [Prerequisites for Customization](#prerequisites-for-customization) 11 | - [Build](#build) 12 | - [Upload Deployment Assets](#upload-deployment-assets) 13 | - [Deploy](#deploy) 14 | - [File Structure](#file-structure) 15 | - [License](#license) 16 | 17 | --- 18 | 19 | # Solution overview 20 | 21 | The Security Automations for AWS WAF solution automatically deploys a set of AWS WAF (web application firewall) rules that filter common web-based attacks. Users can select from preconfigured protective features that define the rules included in an AWS WAF web access control list (web ACL). Once deployed, AWS WAF protects your Amazon CloudFront distributions or Application Load Balancers by inspecting web requests. 22 | 23 | You can use AWS WAF to create custom, application-specific rules that block attack patterns to ensure application availability, secure resources, and prevent excessive resource consumption. 24 | 25 | You can install this solution in your AWS accounts by launching the provided AWS CloudFormation template. 26 | 27 | For a detailed solution implementation guide, refer to Solution Landing Page [Security Automations for AWS WAF](https://aws.amazon.com/solutions/implementations/security-automations-for-aws-waf). 28 | 29 | --- 30 | 31 | # Architecture diagram 32 | 33 | ![Diagram](source/image/architecture_diagram.png) 34 | 35 | *Security Automations for AWS WAF architecture* 36 | 37 | The components of this solution can be grouped into the following areas of protection. 38 | 39 | **Note:** The group labels don’t reflect the priority level of the WAF rules. 40 | 41 | * **AWS Managed Rules (A)** – This component contains AWS Managed Rules IP reputation rule groups, baseline rule groups, and use-case specific rule groups. These rule groups protect against exploitation of common application vulnerabilities or other unwanted traffic, including those described in OWASP publications, without having to write your own rules. 42 | * **Manual IP lists (B and C)** – These components create two AWS WAF rules. With these rules, you can manually insert IP addresses that you want to allow or deny. You can also configure IP retention and remove expired IP addresses from these IP lists. 43 | * **SQL Injection (D) and XSS (E)** – These components configure two AWS WAF rules that are designed to protect against common SQL injection or cross-site scripting (XSS) patterns in the URI, query string, or body of a request. 44 | * **HTTP Flood (F)** – This component protects against attacks that consist of a large number of requests from a particular IP address, such as a web-layer DDoS attack or a brute-force login attempt. 45 | * **Scanner and Probe (G)** – This component parses application access logs searching for suspicious behavior, such as an abnormal amount of errors generated by an origin. Then it blocks those suspicious source IP addresses for a customer-defined period of time. 46 | * **IP Reputation Lists (H)** – This component is the IP Lists Parser Lambda function that checks third-party IP reputation lists hourly for new ranges to block. These lists include the Spamhaus Don’t Route Or Peer (DROP) and Extended DROP (EDROP) lists, the Proofpoint Emerging Threats IP list, and the Tor exit node list. 47 | * **Bad Bot (I)** – This component automatically sets up a honeypot, which is a security mechanism intended to lure and deflect an attempted attack. 48 | 49 | --- 50 | 51 | 52 | # Customizing the solution 53 | 54 | 55 | ## Prerequisites for customization 56 | 57 | - [AWS Command Line Interface](https://aws.amazon.com/cli/) 58 | - Python 3.12 59 | - Poetry 60 | 61 | ## Build 62 | 63 | Building from GitHub source allows you to modify the solution, such as adding custom actions or upgrading to a new release. The process consists of downloading the source from GitHub, creating Amazon S3 buckets to store artifacts for deployment, building the solution, and uploading the artifacts to S3 buckets in your AWS account. 64 | 65 | #### 1. Clone the repository 66 | 67 | Clone or download the repository to a local directory on your Linux client. 68 | 69 | **Note:** If you intend to modify the source code, can create your own fork of the GitHub repo and work from that. This way, you can check in your changes to your private copy of the solution. 70 | 71 | **Git Clone example:** 72 | 73 | ``` 74 | git clone https://github.com/aws-solutions/aws-waf-security-automations.git 75 | ``` 76 | 77 | **Download Zip example:** 78 | 79 | ``` 80 | wget https://github.com/aws-solutions/aws-waf-security-automations/archive/master.zip 81 | ``` 82 | 83 | #### 2. Unit test 84 | 85 | Next, run unit tests to ensure that your customized code passes the tests: 86 | 87 | ``` 88 | cd /deployment 89 | chmod +x ./run-unit-tests.sh 90 | ./run-unit-tests.sh 91 | ``` 92 | 93 | #### 3. Create S3 buckets for storing deployment assets 94 | 95 | AWS Solutions use two buckets: 96 | 97 | - One global bucket that you access with the http endpoint. AWS CloudFormation templates are stored here. For example, `mybucket`. 98 | - One regional bucket for each AWS Region where you plan to deploy the solution. Use the name of the global bucket as the prefix of the bucket name, and suffix it with the region name. Regional assets such as Lambda code are stored here. For example, `mybucket-us-east-1`. 99 | 100 | The assets in buckets must be accessible by your account. 101 | 102 | #### 4. Declare enviroment variables 103 | 104 | ``` 105 | export TEMPLATE_OUTPUT_BUCKET= # Name of the global bucket where CloudFormation templates are stored 106 | export DIST_OUTPUT_BUCKET= # Name for the regional bucket where regional assets are stored 107 | export SOLUTION_NAME= # name of the solution. 108 | export VERSION= # version number for the customized code 109 | export AWS_REGION= # region where the solution is deployed 110 | ``` 111 | 112 | #### 5. Build the solution 113 | 114 | ``` 115 | cd /deployment 116 | chmod +x ./build-s3-dist.sh && ./build-s3-dist.sh $TEMPLATE_OUTPUT_BUCKET $DIST_OUTPUT_BUCKET $SOLUTION_NAME $VERSION 117 | ``` 118 | 119 | 120 | ## Upload deployment assets 121 | 122 | ``` 123 | aws s3 cp ./deployment/global-s3-assets s3://$TEMPLATE_OUTPUT_BUCKET/$SOLUTION_NAME/$VERSION --recursive --acl bucket-owner-full-control 124 | aws s3 cp ./deployment/regional-s3-assets s3://$DIST_OUTPUT_BUCKET-$AWS_REGION/$SOLUTION_NAME/$VERSION --recursive --acl bucket-owner-full-control 125 | ``` 126 | 127 | **Note:** You must use a proper ACL and profile for the copy operation as applicable. Using randomized bucket names is recommended. 128 | 129 | 130 | ## Deploy 131 | 132 | - From your designated S3 bucket where you uploaded the deployment assets, copy the link location for the `aws-waf-security-automations.template` file. 133 | - Using AWS CloudFormation, launch the Security Automations for AWS WAF solution stack using the copied Amazon S3 link for the `aws-waf-security-automations.template` file. 134 | 135 | **Note:** When deploying the template for your CloudFront endpoint, you can launch it only from the `us-east-1` Region. 136 | 137 | --- 138 | 139 | # File structure 140 | 141 | This project consists of microservices that facilitate the functional areas of the solution. These microservices are deployed to a serverless environment in AWS Lambda. 142 | 143 | ``` 144 | |-deployment/ [folder containing templates and build scripts] 145 | |-source/ 146 | |-access_handler/ [microservice for processing bad bots honeypot endpoint access. This AWS Lambda function intercepts the suspicious request and adds the source IP address to the AWS WAF block list] 147 | |-custom_resource/ [custom helper for CloudFormation deployment template] 148 | |-helper/ [custom helper for CloudFormation deployment dependency check and auxiliary functions] 149 | |-image/ [folder containing images of the solution such as architecture diagram] 150 | |-lib/ [library files including waf api calls and other common functions used in the solution] 151 | |-ip_retention_handler/ [lambda code for setting ip retention and removing expired ips] 152 | |-log_parser/ [microservice for processing access logs searching for suspicious behavior and add the corresponding source IP addresses to an AWS WAF block list] 153 | |-reputation_lists_parser/ [microservice for processing third-party IP reputation lists and add malicious IP addresses to an AWS WAF block list] 154 | |-timer/ [creates a sleep function for cloudformation to pace the creation of ip_sets] 155 | ``` 156 | 157 | --- 158 | 159 | # Collection of operational metrics 160 | 161 | This solution collects anonymized operational metrics to help AWS improve the quality and features of the solution. For more information, including how to disable this capability, please see the [implementation guide](https://docs.aws.amazon.com/solutions/latest/security-automations-for-aws-waf/reference.html). 162 | 163 | 164 | --- 165 | 166 | # License 167 | 168 | See license [here](https://github.com/aws-solutions/aws-waf-security-automations/blob/master/LICENSE.txt). 169 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | ## Reporting Security Issues 2 | 3 | We take all security reports seriously. When we receive such reports, 4 | we will investigate and subsequently address any potential vulnerabilities as 5 | quickly as possible. If you discover a potential security issue in this project, 6 | please notify AWS/Amazon Security via our [vulnerability reporting page] 7 | (http://aws.amazon.com/security/vulnerability-reporting/) or directly via email 8 | to [AWS Security](mailto:aws-security@amazon.com). 9 | Please do *not* create a public GitHub issue in this project. -------------------------------------------------------------------------------- /deployment/run-unit-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # This assumes all of the OS-level configuration has been completed and git repo has already been cloned 4 | # 5 | # This script should be run from the repo's deployment directory 6 | # cd deployment 7 | # ./run-unit-tests.sh 8 | # 9 | 10 | [ "$DEBUG" == 'true' ] && set -x 11 | set -e 12 | 13 | template_dir="$PWD" 14 | source_dir="$(cd $template_dir/../source; pwd -P)" 15 | 16 | echo "Current directory: $template_dir" 17 | echo "Source directory: $source_dir" 18 | 19 | run_python_lambda_test() { 20 | lambda_name=$1 21 | lambda_description=$2 22 | echo "------------------------------------------------------------------------------" 23 | echo "[Test] Python Unit Test: $lambda_description" 24 | echo "------------------------------------------------------------------------------" 25 | 26 | cd $source_dir/$lambda_name 27 | echo "run_python_lambda_test: Current directory: $source_dir/$lambda_name" 28 | 29 | echo "Installing python packages" 30 | 31 | # Check if poetry is available in the shell 32 | if command -v poetry >/dev/null 2>&1; then 33 | POETRY_COMMAND="poetry" 34 | elif [ -n "$POETRY_HOME" ] && [ -x "$POETRY_HOME/bin/poetry" ]; then 35 | POETRY_COMMAND="$POETRY_HOME/bin/poetry" 36 | else 37 | echo "Poetry is not available. Aborting script." >&2 38 | exit 1 39 | fi 40 | 41 | # This creates a virtual environment based on the project name in pyproject.toml. 42 | "$POETRY_COMMAND" install 43 | 44 | # Activate the virtual environment. 45 | source $("$POETRY_COMMAND" env info --path)/bin/activate 46 | 47 | # Set coverage report path 48 | mkdir -p $source_dir/test/coverage-reports 49 | coverage_report_path=$source_dir/test/coverage-reports/$lambda_name.coverage.xml 50 | echo "coverage report path set to $coverage_report_path" 51 | 52 | # Run unit tests with coverage 53 | python3 -m pytest --cov --cov-report=term-missing --cov-report "xml:$coverage_report_path" 54 | 55 | if [ "$?" = "1" ]; then 56 | echo "(deployment/run-unit-tests.sh) ERROR: there is likely output above." 1>&2 57 | exit 1 58 | fi 59 | 60 | # The pytest --cov with its parameters and .coveragerc generates a xml cov-report with `coverage/sources` list 61 | # with absolute path for the source directories. To avoid dependencies of tools (such as SonarQube) on different 62 | # absolute paths for source directories, this substitution is used to convert each absolute source directory 63 | # path to the corresponding project relative path. The $source_dir holds the absolute path for source directory. 64 | sed -i -e "s,$source_dir,source,g" $coverage_report_path 65 | 66 | deactivate 67 | 68 | if [ "${CLEAN:-true}" = "true" ]; then 69 | # Note: leaving $source_dir/test/coverage-reports to allow further processing of coverage reports 70 | rm -fr coverage 71 | rm .coverage 72 | fi 73 | } 74 | 75 | # Run Python unit tests 76 | run_python_lambda_test access_handler "BadBot Access Handler Lambda" 77 | run_python_lambda_test custom_resource "Custom Resource Lambda" 78 | run_python_lambda_test helper "Helper Lambda" 79 | run_python_lambda_test ip_retention_handler "Set IP Retention Lambda" 80 | run_python_lambda_test log_parser "Log Parser Lambda" 81 | run_python_lambda_test reputation_lists_parser "Reputation List Parser Lambda" 82 | run_python_lambda_test timer "Timer Lambda" 83 | 84 | 85 | # Return to the directory where we started 86 | cd $template_dir -------------------------------------------------------------------------------- /source/access_handler/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | test/* 4 | */__init__.py 5 | **/__init__.py 6 | backoff/* 7 | bin/* 8 | boto3/* 9 | botocore/* 10 | certifi/* 11 | charset*/* 12 | crhelper* 13 | chardet* 14 | dateutil/* 15 | idna/* 16 | jmespath/* 17 | lib/* 18 | package* 19 | python_* 20 | requests/* 21 | s3transfer/* 22 | six* 23 | tenacity* 24 | tests 25 | urllib3/* 26 | yaml 27 | PyYAML-* 28 | source = 29 | . -------------------------------------------------------------------------------- /source/access_handler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/access_handler/__init__.py -------------------------------------------------------------------------------- /source/access_handler/access_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | from ipaddress import IPv4Network 6 | from ipaddress import IPv6Network 7 | from ipaddress import ip_address 8 | from os import environ 9 | 10 | from aws_lambda_powertools import Logger 11 | 12 | from lib.cw_metrics_util import WAFCloudWatchMetrics 13 | from lib.solution_metrics import send_metrics 14 | from lib.waflibv2 import WAFLIBv2 15 | 16 | logger = Logger( 17 | level=os.getenv('LOG_LEVEL') 18 | ) 19 | 20 | waflib = WAFLIBv2() 21 | CW_METRIC_PERIOD_SECONDS = 12 * 3600 # Twelve hours in seconds 22 | 23 | def initialize_usage_data(): 24 | usage_data = { 25 | "data_type": "bad_bot", 26 | "bad_bot_ip_set_size": 0, 27 | "allowed_requests": 0, 28 | "blocked_requests_all": 0, 29 | "blocked_requests_bad_bot": 0, 30 | "waf_type": os.getenv('LOG_TYPE'), 31 | "provisioner": os.getenv('provisioner') if "provisioner" in environ else "cfn" 32 | 33 | } 34 | return usage_data 35 | 36 | 37 | def get_bad_bot_usage_data(scope, cw, ipset_name_v4, ipset_arn_v4, ipset_name_v6, ipset_arn_v6, usage_data): 38 | logger.info("[get_bad_bot_usage_data] Get bad bot data") 39 | 40 | if 'IP_SET_ID_BAD_BOTV4' in environ or 'IP_SET_ID_BAD_BOTV6' in environ: 41 | # Get the count of ipv4 and ipv6 in bad bot ip sets 42 | ipv4_count = waflib.get_ip_address_count(logger, scope, ipset_name_v4, ipset_arn_v4) 43 | ipv6_count = waflib.get_ip_address_count(logger, scope, ipset_name_v6, ipset_arn_v6) 44 | usage_data['bad_bot_ip_set_size'] = str(ipv4_count + ipv6_count) 45 | 46 | # Get the count of blocked requests for the bad bot rule from cloudwatch metrics 47 | usage_data = cw.add_waf_cw_metric_to_usage_data( 48 | 'BlockedRequests', 49 | CW_METRIC_PERIOD_SECONDS, 50 | os.getenv('METRIC_NAME_PREFIX') + 'BadBotRule', 51 | usage_data, 52 | 'blocked_requests_bad_bot', 53 | 0 54 | ) 55 | return usage_data 56 | 57 | 58 | def send_anonymized_usage_data(scope, ipset_name_v4, ipset_arn_v4, ipset_name_v6, ipset_arn_v6): 59 | try: 60 | if 'SEND_ANONYMIZED_USAGE_DATA' not in environ or os.getenv('SEND_ANONYMIZED_USAGE_DATA').lower() != 'yes': 61 | return 62 | 63 | logger.info("[send_anonymized_usage_data] Start") 64 | 65 | cw = WAFCloudWatchMetrics(logger) 66 | usage_data = initialize_usage_data() 67 | 68 | # Get the count of allowed requests for all the waf rules from cloudwatch metrics 69 | usage_data = cw.add_waf_cw_metric_to_usage_data( 70 | 'AllowedRequests', 71 | CW_METRIC_PERIOD_SECONDS, 72 | 'ALL', 73 | usage_data, 74 | 'allowed_requests', 75 | 0 76 | ) 77 | 78 | # Get the count of blocked requests for all the waf rules from cloudwatch metrics 79 | usage_data = cw.add_waf_cw_metric_to_usage_data( 80 | 'BlockedRequests', 81 | CW_METRIC_PERIOD_SECONDS, 82 | 'ALL', 83 | usage_data, 84 | 'blocked_requests_all', 85 | 0 86 | ) 87 | 88 | # Get bad bot specific usage data 89 | usage_data = get_bad_bot_usage_data(scope, cw, ipset_name_v4, ipset_arn_v4, ipset_name_v6, ipset_arn_v6, 90 | usage_data) 91 | 92 | # Send usage data 93 | logger.info('[send_anonymized_usage_data] Send usage data: \n{}'.format(usage_data)) 94 | response = send_metrics(data=usage_data) 95 | response_code = response.status_code 96 | logger.info('[send_anonymized_usage_data] Response Code: {}'.format(response_code)) 97 | logger.info("[send_anonymized_usage_data] End") 98 | 99 | except Exception as error: 100 | logger.info("[send_anonymized_usage_data] Failed to Send Data") 101 | logger.error(str(error)) 102 | 103 | 104 | def add_ip_to_ip_set(scope, ip_type, source_ip, ipset_name, ipset_arn): 105 | new_address = [] 106 | output = None 107 | 108 | if ip_type == "IPV4": 109 | new_address.append(IPv4Network(source_ip).with_prefixlen) 110 | elif ip_type == "IPV6": 111 | new_address.append(IPv6Network(source_ip).with_prefixlen) 112 | 113 | ipset = waflib.get_ip_set(logger, scope, ipset_name, ipset_arn) 114 | # merge old addresses with this one 115 | logger.info(ipset) 116 | current_list = ipset["IPSet"]["Addresses"] 117 | logger.info(current_list) 118 | new_list = list(set(current_list) | set(new_address)) 119 | logger.info(new_list) 120 | output = waflib.update_ip_set(logger, scope, ipset_name, ipset_arn, new_list) 121 | 122 | return output 123 | 124 | 125 | # ====================================================================================================================== 126 | # Lambda Entry Point 127 | # ====================================================================================================================== 128 | @logger.inject_lambda_context 129 | def lambda_handler(event, _): 130 | logger.info('[lambda_handler] Start') 131 | 132 | # ---------------------------------------------------------- 133 | # Read inputs parameters 134 | # ---------------------------------------------------------- 135 | try: 136 | scope = os.getenv('SCOPE') 137 | ipset_name_v4 = os.getenv('IP_SET_NAME_BAD_BOTV4') 138 | ipset_name_v6 = os.getenv('IP_SET_NAME_BAD_BOTV6') 139 | ipset_arn_v4 = os.getenv('IP_SET_ID_BAD_BOTV4') 140 | ipset_arn_v6 = os.getenv('IP_SET_ID_BAD_BOTV6') 141 | 142 | # Fixed as old line had security exposure based on user supplied IP address 143 | logger.info("Event->%s<-", str(event)) 144 | if event['requestContext']['identity']['userAgent'] == 'Amazon CloudFront': 145 | source_ip = str(event['headers']['X-Forwarded-For'].split(',')[0].strip()) 146 | else: 147 | source_ip = str(event['requestContext']['identity']['sourceIp']) 148 | 149 | logger.info("scope = %s", scope) 150 | logger.info("ipset_name_v4 = %s", ipset_name_v4) 151 | logger.info("ipset_name_v6 = %s", ipset_name_v6) 152 | logger.info("IPARNV4 = %s", ipset_arn_v4) 153 | logger.info("IPARNV6 = %s", ipset_arn_v6) 154 | logger.info("source_ip = %s", source_ip) 155 | 156 | ip_type = "IPV%s" % ip_address(source_ip).version 157 | output = None 158 | if ip_type == "IPV4": 159 | output = add_ip_to_ip_set(scope, ip_type, source_ip, ipset_name_v4, ipset_arn_v4) 160 | elif ip_type == "IPV6": 161 | output = add_ip_to_ip_set(scope, ip_type, source_ip, ipset_name_v6, ipset_arn_v6) 162 | except Exception as e: 163 | logger.error(e) 164 | raise 165 | finally: 166 | logger.info("Output->%s<-", output) 167 | message = "message: [%s] Thanks for the visit." % source_ip 168 | response = { 169 | 'statusCode': 200, 170 | 'headers': {'Content-Type': 'application/json'}, 171 | 'body': message 172 | } 173 | 174 | if output is not None: 175 | send_anonymized_usage_data(scope, ipset_name_v4, ipset_arn_v4, ipset_name_v6, ipset_arn_v6) 176 | logger.info('[lambda_handler] End') 177 | 178 | return response 179 | -------------------------------------------------------------------------------- /source/access_handler/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "access-handler" 3 | package-mode = false 4 | 5 | [tool.poetry.dependencies] 6 | python = "~3.12" 7 | requests = "^2.32.3" 8 | backoff = "^2.2.1" 9 | aws-lambda-powertools = "~3.2.0" 10 | 11 | 12 | [tool.poetry.group.dev.dependencies] 13 | moto = "^4.1.4" 14 | pytest = "^7.2.2" 15 | pytest-mock = "^3.12.0" 16 | pytest-runner = "^6.0.0" 17 | pytest-cov = "^4.0.0" 18 | pytest-env = "^0.8.1" 19 | boto3 = "^1.35.30" 20 | 21 | [build-system] 22 | requires = ["poetry-core"] 23 | build-backend = "poetry.core.masonry.api" 24 | -------------------------------------------------------------------------------- /source/access_handler/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/access_handler/test/__init__.py -------------------------------------------------------------------------------- /source/access_handler/test/conftest.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). # 5 | # You may not use this file except in compliance # 6 | # with the License. A copy of the License is located at # 7 | # # 8 | # http://www.apache.org/licenses/LICENSE-2.0 # 9 | # # 10 | # or in the "license" file accompanying this file. This file is # 11 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # 12 | # KIND, express or implied. See the License for the specific language # 13 | # governing permissions and limitations under the License. # 14 | ############################################################################## 15 | 16 | import pytest 17 | import boto3 18 | from os import environ 19 | from moto import ( 20 | mock_wafv2, 21 | mock_cloudwatch 22 | ) 23 | 24 | @pytest.fixture(scope='module', autouse=True) 25 | def aws_credentials(): 26 | """Mocked AWS Credentials for moto""" 27 | environ['AWS_ACCESS_KEY_ID'] = 'testing' 28 | environ['AWS_SECRET_ACCESS_KEY'] = 'testing' 29 | environ['AWS_SECURITY_TOKEN'] = 'testing' 30 | environ['AWS_SESSION_TOKEN'] = 'testing' 31 | environ['AWS_DEFAULT_REGION'] = 'us-east-1' 32 | environ['AWS_REGION'] = 'us-east-1' 33 | 34 | @pytest.fixture(scope='session') 35 | def ipset_env_var_setup(): 36 | environ["SCOPE"] = 'ALB' 37 | environ['IP_SET_NAME_BAD_BOTV4'] = 'IP_SET_NAME_BAD_BOTV4' 38 | environ['IP_SET_NAME_BAD_BOTV6'] = 'IP_SET_NAME_BAD_BOTV6' 39 | environ["IP_SET_ID_BAD_BOTV4"] = 'IP_SET_ID_BAD_BOTV4' 40 | environ['IP_SET_ID_BAD_BOTV6'] = 'IP_SET_ID_BAD_BOTV6' 41 | 42 | @pytest.fixture(scope="session") 43 | def wafv2_client(): 44 | with mock_wafv2(): 45 | wafv2_client = boto3.client('wafv2') 46 | yield wafv2_client 47 | 48 | @pytest.fixture(scope="session") 49 | def cloudwatch_client(): 50 | with mock_cloudwatch(): 51 | cloudwatch_client = boto3.client('cloudwatch') 52 | yield cloudwatch_client 53 | 54 | @pytest.fixture(scope="session") 55 | def expected_exception_access_handler_error(): 56 | return "'NoneType' object is not subscriptable" 57 | 58 | @pytest.fixture(scope="session") 59 | def expected_cw_resp(): 60 | return None 61 | 62 | @pytest.fixture(scope="session") 63 | def badbot_event(): 64 | return { 65 | 'body': None, 66 | 'headers': { 67 | 'Host': '0xxxx0xx0.execute-api.us-east-2.amazonaws.com', 68 | 'Referer': 'https://us-east-2.console.aws.amazon.com/', 69 | }, 70 | 'httpMethod': 'GET', 71 | 'isBase64Encoded': False, 72 | 'multiValueHeaders': { 73 | 'Accept': [ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'], 74 | 'Accept-Encoding': ['gzip, deflate, br'], 75 | 'Accept-Language': ['en-US,en;q=0.5'], 76 | 'CloudFront-Forwarded-Proto': ['https'], 77 | 'CloudFront-Is-Desktop-Viewer': ['true'], 78 | 'CloudFront-Is-Mobile-Viewer': ['false'], 79 | 'CloudFront-Is-SmartTV-Viewer': ['false'], 80 | 'CloudFront-Is-Tablet-Viewer': ['false'], 81 | 'CloudFront-Viewer-ASN': ['16509'], 82 | 'CloudFront-Viewer-Country': ['US'], 83 | 'Host': [ '0xxxx0xx0.execute-api.us-east-2.amazonaws.com'], 84 | 'Referer': [ 'https://us-east-2.console.aws.amazon.com/'], 85 | 'User-Agent': [ 'Mozilla/5.0 (Macintosh; Intel ' 86 | 'Mac OS X 10.15; rv:102.0) ' 87 | 'Gecko/20100101 Firefox/102.0'], 88 | 'Via': [ '2.0 ' 89 | 'fde752a2d4e95c2353cf5fc17ef7bf2a.cloudfront.net ' 90 | '(CloudFront)'], 91 | 'X-Amz-Cf-Id': [ 'eee9ZGRfH0AhZToSkR1ubIekS_uz5ZoaJRvYCg6cMrBnF090iUyIQg=='], 92 | 'X-Amzn-Trace-Id': [ 'Root=1-61196a2b-1c401acb6e744c82255d9844'], 93 | 'X-Forwarded-For': ['99.99.99.99, 99.99.99.99'], 94 | 'X-Forwarded-Port': ['443'], 95 | 'X-Forwarded-Proto': ['https'], 96 | 'sec-fetch-dest': ['document'], 97 | 'sec-fetch-mode': ['navigate'], 98 | 'sec-fetch-site': ['cross-site'], 99 | 'sec-fetch-user': ['?1'], 100 | 'upgrade-insecure-requests': ['1'] 101 | }, 102 | 'multiValueQueryStringParameters': None, 103 | 'path': '/', 104 | 'pathParameters': None, 105 | 'queryStringParameters': None, 106 | 'requestContext': { 107 | 'accountId': 'xxxxxxxxxxxx', 108 | 'apiId': '0xxxx0xx0', 109 | 'domainName': '0xxxx0xx0.execute-api.us-east-2.amazonaws.com', 110 | 'domainPrefix': '0xxxx0xx0', 111 | 'extendedRequestId': 'D_2GyFwDiYcFofg=', 112 | 'httpMethod': 'GET', 113 | 'identity': { 114 | 'accessKey': None, 115 | 'accountId': None, 116 | 'caller': None, 117 | 'cognitoAuthenticationProvider': None, 118 | 'cognitoAuthenticationType': None, 119 | 'cognitoIdentityId': None, 120 | 'cognitoIdentityPoolId': None, 121 | 'principalOrgId': None, 122 | 'sourceIp': '99.99.99.99', 123 | 'user': None, 124 | 'userAgent': 'Mozilla/5.0 ' 125 | '(Macintosh; Intel Mac ' 126 | 'OS X 10.15; rv:102.0) ' 127 | 'Gecko/20100101 ' 128 | 'Firefox/102.0', 129 | 'userArn': None 130 | }, 131 | 'path': '/ProdStage', 132 | 'protocol': 'HTTP/1.1', 133 | 'requestId': '4375792d-c6d0-4f84-8a40-d52f5d18dedd', 134 | 'requestTime': '26/Apr/2023:18:15:07 +0000', 135 | 'requestTimeEpoch': 1682532907129, 136 | 'resourceId': 'yw40vqjfia', 137 | 'resourcePath': '/', 138 | 'stage': 'ProdStage' 139 | }, 140 | 'resource': '/', 141 | 'stageVariables': None 142 | } -------------------------------------------------------------------------------- /source/access_handler/test/test_access_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import logging 5 | from types import SimpleNamespace 6 | 7 | from aws_lambda_powertools.utilities.typing import LambdaContext 8 | 9 | from access_handler.access_handler import * 10 | import os 11 | 12 | 13 | log_level = 'DEBUG' 14 | logging.getLogger().setLevel(log_level) 15 | log = logging.getLogger('test_access_handler') 16 | 17 | context = SimpleNamespace(**{ 18 | 'function_name': 'foo', 19 | 'memory_limit_in_mb': '512', 20 | 'invoked_function_arn': 'bar', 21 | 'aws_request_id': 'baz' 22 | }) 23 | 24 | def test_access_handler_error(ipset_env_var_setup, badbot_event, expected_exception_access_handler_error): 25 | try: 26 | lambda_handler(badbot_event, context) 27 | except Exception as e: 28 | expected = expected_exception_access_handler_error 29 | assert str(e) == expected 30 | 31 | def test_initialize_usage_data(): 32 | os.environ['LOG_TYPE'] = 'LOG_TYPE' 33 | result = initialize_usage_data() 34 | expected = { 35 | "data_type": "bad_bot", 36 | "bad_bot_ip_set_size": 0, 37 | "allowed_requests": 0, 38 | "blocked_requests_all": 0, 39 | "blocked_requests_bad_bot": 0, 40 | "waf_type": 'LOG_TYPE', 41 | "provisioner": "cfn" 42 | } 43 | assert result == expected 44 | 45 | def test_send_anonymized_usage_data(cloudwatch_client, expected_cw_resp): 46 | result = send_anonymized_usage_data( 47 | scope='ALB', 48 | ipset_name_v4='ipset_name_v4', 49 | ipset_arn_v4='ipset_arn_v4', 50 | ipset_name_v6='ipset_name_v6', 51 | ipset_arn_v6='ipset_arn_v6' 52 | ) 53 | assert result == expected_cw_resp 54 | -------------------------------------------------------------------------------- /source/custom_resource/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | test/* 4 | */__init__.py 5 | **/__init__.py 6 | backoff/* 7 | bin/* 8 | boto3/* 9 | botocore/* 10 | certifi/* 11 | charset*/* 12 | crhelper* 13 | chardet* 14 | dateutil/* 15 | idna/* 16 | jmespath/* 17 | lib/* 18 | package* 19 | python_* 20 | requests/* 21 | s3transfer/* 22 | six* 23 | tenacity* 24 | tests 25 | urllib3/* 26 | yaml 27 | PyYAML-* 28 | source = 29 | . -------------------------------------------------------------------------------- /source/custom_resource/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/custom_resource/__init__.py -------------------------------------------------------------------------------- /source/custom_resource/custom_resource.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import json 5 | from os import getenv 6 | 7 | from lib.cfn_response import send_response 8 | from operations import ( 9 | operation_types, 10 | set_log_group_retention, 11 | config_app_access_log_bucket, 12 | config_waf_log_bucket, 13 | config_web_acl, 14 | config_aws_waf_logs, 15 | generate_app_log_parser_conf, 16 | generate_waf_log_parser_conf, 17 | add_athena_partitions 18 | ) 19 | from operations.operation_types import RESOURCE_TYPE 20 | from aws_lambda_powertools import Logger 21 | 22 | logger = Logger( 23 | level=getenv('LOG_LEVEL') 24 | ) 25 | 26 | 27 | operations_dictionary = { 28 | operation_types.SET_CLOUDWATCH_LOGGROUP_RETENTION: set_log_group_retention.execute, 29 | operation_types.CONFIG_AWS_WAF_LOGS: config_aws_waf_logs.execute, 30 | operation_types.CONFIG_APP_ACCESS_LOG_BUCKET: config_app_access_log_bucket.execute, 31 | operation_types.CONFIG_WAF_LOG_BUCKET: config_waf_log_bucket.execute, 32 | operation_types.CONFIG_WEB_ACL: config_web_acl.execute, 33 | operation_types.GENERATE_APP_LOG_PARSER_CONF_FILE: generate_app_log_parser_conf.execute, 34 | operation_types.GENERATE_WAF_LOG_PARSER_CONF_FILE: generate_waf_log_parser_conf.execute, 35 | operation_types.ADD_ATHENA_PARTITIONS: add_athena_partitions.execute 36 | } 37 | 38 | class UnSupportedOperationTypeException(Exception): 39 | pass 40 | 41 | def get_function_for_resource(resource, log): 42 | try: 43 | return operations_dictionary[resource] 44 | except KeyError as key_error: 45 | log.error(key_error) 46 | raise UnSupportedOperationTypeException(f"The operation {resource} is not supported") 47 | 48 | 49 | # ====================================================================================================================== 50 | # Lambda Entry Point 51 | # ====================================================================================================================== 52 | @logger.inject_lambda_context 53 | def lambda_handler(event, context): 54 | response_status = 'SUCCESS' 55 | reason = None 56 | response_data = {} 57 | resource_id = event.get('PhysicalResourceId', event['LogicalResourceId']) 58 | result = { 59 | 'StatusCode': '200', 60 | 'Body': {'message': 'success'} 61 | } 62 | 63 | try: 64 | # ---------------------------------------------------------- 65 | # Read inputs parameters 66 | # ---------------------------------------------------------- 67 | request_type = event.get('RequestType', "").upper() 68 | logger.info(request_type) 69 | 70 | # ---------------------------------------------------------- 71 | # Process event 72 | # ---------------------------------------------------------- 73 | operation = get_function_for_resource(event[RESOURCE_TYPE], logger) 74 | if operation: 75 | operation(event, context, logger) 76 | 77 | except Exception as error: 78 | logger.error(error) 79 | response_status = 'FAILED' 80 | reason = str(error) 81 | result = { 82 | 'statusCode': '500', 83 | 'body': {'message': reason} 84 | } 85 | 86 | finally: 87 | # ------------------------------------------------------------------ 88 | # Send Result 89 | # ------------------------------------------------------------------ 90 | if 'ResponseURL' in event: 91 | send_response(logger, event, context, response_status, response_data, resource_id, reason) 92 | 93 | return json.dumps(result) #NOSONAR needed to send a response of the result 94 | -------------------------------------------------------------------------------- /source/custom_resource/log_group_retention.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from lib.boto3_util import create_client 15 | 16 | TRUNC_STACK_NAME_MAX_LEN = 20 17 | 18 | class LogGroupRetention: 19 | def __init__(self, log): 20 | self.log = log 21 | 22 | def update_retention(self, event): 23 | cloudwatch = create_client('logs') 24 | 25 | log_group_prefix = self.get_log_group_prefix( 26 | stack_name=event['ResourceProperties']['StackName'] 27 | ) 28 | 29 | log_groups = cloudwatch.describe_log_groups( 30 | logGroupNamePrefix=log_group_prefix 31 | ) 32 | 33 | lambda_names = self.get_lambda_names( 34 | resource_props=event['ResourceProperties'] 35 | ) 36 | 37 | self.set_log_group_retention( 38 | client=cloudwatch, 39 | log_groups=log_groups, 40 | lambda_names=lambda_names, 41 | retention_period=int(event['ResourceProperties']['LogGroupRetention']) 42 | ) 43 | 44 | 45 | def get_lambda_names(self, resource_props): 46 | lambdas = [ 47 | 'CustomResourceLambdaName', 48 | 'MoveS3LogsForPartitionLambdaName', 49 | 'AddAthenaPartitionsLambdaName', 50 | 'SetIPRetentionLambdaName', 51 | 'RemoveExpiredIPLambdaName', 52 | 'ReputationListsParserLambdaName', 53 | 'BadBotParserLambdaName', 54 | 'HelperLambdaName', 55 | 'LogParserLambdaName', 56 | 'CustomTimerLambdaName' 57 | ] 58 | lambda_names = set() 59 | for lam in lambdas: 60 | lambda_name = resource_props.get(lam,'') 61 | if lambda_name: 62 | lambda_names.add(f'/aws/lambda/{lambda_name}') 63 | return lambda_names 64 | 65 | 66 | def truncate_stack_name(self, stack_name): 67 | # in case StackName is too long (up to 128 chars), 68 | # lambda function name (up to 64 chars) will use a truncated StackName 69 | if len(stack_name) < TRUNC_STACK_NAME_MAX_LEN: 70 | return stack_name 71 | return stack_name[0:TRUNC_STACK_NAME_MAX_LEN] 72 | 73 | 74 | def get_log_group_prefix(self, stack_name): 75 | truncated_stack_name = self.truncate_stack_name(stack_name) 76 | return f'/aws/lambda/{truncated_stack_name}' 77 | 78 | 79 | def set_log_group_retention(self, client, log_groups, lambda_names, retention_period): 80 | for log_group in log_groups['logGroups']: 81 | if log_group['logGroupName'] in lambda_names: 82 | client.put_retention_policy( 83 | logGroupName = log_group['logGroupName'], 84 | retentionInDays = int(retention_period) 85 | ) 86 | self.log.info(f'put retention for log group {log_group["logGroupName"]}') -------------------------------------------------------------------------------- /source/custom_resource/operations/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | -------------------------------------------------------------------------------- /source/custom_resource/operations/add_athena_partitions.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from resource_manager import ResourceManager 15 | from operations.operation_types import ( 16 | REQUEST_TYPE, 17 | CREATE, 18 | UPDATE 19 | ) 20 | 21 | def execute(event, _, log): 22 | resource_manager = ResourceManager(log=log) 23 | 24 | if event[REQUEST_TYPE] in {CREATE, UPDATE}: 25 | resource_manager.add_athena_partitions(event) -------------------------------------------------------------------------------- /source/custom_resource/operations/config_app_access_log_bucket.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from resource_manager import ResourceManager 15 | from operations.operation_types import ( 16 | CREATE, 17 | UPDATE, 18 | DELETE, 19 | REQUEST_TYPE 20 | ) 21 | 22 | def execute(event, _, log): 23 | resource_manager = ResourceManager(log=log) 24 | if event[REQUEST_TYPE] == CREATE: 25 | resource_manager.configure_s3_bucket(event) 26 | app_access_params = resource_manager.get_params_app_access_create_event(event) 27 | resource_manager.add_s3_bucket_lambda_event(**app_access_params) 28 | 29 | elif event[REQUEST_TYPE] == UPDATE: 30 | resource_manager.configure_s3_bucket(event) 31 | if resource_manager.contains_old_app_access_resources(event): 32 | resource_manager.update_app_access_log_bucket(event) 33 | 34 | elif event[REQUEST_TYPE] == DELETE: 35 | bucket_lambda_params = resource_manager.get_params_app_access_delete_event(event) 36 | resource_manager.remove_s3_bucket_lambda_event(**bucket_lambda_params) -------------------------------------------------------------------------------- /source/custom_resource/operations/config_aws_waf_logs.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from resource_manager import ResourceManager 15 | from operations.operation_types import ( 16 | DELETE, 17 | REQUEST_TYPE, 18 | CREATE, 19 | UPDATE 20 | ) 21 | 22 | def execute(event, _, log): 23 | resource_manager = ResourceManager(log=log) 24 | 25 | if event[REQUEST_TYPE] == CREATE: 26 | resource_manager.put_logging_configuration(event) 27 | 28 | elif event[REQUEST_TYPE] == UPDATE: 29 | resource_manager.delete_logging_configuration(event) 30 | resource_manager.put_logging_configuration(event) 31 | 32 | elif event[REQUEST_TYPE] == DELETE: 33 | resource_manager.delete_logging_configuration(event) -------------------------------------------------------------------------------- /source/custom_resource/operations/config_waf_log_bucket.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from resource_manager import ResourceManager 15 | from operations.operation_types import ( 16 | CREATE, 17 | UPDATE, 18 | DELETE, 19 | REQUEST_TYPE 20 | ) 21 | 22 | def execute(event, _, log): 23 | resource_manager = ResourceManager(log=log) 24 | if event[REQUEST_TYPE] == CREATE: 25 | waf_params = resource_manager.get_params_waf_event(event) 26 | resource_manager.add_s3_bucket_lambda_event(**waf_params) 27 | 28 | elif event[REQUEST_TYPE] == UPDATE: 29 | if resource_manager.waf_has_old_resources(event): 30 | resource_manager.update_waf_log_bucket(event) 31 | 32 | elif event[REQUEST_TYPE] == DELETE: 33 | bucket_lambda_params = resource_manager.get_params_bucket_lambda_delete_event(event) 34 | resource_manager.remove_s3_bucket_lambda_event(**bucket_lambda_params) -------------------------------------------------------------------------------- /source/custom_resource/operations/config_web_acl.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from resource_manager import ResourceManager 15 | from operations.operation_types import ( 16 | DELETE, 17 | REQUEST_TYPE 18 | ) 19 | 20 | def execute(event, _, log): 21 | resource_manager = ResourceManager(log=log) 22 | if event[REQUEST_TYPE] == DELETE: 23 | resource_manager.delete_ip_sets(event) 24 | 25 | resource_manager.send_anonymized_usage_data(event['RequestType'], event.get('ResourceProperties', {})) -------------------------------------------------------------------------------- /source/custom_resource/operations/generate_app_log_parser_conf.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from resource_manager import ResourceManager 15 | from operations.operation_types import ( 16 | REQUEST_TYPE, 17 | CREATE, 18 | UPDATE 19 | ) 20 | 21 | def execute(event, _, log): 22 | resource_manager = ResourceManager(log=log) 23 | 24 | if event[REQUEST_TYPE] == CREATE: 25 | resource_manager.generate_app_log_parser_conf_file(event, overwrite=True) 26 | 27 | elif event[REQUEST_TYPE] == UPDATE: 28 | resource_manager.generate_app_log_parser_conf_file(event, overwrite=False) 29 | 30 | # DELETE: do nothing -------------------------------------------------------------------------------- /source/custom_resource/operations/generate_waf_log_parser_conf.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from resource_manager import ResourceManager 15 | from operations.operation_types import ( 16 | REQUEST_TYPE, 17 | CREATE, 18 | UPDATE 19 | ) 20 | 21 | def execute(event, _, log): 22 | resource_manager = ResourceManager(log=log) 23 | 24 | if event[REQUEST_TYPE] == CREATE: 25 | resource_manager.generate_waf_log_parser_conf_file(event, overwrite=True) 26 | 27 | elif event[REQUEST_TYPE] == UPDATE: 28 | resource_manager.generate_waf_log_parser_conf_file(event, overwrite=False) 29 | 30 | # DELETE: do nothing -------------------------------------------------------------------------------- /source/custom_resource/operations/operation_types.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | # list of operation names as constants 15 | SET_CLOUDWATCH_LOGGROUP_RETENTION = "Custom::SetCloudWatchLogGroupRetention" 16 | CONFIG_APP_ACCESS_LOG_BUCKET = "Custom::ConfigureAppAccessLogBucket" 17 | CONFIG_WAF_LOG_BUCKET = "Custom::ConfigureWafLogBucket" 18 | CONFIG_WEB_ACL = "Custom::ConfigureWebAcl" 19 | CONFIG_AWS_WAF_LOGS = "Custom::ConfigureAWSWAFLogs" 20 | GENERATE_APP_LOG_PARSER_CONF_FILE = "Custom::GenerateAppLogParserConfFile" 21 | GENERATE_WAF_LOG_PARSER_CONF_FILE = "Custom::GenerateWafLogParserConfFile" 22 | ADD_ATHENA_PARTITIONS = "Custom::AddAthenaPartitions" 23 | 24 | CREATE = "Create" 25 | UPDATE = "Update" 26 | DELETE = "Delete" 27 | 28 | 29 | # additional constants 30 | RESOURCE_PROPERTIES = "ResourceProperties" 31 | REQUEST_TYPE = "RequestType" 32 | RESOURCE_TYPE = "ResourceType" -------------------------------------------------------------------------------- /source/custom_resource/operations/set_log_group_retention.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from operations.operation_types import ( 15 | CREATE, 16 | UPDATE, 17 | REQUEST_TYPE 18 | ) 19 | from log_group_retention import LogGroupRetention 20 | 21 | def execute(event, _, log): 22 | if event[REQUEST_TYPE] in {UPDATE, CREATE}: 23 | log_group_retention = LogGroupRetention(log) 24 | log_group_retention.update_retention( 25 | event=event 26 | ) -------------------------------------------------------------------------------- /source/custom_resource/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | package-mode = false 3 | name = "custom-resource" 4 | 5 | [tool.poetry.dependencies] 6 | python = "~3.12" 7 | requests = "^2.32.3" 8 | backoff = "^2.2.1" 9 | aws-lambda-powertools = "~3.2.0" 10 | 11 | 12 | [tool.poetry.group.dev.dependencies] 13 | moto = "^4.1.4" 14 | pytest = "^7.2.2" 15 | pytest-mock = "^3.12.0" 16 | pytest-runner = "^6.0.0" 17 | pytest-cov = "^4.0.0" 18 | pytest-env = "^0.8.1" 19 | boto3 = "^1.35.30" 20 | 21 | [build-system] 22 | requires = ["poetry-core"] 23 | build-backend = "poetry.core.masonry.api" 24 | -------------------------------------------------------------------------------- /source/custom_resource/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/custom_resource/test/__init__.py -------------------------------------------------------------------------------- /source/custom_resource/test/test_custom_resource.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from custom_resource.custom_resource import lambda_handler 15 | 16 | def test_set_cloud_watch_group_retention(configure_cloud_watch_group_retention_event, example_context, cloudwatch_client, successful_response): 17 | result = lambda_handler(configure_cloud_watch_group_retention_event,example_context) 18 | expected = successful_response 19 | assert result == expected 20 | 21 | def test_generate_waf_log_parser_conf_create_event(generate_waf_log_parser_conf_create_event, example_context, wafv2_client, s3_bucket, s3_client, successful_response): 22 | result = lambda_handler(generate_waf_log_parser_conf_create_event, example_context) 23 | expected = successful_response 24 | assert result == expected 25 | 26 | def test_generate_waf_log_parser_conf_create_event(generate_waf_log_parser_conf_update_event, example_context, wafv2_client, s3_bucket, s3_client, successful_response): 27 | result = lambda_handler(generate_waf_log_parser_conf_update_event, example_context) 28 | expected = successful_response 29 | assert result == expected 30 | 31 | def test_generate_app_log_parser_conf_create_event(generate_app_log_parser_conf_create_event, example_context, wafv2_client, s3_bucket, s3_client, successful_response): 32 | result = lambda_handler(generate_app_log_parser_conf_create_event, example_context) 33 | expected = successful_response 34 | assert result == expected 35 | 36 | def test_generate_app_log_parser_conf_update_event(generate_app_log_parser_conf_update_event, example_context, wafv2_client, s3_bucket, s3_client, successful_response): 37 | result = lambda_handler(generate_app_log_parser_conf_update_event, example_context) 38 | expected = successful_response 39 | assert result == expected 40 | 41 | def test_configure_aws_waf_logs_create_event(configure_aws_waf_logs_create_event, example_context, wafv2_client, successful_response): 42 | result = lambda_handler(configure_aws_waf_logs_create_event, example_context) 43 | expected = successful_response 44 | assert result == expected 45 | 46 | def test_configure_aws_waf_logs_update_event(configure_aws_waf_logs_update_event, example_context, wafv2_client, successful_response): 47 | result = lambda_handler(configure_aws_waf_logs_update_event, example_context) 48 | expected = successful_response 49 | assert result == expected 50 | 51 | def test_configure_aws_waf_logs_update_event(configure_aws_waf_logs_delete_event, example_context, wafv2_client, successful_response): 52 | result = lambda_handler(configure_aws_waf_logs_delete_event, example_context) 53 | expected = successful_response 54 | assert result == expected 55 | 56 | def test_configure_web_acl_delete(configure_web_acl_delete, example_context, successful_response): 57 | result = lambda_handler(configure_web_acl_delete, example_context) 58 | expected = successful_response 59 | assert result == expected 60 | 61 | def test_configure_waf_log_bucket_create_event(configure_waf_log_bucket_create_event, example_context, s3_bucket, s3_client, successful_response): 62 | result = lambda_handler(configure_waf_log_bucket_create_event, example_context) 63 | expected = successful_response 64 | assert result == expected 65 | 66 | def test_configure_waf_log_bucket_delete_event(configure_waf_log_bucket_delete_event, example_context, s3_bucket, s3_client, successful_response): 67 | result = lambda_handler(configure_waf_log_bucket_delete_event, example_context) 68 | expected = successful_response 69 | assert result == expected 70 | 71 | def test_configure_app_access_log_bucket_create_event(configure_app_access_log_bucket_create_error_event, example_context, s3_bucket, s3_client, app_access_log_bucket_create_event_error_response): 72 | result = lambda_handler(configure_app_access_log_bucket_create_error_event, example_context) 73 | expected = app_access_log_bucket_create_event_error_response 74 | assert result == expected 75 | 76 | def test_configure_app_access_log_bucket_delete_event(configure_app_access_log_bucket_delete_event, example_context, s3_bucket, s3_client, successful_response): 77 | result = lambda_handler(configure_app_access_log_bucket_delete_event, example_context) 78 | expected = successful_response 79 | assert result == expected -------------------------------------------------------------------------------- /source/custom_resource/test/test_log_group_retention.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from log_group_retention import LogGroupRetention 15 | import logging 16 | 17 | log_level = 'DEBUG' 18 | logging.getLogger().setLevel(log_level) 19 | log = logging.getLogger('test_log_group_retention') 20 | 21 | lgr = LogGroupRetention(log) 22 | 23 | def test_truncate_stack_name_empty(): 24 | stack_name = '' 25 | expected = '' 26 | res = lgr.truncate_stack_name(stack_name) 27 | assert res == expected 28 | 29 | 30 | def test_truncate_stack_name_short(): 31 | stack_name = 'undertwentychars' 32 | expected = 'undertwentychars' 33 | res = lgr.truncate_stack_name(stack_name) 34 | assert res == expected 35 | 36 | 37 | def test_truncate_stack_name_long(): 38 | stack_name = 'thisisovertwentycharacts' 39 | expected = 'thisisovertwentychar' 40 | res = lgr.truncate_stack_name(stack_name) 41 | assert res == expected 42 | 43 | 44 | def test_get_log_group_prefix(): 45 | stack_name = 'stackname' 46 | expected = '/aws/lambda/stackname' 47 | res = lgr.get_log_group_prefix(stack_name) 48 | assert res == expected 49 | 50 | 51 | def test_get_lambda_names(): 52 | resource_props = { 53 | 'CustomResourceLambdaName': 'TESTCustomResourceLambdaName', 54 | 'MoveS3LogsForPartitionLambdaName': 'TESTMoveS3LogsForPartitionLambdaName', 55 | 'AddAthenaPartitionsLambdaName': 'TESTAddAthenaPartitionsLambdaName', 56 | 'SetIPRetentionLambdaName': 'TESTSetIPRetentionLambdaName', 57 | 'RemoveExpiredIPLambdaName': 'TESTRemoveExpiredIPLambdaName', 58 | 'ReputationListsParserLambdaName': 'TESTReputationListsParserLambdaName', 59 | 'BadBotParserLambdaName': 'TESTBadBotParserLambdaName', 60 | 'CustomResourceLambdaName': 'TESTCustomResourceLambdaName', 61 | 'CustomTimerLambdaName': 'TESTCustomTimerLambdaName', 62 | 'RandomProp': 'TESTRandomProp' 63 | } 64 | expected = { 65 | '/aws/lambda/TESTCustomResourceLambdaName', 66 | '/aws/lambda/TESTMoveS3LogsForPartitionLambdaName', 67 | '/aws/lambda/TESTAddAthenaPartitionsLambdaName', 68 | '/aws/lambda/TESTSetIPRetentionLambdaName', 69 | '/aws/lambda/TESTRemoveExpiredIPLambdaName', 70 | '/aws/lambda/TESTReputationListsParserLambdaName', 71 | '/aws/lambda/TESTBadBotParserLambdaName', 72 | '/aws/lambda/TESTCustomResourceLambdaName', 73 | '/aws/lambda/TESTCustomTimerLambdaName' 74 | } 75 | res = lgr.get_lambda_names(resource_props) 76 | assert res == expected 77 | -------------------------------------------------------------------------------- /source/helper/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | test/* 4 | */__init__.py 5 | **/__init__.py 6 | backoff/* 7 | bin/* 8 | boto3/* 9 | botocore/* 10 | certifi/* 11 | charset*/* 12 | crhelper* 13 | chardet* 14 | dateutil/* 15 | idna/* 16 | jmespath/* 17 | lib/* 18 | package* 19 | python_* 20 | requests/* 21 | s3transfer/* 22 | six* 23 | tenacity* 24 | tests 25 | urllib3/* 26 | yaml 27 | PyYAML-* 28 | source = 29 | . -------------------------------------------------------------------------------- /source/helper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/helper/__init__.py -------------------------------------------------------------------------------- /source/helper/helper.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import json 5 | from os import getenv 6 | 7 | from stack_requirements import StackRequirements 8 | from lib.cfn_response import send_response 9 | from aws_lambda_powertools import Logger 10 | 11 | logger = Logger( 12 | level=getenv('LOG_LEVEL') 13 | ) 14 | 15 | # ====================================================================================================================== 16 | # Lambda Entry Point 17 | # ====================================================================================================================== 18 | def lambda_handler(event, context): 19 | response_status = 'SUCCESS' 20 | reason = None 21 | response_data = {} 22 | resource_id = event['PhysicalResourceId'] if 'PhysicalResourceId' in event else event['LogicalResourceId'] 23 | result = { 24 | 'StatusCode': '200', 25 | 'Body': {'message': 'success'} 26 | } 27 | 28 | stack_requirements = StackRequirements(logger) 29 | 30 | logger.info(f'context: {context}') 31 | try: 32 | # ---------------------------------------------------------- 33 | # Read inputs parameters 34 | # ---------------------------------------------------------- 35 | 36 | request_type = event['RequestType'].upper() if ('RequestType' in event) else "" 37 | logger.info(request_type) 38 | 39 | # ---------------------------------------------------------- 40 | # Process event 41 | # ---------------------------------------------------------- 42 | if event['ResourceType'] == "Custom::CheckRequirements" and request_type in {'CREATE', 'UPDATE'}: 43 | stack_requirements.verify_requirements_and_dependencies(event) 44 | 45 | elif event['ResourceType'] == "Custom::CreateUUID" and request_type == 'CREATE': 46 | stack_requirements.create_uuid(response_data) 47 | 48 | elif event['ResourceType'] == "Custom::CreateDeliveryStreamName" and request_type == 'CREATE': 49 | stack_requirements.create_delivery_stream_name(event, response_data) 50 | 51 | elif event['ResourceType'] == "Custom::CreateGlueDatabaseName" and request_type == 'CREATE': 52 | stack_requirements.create_db_name(event, response_data) 53 | 54 | except Exception as error: 55 | logger.error(error) 56 | response_status = 'FAILED' 57 | reason = str(error) 58 | result = { 59 | 'statusCode': '400', 60 | 'body': {'message': reason} 61 | } 62 | 63 | finally: 64 | # ------------------------------------------------------------------ 65 | # Send Result 66 | # ------------------------------------------------------------------ 67 | if 'ResponseURL' in event: 68 | send_response(logger, event, context, response_status, response_data, resource_id, reason) 69 | 70 | return json.dumps(result) #NOSONAR needed to send a response of the result 71 | -------------------------------------------------------------------------------- /source/helper/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | package-mode = false 3 | name = "helper" 4 | 5 | [tool.poetry.dependencies] 6 | python = "~3.12" 7 | requests = "^2.32.3" 8 | backoff = "^2.2.1" 9 | aws-lambda-powertools = "~3.2.0" 10 | 11 | 12 | [tool.poetry.group.dev.dependencies] 13 | moto = "^4.1.4" 14 | pytest = "^7.2.2" 15 | pytest-mock = "^3.12.0" 16 | pytest-runner = "^6.0.0" 17 | pytest-cov = "^4.0.0" 18 | pytest-env = "^0.8.1" 19 | boto3 = "^1.35.30" 20 | 21 | [build-system] 22 | requires = ["poetry-core"] 23 | build-backend = "poetry.core.masonry.api" 24 | -------------------------------------------------------------------------------- /source/helper/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/helper/test/__init__.py -------------------------------------------------------------------------------- /source/helper/test/conftest.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from types import SimpleNamespace 14 | 15 | import pytest 16 | import boto3 17 | from moto import ( 18 | mock_s3 19 | ) 20 | 21 | 22 | @pytest.fixture(scope="session") 23 | def example_context(): 24 | return SimpleNamespace(**{ 25 | 'function_name': 'foo', 26 | 'memory_limit_in_mb': '512', 27 | 'invoked_function_arn': ':::invoked_function_arn', 28 | 'log_group_name': 'log_group_name', 29 | 'log_stream_name': 'log_stream_name', 30 | 'aws_request_id': 'baz' 31 | }) 32 | 33 | @pytest.fixture(scope="session") 34 | def successful_response(): 35 | return '{"StatusCode": "200", "Body": {"message": "success"}}' 36 | 37 | 38 | @pytest.fixture(scope="session") 39 | def error_response(): 40 | return '{"statusCode": "400", "body": {"message": "\'Region\'"}}' 41 | 42 | 43 | @pytest.fixture(scope="session") 44 | def s3_client(): 45 | with mock_s3(): 46 | s3 = boto3.client('s3') 47 | yield s3 48 | 49 | 50 | @pytest.fixture(scope="session") 51 | def s3_bucket(s3_client): 52 | my_bucket = 'bucket_name' 53 | s3_client.create_bucket(Bucket=my_bucket) 54 | return my_bucket 55 | 56 | 57 | @pytest.fixture(scope="session") 58 | def check_requirements_event(): 59 | return { 60 | 'LogicalResourceId': 'CheckRequirements', 61 | 'RequestId': 'cf0d8086-5b6f-4758-a323-e723925fcb30', 62 | 'RequestType': 'Create', 63 | 'ResourceProperties': { 64 | 'AppAccessLogBucket': 'wiq-wafohio424243-wafohio424243', 65 | 'AthenaLogParser': 'yes', 66 | 'EndpointType': 'ALB', 67 | 'HttpFloodProtectionLogParserActivated': 'yes', 68 | 'HttpFloodProtectionRateBasedRuleActivated': 'no', 69 | 'ProtectionActivatedScannersProbes': 'yes', 70 | 'Region': 'us-east-2', 71 | 'RequestThreshold': '100', 72 | 'ServiceToken': 'arn:aws:lambda:us-east-2:XXXXXXXXXXXX:function:wafohio424243-Helper-xse5nh2WeWlc'}, 73 | 'ResourceType': 'Custom::CheckRequirements', 74 | 'ResponseURL': 'https://cloudformation-custom-resource-response-useast2.s3.us-east-2.amazonaws.com/', 75 | 'ServiceToken': 'arn:aws:lambda:us-east-2:XXXXXXXXXXXX:function:wafohio424243-Helper-xse5nh2WeWlc', 76 | 'StackId': 'arn:aws:cloudformation:us-east-2:XXXXXXXXXXXX:stack/wafohio424243/276aee50-e2e9-11ed-89eb-067ac5804c7f' 77 | } 78 | 79 | 80 | @pytest.fixture(scope="session") 81 | def create_uuid_event(): 82 | return { 83 | 'LogicalResourceId': 'CreateUniqueID', 84 | 'RequestId': 'f84694a1-87c0-4ad8-b483-f7b87147514f', 85 | 'RequestType': 'Create', 86 | 'ResourceProperties': { 87 | 'ServiceToken': 'arn:aws:lambda:us-east-2:XXXXXXXXXXXX:function:wafohio424243-Helper-xse5nh2WeWlc'}, 88 | 'ResourceType': 'Custom::CreateUUID', 89 | 'ResponseURL': 'https://cloudformation-custom-resource-response-useast2.s3.us-east-2.amazonaws.com/', 90 | 'ServiceToken': 'arn:aws:lambda:us-east-2:XXXXXXXXXXXX:function:wafohio424243-Helper-xse5nh2WeWlc', 91 | 'StackId': 'arn:aws:cloudformation:us-east-2:XXXXXXXXXXXX:stack/wafohio424243/276aee50-e2e9-11ed-89eb-067ac5804c7f' 92 | } 93 | 94 | 95 | @pytest.fixture(scope="session") 96 | def create_delivery_stream_name_event(): 97 | return { 98 | 'LogicalResourceId': 'CreateDeliveryStreamName', 99 | 'RequestId': '323e36d8-d20b-446f-9b89-7a7895a30fab', 100 | 'RequestType': 'Create', 101 | 'ResourceProperties': { 102 | 'ServiceToken': 'arn:aws:lambda:us-east-2:XXXXXXXXXXXX:function:wafohio424243-Helper-xse5nh2WeWlc', 103 | 'StackName': 'wafohio424243' 104 | }, 105 | 'ResourceType': 'Custom::CreateDeliveryStreamName', 106 | 'ResponseURL': 'https://cloudformation-custom-resource-response-useast2.s3.us-east-2.amazonaws.com/', 107 | 'ServiceToken': 'arn:aws:lambda:us-east-2:XXXXXXXXXXXX:function:wafohio424243-Helper-xse5nh2WeWlc', 108 | 'StackId': 'arn:aws:cloudformation:us-east-2:XXXXXXXXXXXX:stack/wafohio424243/276aee50-e2e9-11ed-89eb-067ac5804c7f' 109 | } 110 | 111 | 112 | @pytest.fixture(scope="session") 113 | def create_db_name_event(): 114 | return { 115 | 'LogicalResourceId': 'CreateGlueDatabaseName', 116 | 'RequestId': 'e5a8e6c9-3f75-4da9-bcce-c0ac3d2ba823', 117 | 'RequestType': 'Create', 118 | 'ResourceProperties': { 119 | 'ServiceToken': 'arn:aws:lambda:us-east-2:XXXXXXXXXXXX:function:wafohio424243-Helper-xse5nh2WeWlc', 120 | 'StackName': 'wafohio424243' 121 | }, 122 | 'ResourceType': 'Custom::CreateGlueDatabaseName', 123 | 'ResponseURL': 'https://cloudformation-custom-resource-response-useast2.s3.us-east-2.amazonaws.com/', 124 | 'ServiceToken': 'arn:aws:lambda:us-east-2:XXXXXXXXXXXX:function:wafohio424243-Helper-xse5nh2WeWlc', 125 | 'StackId': 'arn:aws:cloudformation:us-east-2:XXXXXXXXXXXX:stack/wafohio424243/276aee50-e2e9-11ed-89eb-067ac5804c7f' 126 | } 127 | 128 | 129 | @pytest.fixture(scope="session") 130 | def erroneous_check_requirements_event(): 131 | return { 132 | 'LogicalResourceId': 'CheckRequirements', 133 | 'RequestId': 'cf0d8086-5b6f-4758-a323-e723925fcb30', 134 | 'RequestType': 'Create', 135 | 'ResourceProperties': { 136 | 'AthenaLogParser': 'yes', 137 | 'EndpointType': 'ALB', 138 | 'HttpFloodProtectionLogParserActivated': 'yes', 139 | 'HttpFloodProtectionRateBasedRuleActivated': 'no', 140 | 'ProtectionActivatedScannersProbes': 'yes', 141 | 'RequestThreshold': '100', 142 | 'ServiceToken': 'arn:aws:lambda:us-east-2:XXXXXXXXXXXX:function:wafohio424243-Helper-xse5nh2WeWlc'}, 143 | 'ResourceType': 'Custom::CheckRequirements', 144 | 'ResponseURL': 'https://cloudformation-custom-resource-response-useast2.s3.us-east-2.amazonaws.com/', 145 | 'ServiceToken': 'arn:aws:lambda:us-east-2:XXXXXXXXXXXX:function:wafohio424243-Helper-xse5nh2WeWlc', 146 | 'StackId': 'arn:aws:cloudformation:us-east-2:XXXXXXXXXXXX:stack/wafohio424243/276aee50-e2e9-11ed-89eb-067ac5804c7f' 147 | } 148 | -------------------------------------------------------------------------------- /source/helper/test/test_helper.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from helper.helper import lambda_handler 15 | 16 | def test_check_requirements(check_requirements_event, example_context, successful_response): 17 | result = lambda_handler(check_requirements_event, example_context) 18 | expected = successful_response 19 | assert result == expected 20 | 21 | def test_create_uuid(create_uuid_event, example_context, successful_response): 22 | result = lambda_handler(create_uuid_event, example_context) 23 | expected = successful_response 24 | assert result == expected 25 | 26 | def test_create_delivery_stream_name_event(create_delivery_stream_name_event, example_context, successful_response): 27 | result = lambda_handler(create_delivery_stream_name_event, example_context) 28 | expected = successful_response 29 | assert result == expected 30 | 31 | def test_create_db_name(create_db_name_event, example_context, successful_response): 32 | result = lambda_handler(create_db_name_event, example_context) 33 | expected = successful_response 34 | assert result == expected 35 | 36 | def test_error(erroneous_check_requirements_event, example_context, error_response): 37 | result = lambda_handler(erroneous_check_requirements_event, example_context) 38 | expected = error_response 39 | assert result == expected 40 | -------------------------------------------------------------------------------- /source/helper/test/test_stack_requirements.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from helper.stack_requirements import ( 15 | StackRequirements, 16 | WAF_FOR_CLOUDFRONT_EXCEPTION_MESSAGE, 17 | INVALID_FLOOD_THRESHOLD_MESSAGE, 18 | EMPTY_S3_BUCKET_NAME_EXCEPTION_MESSAGE, 19 | INCORRECT_REGION_S3_LAMBDA_EXCEPTION_MESSAGE, 20 | ACCESS_ISSUE_S3_BUCKET_EXCEPTION_MESSAGE 21 | ) 22 | from moto import ( 23 | mock_s3 24 | ) 25 | from uuid import UUID 26 | from lib.boto3_util import create_client 27 | import logging 28 | import boto3 29 | 30 | 31 | log_level = 'DEBUG' 32 | logging.getLogger().setLevel(log_level) 33 | log = logging.getLogger('test_help') 34 | 35 | stack_requirements = StackRequirements(log=log) 36 | 37 | 38 | def test_create_delivery_stream_name(): 39 | event = { 40 | 'ResourceProperties': { 41 | 'StackName': 'stack-name' 42 | } 43 | } 44 | response_data = {} 45 | stack_requirements.create_delivery_stream_name(event, response_data) 46 | 47 | expected = 'aws-waf-logs-stackname' 48 | # ingore randomly generated 7 char suffix 49 | assert response_data['DeliveryStreamName'][:-7] == expected 50 | 51 | 52 | def test_normalize_stack_name(): 53 | stack_name = 'test stack name_)(just over thirty two characters' 54 | suffix = 'adsf13' 55 | expected = 'test_stack_name_just_over' 56 | 57 | res = stack_requirements.normalize_stack_name(stack_name, suffix) 58 | 59 | assert res == expected 60 | 61 | def test_create_db_name(): 62 | event = { 63 | 'ResourceProperties': { 64 | 'StackName': 'stack_name' 65 | } 66 | } 67 | response_data = {} 68 | expected = 'stack_name' 69 | stack_requirements.create_db_name(event, response_data) 70 | 71 | # ingore randomly generated 7 char suffix 72 | assert response_data['DatabaseName'][:-7] == expected 73 | 74 | 75 | def test_create_uuid(): 76 | response_data = {} 77 | stack_requirements.create_uuid(response_data) 78 | try: 79 | UUID(response_data['UUID'], version=4) 80 | assert True 81 | except ValueError: 82 | assert False 83 | 84 | 85 | def test_check_app_log_bucket_empty_bucket_name_exception(): 86 | expected = EMPTY_S3_BUCKET_NAME_EXCEPTION_MESSAGE 87 | try: 88 | stack_requirements.check_app_log_bucket(region='us-east-1', bucket_name="") 89 | except Exception as e: 90 | assert str(e) == expected 91 | 92 | 93 | @mock_s3 94 | def test_check_app_log_bucket(): 95 | conn = boto3.resource("s3", region_name="us-east-1") 96 | conn.create_bucket(Bucket="mybucket") 97 | 98 | expected = INCORRECT_REGION_S3_LAMBDA_EXCEPTION_MESSAGE 99 | try: 100 | stack_requirements.check_app_log_bucket(region='us-east-2', bucket_name="mybucket") 101 | except Exception as e: 102 | assert str(e) == expected 103 | 104 | 105 | @mock_s3 106 | def test_verify_bucket_region_access_issue(): 107 | region = 'us-east-1' 108 | conn = boto3.resource("s3", region_name=region) 109 | conn.create_bucket(Bucket="mybucket1") 110 | 111 | expected = ACCESS_ISSUE_S3_BUCKET_EXCEPTION_MESSAGE 112 | try: 113 | stack_requirements.verify_bucket_region( 114 | bucket_name='nonexistent', 115 | region=region) 116 | except Exception as e: 117 | assert str(e) == expected 118 | 119 | 120 | def test_check_requirements_invalid_flood_threshold(): 121 | resource_properties = { 122 | 'HttpFloodProtectionLogParserActivated': "yes", 123 | 'HttpFloodProtectionRateBasedRuleActivated': "yes", 124 | 'EndpointType': 'cloudfront', 125 | 'Region': 'us-east-1', 126 | 'RequestThreshold': '10' 127 | } 128 | expected = INVALID_FLOOD_THRESHOLD_MESSAGE 129 | 130 | try: 131 | stack_requirements.check_requirements(resource_properties) 132 | except Exception as e: 133 | assert str(e) == expected 134 | 135 | 136 | def test_is_waf_for_cloudfront(): 137 | resource_properties = { 138 | 'HttpFloodProtectionLogParserActivated': "yes", 139 | 'EndpointType': 'cloudfront', 140 | 'Region': 'us-east-2' 141 | } 142 | expected = True 143 | res = stack_requirements.is_waf_for_cloudfront(resource_properties) 144 | assert res == expected 145 | 146 | 147 | 148 | def test_is_invalid_flood_threshold(): 149 | resource_properties = { 150 | 'HttpFloodProtectionRateBasedRuleActivated': "yes", 151 | 'RequestThreshold': '10' 152 | } 153 | expected = True 154 | res = stack_requirements.is_invalid_flood_threshold(resource_properties) 155 | assert res == expected 156 | -------------------------------------------------------------------------------- /source/image/architecture_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/image/architecture_diagram.png -------------------------------------------------------------------------------- /source/ip_retention_handler/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | test/* 4 | */__init__.py 5 | **/__init__.py 6 | backoff/* 7 | bin/* 8 | boto3/* 9 | botocore/* 10 | certifi/* 11 | charset*/* 12 | crhelper* 13 | chardet* 14 | dateutil/* 15 | idna/* 16 | jmespath/* 17 | lib/* 18 | package* 19 | python_* 20 | requests/* 21 | s3transfer/* 22 | six* 23 | tenacity* 24 | tests 25 | urllib3/* 26 | yaml 27 | PyYAML-* 28 | source = 29 | . -------------------------------------------------------------------------------- /source/ip_retention_handler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/ip_retention_handler/__init__.py -------------------------------------------------------------------------------- /source/ip_retention_handler/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "ip_retention_handler" 3 | package-mode = false 4 | 5 | [tool.poetry.dependencies] 6 | python = "~3.12" 7 | requests = "^2.32.3" 8 | backoff = "^2.2.1" 9 | aws-lambda-powertools = "~3.2.0" 10 | 11 | 12 | [tool.poetry.group.dev.dependencies] 13 | moto = "^4.1.4" 14 | pytest = "^7.2.2" 15 | pytest-mock = "^3.12.0" 16 | pytest-runner = "^6.0.0" 17 | pytest-cov = "^4.0.0" 18 | pytest-env = "^0.8.1" 19 | boto3 = "^1.35.30" 20 | 21 | [build-system] 22 | requires = ["poetry-core"] 23 | build-backend = "poetry.core.masonry.api" 24 | -------------------------------------------------------------------------------- /source/ip_retention_handler/set_ip_retention.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | ###################################################################################################################### 5 | ###################################################################################################################### 6 | 7 | from os import environ, getenv 8 | from calendar import timegm 9 | from datetime import datetime, timedelta 10 | from lib.dynamodb_util import DDB 11 | from aws_lambda_powertools import Logger 12 | 13 | logger = Logger( 14 | level=getenv('LOG_LEVEL') 15 | ) 16 | 17 | class SetIPRetention(object): 18 | """ 19 | This class contains functions to put ip retention info into ddb table 20 | """ 21 | 22 | def __init__(self, event, log): 23 | """ 24 | Class init function 25 | """ 26 | 27 | self.event = event 28 | self.log = log 29 | self.log.debug(self.__class__.__name__ + " Class Event:\n{}".format(event)) 30 | 31 | def is_none(self, value): 32 | """ 33 | Return None (string type) if the value is NoneType 34 | """ 35 | 36 | if value is None: 37 | return 'None' 38 | else: 39 | return value 40 | 41 | def get_expiration_time(self, time, ip_retention_period_minute): 42 | """ 43 | Get ip expiration time which is the TTL used by ddb table to delete ip upon expiration 44 | """ 45 | 46 | utc_start_time = datetime.strptime(time, "%Y-%m-%dT%H:%M:%SZ") 47 | utc_end_time = utc_start_time + timedelta(seconds=60*ip_retention_period_minute) 48 | epoch_time = timegm(utc_end_time.utctimetuple()) 49 | return epoch_time 50 | 51 | def make_item(self, event): 52 | """ 53 | Extract ip retention info from event to make ddb item 54 | """ 55 | 56 | item = {} 57 | request_parameters = self.is_none(event.get('requestParameters', {})) 58 | 59 | ip_retention_period = int(environ.get('IP_RETENTION_PERIOD_ALLOWED_MINUTE')) \ 60 | if self.is_none(str(request_parameters.get('name')).find('Whitelist')) != -1 \ 61 | else int(environ.get('IP_RETENTION_PERIOD_DENIED_MINUTE')) 62 | 63 | # If retention period is not set, stop and return 64 | if ip_retention_period == -1: 65 | self.log.info("[set_ip_retention: make_item] IP retention is not set on {}. Stop processing." \ 66 | .format(self.is_none(str(request_parameters.get('name'))))) 67 | return item 68 | 69 | # Set a minimum 15-minute retention period 70 | ip_retention_period = 15 if ip_retention_period in range(0, 15) else ip_retention_period 71 | 72 | item = { 73 | "IPSetId": self.is_none(str(request_parameters.get('id'))), 74 | "IPSetName": self.is_none(str(request_parameters.get('name'))), 75 | "Scope": self.is_none(str(request_parameters.get('scope'))), 76 | "IPAdressList": self.is_none(request_parameters.get('addresses',[])), 77 | "LockToken": self.is_none(str(request_parameters.get('lockToken'))), 78 | "IPRetentionPeriodMinute": ip_retention_period, 79 | "CreationTime": timegm(datetime.utcnow().utctimetuple()), 80 | "ExpirationTime": self.get_expiration_time(event.get('eventTime'), ip_retention_period), 81 | "CreatedByUser": environ.get('STACK_NAME') 82 | } 83 | return item 84 | 85 | def put_item(self, table_name): 86 | """ 87 | Write item into ddb table 88 | """ 89 | try: 90 | self.log.info("[set_ip_retention: put_item] Start") 91 | 92 | ddb = DDB(self.log, table_name) 93 | 94 | item = self.make_item(self.event) 95 | 96 | response = {} 97 | 98 | # put item if it is not empty 99 | if bool(item): 100 | response = ddb.put_item(item) 101 | 102 | self.log.info("[set_ip_retention: put_item] item: \n{}".format(item)) 103 | self.log.info("[set_ip_retention: put_item] put_item response: \n{}:".format(response)) 104 | 105 | except Exception as error: 106 | self.log.error(str(error)) 107 | raise 108 | 109 | self.log.info("[set_ip_retention:put_item] End") 110 | 111 | return response 112 | 113 | @logger.inject_lambda_context 114 | def lambda_handler(event, _): 115 | """ 116 | Invoke functions to put ip retention info into ddb table. 117 | It is triggered by a CloudWatch events rule. 118 | """ 119 | try: 120 | logger.info('[set_ip_retention: lambda_handler] Start') 121 | logger.info("Lambda Handler Event: \n{}".format(event)) 122 | 123 | event_detail = event.get('detail',{}) 124 | event_user_arn = event_detail.get('userIdentity',{}).get('arn') 125 | response = {} 126 | 127 | # If event for UpdateIPSet api call is not created by the RemoveExpiredIP lambda, continue to put item into DDB 128 | if event_user_arn.find(environ.get('REMOVE_EXPIRED_IP_LAMBDA_ROLE_NAME')) == -1: 129 | sipr = SetIPRetention(event_detail, logger) 130 | response = sipr.put_item(environ.get('TABLE_NAME')) 131 | else: 132 | message = "The event for UpdateIPSet API call was made by RemoveExpiredIP lambda instead of user. Skip." 133 | logger.info(message) 134 | response = {"Message": message} 135 | except Exception as error: 136 | logger.error(str(error)) 137 | raise 138 | 139 | logger.info('[set_ip_retention: lambda_handler] End') 140 | return response 141 | -------------------------------------------------------------------------------- /source/ip_retention_handler/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/ip_retention_handler/test/__init__.py -------------------------------------------------------------------------------- /source/ip_retention_handler/test/conftest.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). # 5 | # You may not use this file except in compliance with the License. 6 | # A copy of the License is located at # 7 | # # 8 | # http://www.apache.org/licenses/LICENSE-2.0 # 9 | # # 10 | # or in the "license" file accompanying this file. This file is distributed # 11 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express # 12 | # or implied. See the License for the specific language governing permissions# 13 | # and limitations under the License. # 14 | ############################################################################### 15 | 16 | import boto3 17 | import pytest 18 | from os import environ 19 | from moto import mock_dynamodb, mock_sns, mock_wafv2 20 | from moto.core import DEFAULT_ACCOUNT_ID 21 | from moto.sns import sns_backends 22 | 23 | 24 | REGION = "us-east-1" 25 | TABLE_NAME = "test_table" 26 | 27 | 28 | @pytest.fixture(scope='module', autouse=True) 29 | def test_aws_credentials_setup(): 30 | """Mocked AWS Credentials for moto""" 31 | environ['AWS_ACCESS_KEY_ID'] = 'testing' 32 | environ['AWS_SECRET_ACCESS_KEY'] = 'testing' 33 | environ['AWS_SECURITY_TOKEN'] = 'testing' 34 | environ['AWS_SESSION_TOKEN'] = 'testing' 35 | environ['AWS_DEFAULT_REGION'] = 'us-east-1' 36 | environ['AWS_REGION'] = 'us-east-1' 37 | 38 | 39 | @pytest.fixture(scope='module', autouse=True) 40 | def test_environment_vars_setup(): 41 | environ['TABLE_NAME'] = TABLE_NAME 42 | environ['STACK_NAME'] = 'waf_stack' 43 | environ['SNS_EMAIL'] = 'yes' 44 | environ['UUID'] = "waf_test_uuid" 45 | environ['SOLUTION_ID'] = "waf_test_solution_id" 46 | environ['METRICS_URL'] = "https://testurl.com/generic" 47 | environ['SEND_ANONYMIZED_USAGE_DATA'] = 'yes' 48 | 49 | 50 | @pytest.fixture(scope='module', autouse=True) 51 | def ddb_resource(): 52 | with mock_dynamodb(): 53 | connection = boto3.resource("dynamodb", region_name=REGION) 54 | yield connection 55 | 56 | 57 | @pytest.fixture(scope='module', autouse=True) 58 | def ddb_table(ddb_resource): 59 | conn = ddb_resource 60 | conn.Table(TABLE_NAME) 61 | 62 | 63 | @pytest.fixture(scope='module', autouse=True) 64 | def sns_client(): 65 | with mock_sns(): 66 | connection = boto3.resource("sns", region_name=REGION) 67 | yield connection 68 | 69 | 70 | @pytest.fixture(scope='module', autouse=True) 71 | def sns_topic(): 72 | sns_backend = sns_backends[DEFAULT_ACCOUNT_ID]["us-east-1"] # Use the appropriate account/region 73 | topic_arn = sns_backend.create_topic("some_topic") 74 | return topic_arn 75 | 76 | 77 | @pytest.fixture(scope='module', autouse=True) 78 | def wafv2_client(): 79 | with mock_wafv2(): 80 | connection = boto3.client("wafv2", region_name=REGION) 81 | yield connection 82 | 83 | 84 | # with patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call): 85 | # client = boto3.client('s3') 86 | # # Should return actual result 87 | # o = client.get_object(Bucket='my-bucket', Key='my-key') 88 | # # Should return mocked exception 89 | # e = client.upload_part_copy() 90 | 91 | @pytest.fixture(scope='module', autouse=True) 92 | def set_ip_retention_test_event_setup(ddb_resource): 93 | event = { 94 | "detail": { 95 | "userIdentity": { 96 | "arn": "fake-arn" 97 | }, 98 | "eventTime": "2023-04-27T22:33:04Z", 99 | "requestParameters": { 100 | "name": "fake-Whitelist-ip-set-name", 101 | "scope": "CLOUDFRONT", 102 | "id": "fake-ip-set-id", 103 | "description": "Allow List for IPV4 addresses", 104 | "addresses": [ 105 | "x.x.x.x/32", 106 | "y.y.y.y/32", 107 | "z.z.z.z/32" 108 | ], 109 | "lockToken": "fake-lock-token" 110 | } 111 | } 112 | } 113 | return event 114 | 115 | 116 | @pytest.fixture(scope='function') 117 | def missing_request_parameters_test_event_setup(): 118 | event = { 119 | "detail": { 120 | "userIdentity": { 121 | "arn": "fake-arn" 122 | }, 123 | "eventTime": "2023-04-27T22:33:04Z" 124 | } 125 | } 126 | return event -------------------------------------------------------------------------------- /source/ip_retention_handler/test/test_remove_expired_ip.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import logging 5 | from decimal import Decimal 6 | from os import environ 7 | from types import SimpleNamespace 8 | 9 | from remove_expired_ip import RemoveExpiredIP, lambda_handler 10 | 11 | REMOVE_IP_LIST = ["x.x.x.x", "y.y.y.y"] 12 | EXPECTED_NONE_TYPE_ERROR_MESSAGE = "'NoneType' object has no attribute 'get'" 13 | EXPECTED_NONE_TYPE_NO_ATTRIBUTE_MESSAGE = "'NoneType' object has no attribute 'status_code'" 14 | EVENT = { 15 | "Records": [{ 16 | "eventID": "fake-event-id", 17 | "eventName": "REMOVE", 18 | "eventVersion": "1.1", 19 | "eventSource": "aws:dynamodb", 20 | "awsRegion": "us-east-1", 21 | "dynamodb": { 22 | "ApproximateCreationDateTime": 1628203857.0, 23 | "Keys": { 24 | "ExpirationTime": { 25 | "N": "1628203246" 26 | }, 27 | "IPSetId": { 28 | "S": "fake-ips-set-id" 29 | } 30 | }, 31 | "OldImage": { 32 | "IPSetName": { 33 | "S": "fake-ip-set-name" 34 | }, 35 | "CreatedByUser": { 36 | "S": "fake-user" 37 | }, 38 | "Scope": { 39 | "S": "CLOUDFRONT" 40 | }, 41 | "CreationTime": { 42 | "N": "1628203216" 43 | }, 44 | "LockToken": { 45 | "S": "fake-lock_token" 46 | }, 47 | "IPAdressList": { 48 | "L": [{ 49 | "S": "x.x.x.x/32" 50 | }, { 51 | "S": "y.y.y.y/32" 52 | }] 53 | }, 54 | "ExpirationTime": { 55 | "N": "1628203246" 56 | }, 57 | "IPSetId": { 58 | "S": "fake-ips-set-id" 59 | } 60 | }, 61 | "SequenceNumber": "fake-sequence-number", 62 | "SizeBytes": 339, 63 | "StreamViewType": "OLD_IMAGE" 64 | }, 65 | "userIdentity": { 66 | "principalId": "dynamodb.amazonaws.com", 67 | "type": "Service" 68 | }, 69 | "eventSourceARN": "arn:aws:dynamodb:us-east-1:fake-account:table/fake-ddb-table/stream/2021-07-26T22:26:39.107" 70 | }] 71 | } 72 | 73 | EVENT_NAME_NOT_REMOVE = { 74 | "Records": [{ 75 | "eventID": "fake-event-id", 76 | "eventName": "ADD", 77 | "eventVersion": "1.1", 78 | "eventSource": "aws:dynamodb", 79 | "awsRegion": "us-east-1" 80 | }] 81 | } 82 | 83 | USER_IDENTITY = { 84 | "principalId": "dynamodb.amazonaws.com", 85 | "type": "Service" 86 | } 87 | 88 | USER_IDENTITY_NOT_SERVICE = { 89 | "principalId": "dynamodb.amazonaws.com", 90 | "type": "Any" 91 | } 92 | 93 | log = logging.getLogger() 94 | log.setLevel('INFO') 95 | reip = RemoveExpiredIP(EVENT, log) 96 | 97 | 98 | def test_is_none(): 99 | is_not_none = reip.is_none('some_value') 100 | is_none = reip.is_none(None) 101 | assert is_not_none == 'some_value' and is_none == 'None' 102 | 103 | 104 | def test_is_ddb_stream_event(): 105 | is_ddb_stream_event = reip.is_ddb_stream_event(USER_IDENTITY) 106 | assert is_ddb_stream_event == True 107 | 108 | 109 | def test_deserialize_ddb_data(): 110 | record = EVENT['Records'][0] 111 | ddb_ip_set = reip.is_none(record.get('dynamodb', {}).get('OldImage', {})) 112 | desiralized_ddb_ip_set = reip.deserialize_ddb_data(ddb_ip_set) 113 | expected_desiralized_ddb_ip_set = {'IPSetName': 'fake-ip-set-name', 'CreatedByUser': 'fake-user', 114 | 'Scope': 'CLOUDFRONT', 'CreationTime': Decimal('1628203216'), 115 | 'LockToken': 'fake-lock_token', 'IPAdressList': ['x.x.x.x/32', 'y.y.y.y/32'], 116 | 'ExpirationTime': Decimal('1628203246'), 'IPSetId': 'fake-ips-set-id'} 117 | assert desiralized_ddb_ip_set == expected_desiralized_ddb_ip_set 118 | 119 | 120 | def test_make_ip_list(): 121 | waf_ip_list = ['x.x.x.x/32', 'y.y.y.y/32'] 122 | ddb_ip_list = ['x.x.x.x/32', 'y.y.y.y/32', 'z.z.z.z/32', 'x.y.y.y/32', 'x.x.y.y/32'] 123 | keep_ip_list, remove_ip_list = reip.make_ip_list(log, waf_ip_list, ddb_ip_list) 124 | assert keep_ip_list == [] 125 | assert len(remove_ip_list) > 0 126 | 127 | 128 | def test_make_ip_list_no_removed_ips(): 129 | waf_ip_list = ['x.x.x.x/32', 'y.y.y.y/32'] 130 | ddb_ip_list = ['z.z.z.z/32', 'x.y.y.y/32', 'x.x.y.y/32'] 131 | keep_ip_list, remove_ip_list = reip.make_ip_list(log, waf_ip_list, ddb_ip_list) 132 | assert keep_ip_list == [] 133 | assert len(remove_ip_list) == 0 134 | 135 | 136 | def test_send_notification(sns_topic): 137 | topic_arn = str(sns_topic) 138 | result = False 139 | reip.send_notification(log, topic_arn, "fake_ip_set_name", "fake_ip_set_id", 30, "fake_lambda_name") 140 | result = True 141 | assert result == True 142 | 143 | 144 | def test_send_anonymized_usage_data_allowed_list(): 145 | try: 146 | reip.send_anonymized_usage_data(log, REMOVE_IP_LIST, 'Whitelist') 147 | except Exception as e: 148 | assert str(e) == EXPECTED_NONE_TYPE_NO_ATTRIBUTE_MESSAGE 149 | 150 | 151 | def test_send_anonymized_usage_data_denied_list(): 152 | try: 153 | reip.send_anonymized_usage_data(log, REMOVE_IP_LIST, 'Blacklist') 154 | except Exception as e: 155 | assert str(e) == EXPECTED_NONE_TYPE_NO_ATTRIBUTE_MESSAGE 156 | 157 | 158 | def test_send_anonymized_usage_data_other_list(): 159 | try: 160 | reip.send_anonymized_usage_data(log, REMOVE_IP_LIST, 'Otherlist') 161 | except Exception as e: 162 | assert str(e) == EXPECTED_NONE_TYPE_NO_ATTRIBUTE_MESSAGE 163 | 164 | 165 | def test_send_anonymized_usage_data_empty_list(): 166 | try: 167 | reip.send_anonymized_usage_data(log, [], 'Otherlist') 168 | except Exception as e: 169 | assert str(e) == EXPECTED_NONE_TYPE_NO_ATTRIBUTE_MESSAGE 170 | 171 | 172 | def test_no_send_anonymized_usage_data(): 173 | environ['SEND_ANONYMIZED_USAGE_DATA'] = 'no' 174 | result = reip.send_anonymized_usage_data(log, [], 'Otherlist') 175 | result is not None 176 | 177 | 178 | def test_none_ip_set(): 179 | environ['SEND_ANONYMIZED_USAGE_DATA'] = 'no' 180 | result = reip.get_ip_set(log, None, 'fake-ip-set-name', 'fake-ip-set-id') 181 | result is None 182 | 183 | 184 | def test_remove_expired_ip(): 185 | try: 186 | lambda_handler(EVENT, SimpleNamespace(**{ 187 | 'function_name': 'foo', 188 | 'memory_limit_in_mb': '512', 189 | 'invoked_function_arn': ':::invoked_function_arn', 190 | 'log_group_name': 'log_group_name', 191 | 'log_stream_name': 'log_stream_name', 192 | 'aws_request_id': 'baz' 193 | })) 194 | except Exception as e: 195 | assert str(e) == EXPECTED_NONE_TYPE_ERROR_MESSAGE 196 | -------------------------------------------------------------------------------- /source/ip_retention_handler/test/test_set_ip_retention.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from os import environ 5 | from types import SimpleNamespace 6 | 7 | from set_ip_retention import lambda_handler 8 | 9 | context = SimpleNamespace(**{ 10 | 'function_name': 'foo', 11 | 'memory_limit_in_mb': '512', 12 | 'invoked_function_arn': ':::invoked_function_arn', 13 | 'log_group_name': 'log_group_name', 14 | 'log_stream_name': 'log_stream_name', 15 | 'aws_request_id': 'baz' 16 | }) 17 | 18 | SKIP_PROCESS_MESSAGE = "The event for UpdateIPSet API call was made by RemoveExpiredIP lambda instead of user. Skip." 19 | 20 | 21 | def test_set_ip_retention(set_ip_retention_test_event_setup): 22 | environ['REMOVE_EXPIRED_IP_LAMBDA_ROLE_NAME'] = 'some_role' 23 | environ['IP_RETENTION_PERIOD_ALLOWED_MINUTE'] = '60' 24 | environ['IP_RETENTION_PERIOD_DENIED_MINUTE'] = '60' 25 | environ['TABLE_NAME'] = "test_table" 26 | event = set_ip_retention_test_event_setup 27 | result = lambda_handler(event, context) 28 | assert result is None 29 | 30 | 31 | def test_ip_retention_not_activated(set_ip_retention_test_event_setup): 32 | environ['REMOVE_EXPIRED_IP_LAMBDA_ROLE_NAME'] = 'some_role' 33 | environ['IP_RETENTION_PERIOD_ALLOWED_MINUTE'] = '-1' 34 | environ['IP_RETENTION_PERIOD_DENIED_MINUTE'] = '-1' 35 | event = set_ip_retention_test_event_setup 36 | result = lambda_handler(event, context) 37 | assert result is not None 38 | 39 | def test_missing_request_parameters_in_event(missing_request_parameters_test_event_setup): 40 | environ['REMOVE_EXPIRED_IP_LAMBDA_ROLE_NAME'] = 'some_role' 41 | environ['IP_RETENTION_PERIOD_ALLOWED_MINUTE'] = '60' 42 | environ['IP_RETENTION_PERIOD_DENIED_MINUTE'] = '60' 43 | event = missing_request_parameters_test_event_setup 44 | result = lambda_handler(event, context) 45 | assert result is None 46 | 47 | 48 | def test_skip_process(set_ip_retention_test_event_setup): 49 | environ['REMOVE_EXPIRED_IP_LAMBDA_ROLE_NAME'] = 'fake-arn' 50 | event = set_ip_retention_test_event_setup 51 | result = {"Message": SKIP_PROCESS_MESSAGE} 52 | assert result == lambda_handler(event, context) 53 | 54 | 55 | def test_put_item_exception(set_ip_retention_test_event_setup): 56 | try: 57 | environ['REMOVE_EXPIRED_IP_LAMBDA_ROLE_NAME'] = 'some_role' 58 | environ['IP_RETENTION_PERIOD_ALLOWED_MINUTE'] = '-1' 59 | environ['IP_RETENTION_PERIOD_DENIED_MINUTE'] = '60' 60 | environ.pop('TABLE_NAME') 61 | event = set_ip_retention_test_event_setup 62 | result = False 63 | lambda_handler(event, context) 64 | result = True 65 | except Exception as e: 66 | assert result == False -------------------------------------------------------------------------------- /source/lib/boto3_util.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | #!/bin/python 14 | 15 | import boto3 16 | import logging 17 | from os import environ 18 | from botocore.config import Config 19 | 20 | log = logging.getLogger() 21 | 22 | def create_client(service_name, max_attempt=5, mode='standard', user_agent_extra=environ.get('USER_AGENT_EXTRA'), my_config = {}): 23 | """ 24 | This function creates a boto3 client given a service and its configurations 25 | """ 26 | try: 27 | config = Config( 28 | user_agent_extra=user_agent_extra, 29 | retries={'max_attempts': max_attempt, 'mode': mode} 30 | ) 31 | if my_config != {}: 32 | config = my_config 33 | 34 | return boto3.client( 35 | service_name, 36 | config = config 37 | ) 38 | except Exception as e: 39 | log.error("[boto3_util: create_client] failed to create client") 40 | log.error(e) 41 | raise e 42 | 43 | 44 | def create_resource(service_name, max_attempt=5, mode='standard', user_agent_extra=environ.get('USER_AGENT_EXTRA'), my_config = {}): 45 | """ 46 | This function creates a boto3 resource given a service and its configurations 47 | """ 48 | try: 49 | config = Config( 50 | user_agent_extra=user_agent_extra, 51 | retries={'max_attempts': max_attempt, 'mode': mode} 52 | ) 53 | if my_config != {}: 54 | config = my_config 55 | 56 | return boto3.resource( 57 | service_name, 58 | config = config 59 | ) 60 | except Exception as e: 61 | log.error("[boto3_util: create_resource] failed to create resource") 62 | log.error(e) 63 | raise e -------------------------------------------------------------------------------- /source/lib/cfn_response.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | # !/bin/python 14 | 15 | import requests 16 | import json 17 | 18 | def send_response(log, event, context, response_status, response_data, resource_id, reason=None): 19 | """ 20 | Send a response to an AWS CloudFormation custom resource. 21 | Parameters: 22 | event: The fields in a custom resource request 23 | context: An object, specific to Lambda functions, that you can use to specify 24 | when the function and any callbacks have completed execution, or to 25 | access information from within the Lambda execution environment 26 | response_status: Whether the function successfully completed - SUCCESS or FAILED 27 | response_data: The Data field of a custom resource response object 28 | resource_id: The id of the custom resource that invoked the function 29 | reason: The error message if the function fails 30 | 31 | Returns: None 32 | """ 33 | log.debug("[send_response] Start") 34 | 35 | responseUrl = event['ResponseURL'] 36 | cw_logs_url = "https://console.aws.amazon.com/cloudwatch/home?region=%s#logEventViewer:group=%s;stream=%s" % ( 37 | context.invoked_function_arn.split(':')[3], context.log_group_name, context.log_stream_name) 38 | 39 | log.info("[send_response] Sending cfn response url: %s", responseUrl) 40 | responseBody = {} 41 | responseBody['Status'] = response_status 42 | responseBody['Reason'] = reason or ('See the details in CloudWatch Logs: ' + cw_logs_url) 43 | responseBody['PhysicalResourceId'] = resource_id 44 | responseBody['StackId'] = event['StackId'] 45 | responseBody['RequestId'] = event['RequestId'] 46 | responseBody['LogicalResourceId'] = event['LogicalResourceId'] 47 | responseBody['NoEcho'] = False 48 | responseBody['Data'] = response_data 49 | 50 | json_responseBody = json.dumps(responseBody) 51 | log.debug("Response body:\n" + json_responseBody) 52 | 53 | headers = { 54 | 'content-type': '', 55 | 'content-length': str(len(json_responseBody)) 56 | } 57 | 58 | try: 59 | response = requests.put(responseUrl, 60 | data=json_responseBody, 61 | headers=headers, 62 | timeout=10) 63 | log.info("[send_response] Sending cfn response status code: %s", response.reason) 64 | 65 | except Exception as error: 66 | log.error("[send_response] Failed executing requests.put(..)") 67 | log.error(str(error)) 68 | 69 | log.debug("[send_response] End") -------------------------------------------------------------------------------- /source/lib/cw_metrics_util.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | #!/bin/python 14 | 15 | import datetime 16 | from os import environ 17 | from lib.boto3_util import create_client 18 | 19 | class WAFCloudWatchMetrics(object): 20 | """ 21 | This class creates a wrapper function for cloudwatch get_metric_statistics API 22 | and another function to add the waf cw metric statistics to the anonymized usage 23 | data that the solution collects 24 | """ 25 | def __init__(self, log): 26 | self.log = log 27 | self.cw_client = create_client('cloudwatch') 28 | 29 | def get_cw_metric_statistics(self, metric_name, period_seconds, waf_rule, 30 | namespace='AWS/WAFV2', 31 | statistics=['Sum'], 32 | start_time=datetime.datetime.utcnow(), 33 | end_time=datetime.datetime.utcnow(), 34 | web_acl='STACK_NAME'): 35 | """ 36 | Get a WAF CloudWatch metric given a WAF rule and metric name. 37 | Parameters: 38 | metric_name: string. The name of the metric. Optional. 39 | period_seconds: integer. The granularity, in seconds, of the returned data points. 40 | waf_rule: string. The name of the WAF rule. 41 | namespace: string. The namespace of the metric. Optional. 42 | statistics: list. The metric statistics, other than percentile. Optional. 43 | start_time: datetime. The time stamp that determines the first data point to return. Optional. 44 | end_time: datetime. The time stamp that determines the last data point to return. Optional. 45 | web_acl: string. The name of the WebACL. Optional 46 | 47 | Returns: Metric data points if any, or None 48 | """ 49 | try: 50 | response = self.cw_client.get_metric_statistics( 51 | MetricName=metric_name, 52 | Namespace=namespace, 53 | Statistics=statistics, 54 | Period=period_seconds, 55 | StartTime=start_time - datetime.timedelta(seconds=period_seconds), 56 | EndTime=end_time, 57 | Dimensions=[ 58 | { 59 | "Name": "Rule", 60 | "Value": waf_rule 61 | }, 62 | { 63 | "Name": "WebACL", 64 | "Value": environ.get(web_acl) 65 | }, 66 | { 67 | "Name": "Region", 68 | "Value": environ.get('AWS_REGION') 69 | } 70 | ] 71 | ) 72 | self.log.debug("[cw_metrics_util: get_cw_metric_statistics] response:\n{}".format(response)) 73 | return response if len(response['Datapoints']) > 0 else None 74 | except Exception as e: 75 | self.log.error("[cw_metrics_util: get_cw_metric_statistics] Failed to get metric %s.", metric_name) 76 | self.log.error(e) 77 | return None 78 | 79 | def add_waf_cw_metric_to_usage_data(self, metric_name, period_seconds, waf_rule, 80 | usage_data, usage_data_field_name, default_value): 81 | """ 82 | Get the CloudWatch metric statistics given a WAF rule and metric name, and 83 | add it to the anonymized usage data collected by the solution. 84 | Parameters: 85 | metric_name: string. The name of the metric. Optional. 86 | period_seconds: integer. The granularity, in seconds, of the returned data points. 87 | waf_rule: string. The name of the WAF rule. 88 | usage_data: JSON. Anonymized customer usage data of the solution 89 | usage_data_field_name: string. The field name in the usage data whose value will be 90 | replaced by the waf cloudwatch metric (if any) 91 | default_value: number. The default value of the field in the usage data 92 | 93 | Returns: JSON. usage data. 94 | """ 95 | self.log.info("[cw_metrics_util: add_waf_cw_metric_to_usage_data] " 96 | + "Get metric %s for waf rule %s." %(metric_name, waf_rule)) 97 | 98 | response = self.get_cw_metric_statistics( 99 | metric_name=metric_name, 100 | period_seconds=period_seconds, 101 | waf_rule=waf_rule 102 | ) 103 | usage_data[usage_data_field_name] = \ 104 | response['Datapoints'][0]['Sum'] if response is not None else default_value 105 | 106 | self.log.info("[cw_metrics_util: add_waf_cw_metric_to_usage_data] " 107 | + "%s - rule %s: %s"%(metric_name, waf_rule, str(usage_data[usage_data_field_name]))) 108 | 109 | return usage_data -------------------------------------------------------------------------------- /source/lib/dynamodb_util.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | # !/bin/python 15 | 16 | from lib.boto3_util import create_resource 17 | 18 | dynamodb_resource = create_resource('dynamodb') 19 | 20 | class DDB(object): 21 | def __init__(self, log, table_name): 22 | self.log = log 23 | self.table_name = table_name 24 | self.table = dynamodb_resource.Table(self.table_name) 25 | 26 | # DDB API call to put an item 27 | def put_item(self, item): 28 | try: 29 | response = self.table.put_item( 30 | Item=item 31 | ) 32 | return response 33 | except Exception as e: 34 | self.log.error(e) 35 | self.log.error("dynamodblib: failed to put item: \n{}".format(item)) 36 | return None 37 | -------------------------------------------------------------------------------- /source/lib/logging_util.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | #!/bin/python 14 | 15 | import logging 16 | from os import environ 17 | 18 | def set_log_level(default_log_level='ERROR'): 19 | default_log_level = logging.getLevelName(default_log_level.upper()) 20 | log_level = str(environ['LOG_LEVEL'].upper()) \ 21 | if 'LOG_LEVEL' in environ else default_log_level 22 | 23 | log = logging.getLogger() 24 | 25 | if log_level not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']: 26 | log_level = 'ERROR' 27 | log.setLevel(log_level) 28 | 29 | return log -------------------------------------------------------------------------------- /source/lib/sns_util.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | #!/bin/python 14 | 15 | from lib.boto3_util import create_client 16 | 17 | class SNS(object): 18 | def __init__(self, log): 19 | self.log = log 20 | self.sns_client = create_client('sns') 21 | 22 | def publish(self, topic_arn, message, subject): 23 | try: 24 | response = self.sns_client.publish( 25 | TopicArn=topic_arn, 26 | Message=message, 27 | Subject=subject 28 | ) 29 | return response 30 | except Exception as e: 31 | self.log.error("[sns_util: publish] failed to send email notification: \nTopic Arn: %s\nMessage: %s", topic_arn, message) 32 | self.log.error(e) 33 | return None 34 | -------------------------------------------------------------------------------- /source/lib/solution_metrics.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). # 5 | # You may not use this file except in compliance with the License. 6 | # A copy of the License is located at # 7 | # # 8 | # http://www.apache.org/licenses/LICENSE-2.0 # 9 | # # 10 | # or in the "license" file accompanying this file. This file is distributed # 11 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express # 12 | # or implied. See the License for the specific language governing permissions# 13 | # and limitations under the License. # 14 | ############################################################################### 15 | 16 | import os 17 | import requests 18 | from json import dumps 19 | from datetime import datetime 20 | import logging 21 | 22 | log = logging.getLogger(__name__) 23 | log.setLevel('INFO') 24 | 25 | def send_metrics(data, 26 | uuid=os.getenv('UUID'), 27 | solution_id=os.getenv('SOLUTION_ID'), 28 | url=os.getenv('METRICS_URL'), 29 | version=os.getenv('Version')): 30 | """Sends anonymized customer metrics to s3 via API gateway owned and 31 | managed by the Solutions Builder team. 32 | 33 | Args: 34 | data - anonymized customer metrics to be sent 35 | uuid - uuid of the solution 36 | solution_id: unique id of the solution 37 | url: url for API Gateway via which data is sent 38 | 39 | Return: status code returned by https post request 40 | """ 41 | try: 42 | metrics_data = { 43 | "Solution": solution_id, 44 | "UUID": uuid, 45 | "TimeStamp": str(datetime.utcnow().isoformat()), 46 | "Data": data, 47 | "Version": version 48 | } 49 | json_data = dumps(metrics_data) 50 | headers = {'content-type': 'application/json'} 51 | response = requests.post(url, data=json_data, headers=headers, timeout=10) 52 | return response 53 | except Exception as e: 54 | log.error("[solution_metrics:send_metrics] Failed to send solution metrics.") 55 | log.error(str(e)) -------------------------------------------------------------------------------- /source/log_parser/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | test/* 4 | */__init__.py 5 | **/__init__.py 6 | backoff/* 7 | bin/* 8 | boto3/* 9 | botocore/* 10 | certifi/* 11 | charset*/* 12 | crhelper* 13 | chardet* 14 | dateutil/* 15 | idna/* 16 | jmespath/* 17 | lib/* 18 | package* 19 | python_* 20 | requests/* 21 | s3transfer/* 22 | six* 23 | tenacity* 24 | tests 25 | urllib3/* 26 | yaml 27 | PyYAML-* 28 | source = 29 | . -------------------------------------------------------------------------------- /source/log_parser/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/log_parser/__init__.py -------------------------------------------------------------------------------- /source/log_parser/add_athena_partitions.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | ############################################################################## 5 | ############################################################################## 6 | 7 | import datetime 8 | from os import getenv 9 | 10 | from lib.boto3_util import create_client 11 | from aws_lambda_powertools import Logger 12 | 13 | logger = Logger( 14 | level=getenv('LOG_LEVEL') 15 | ) 16 | 17 | @logger.inject_lambda_context 18 | def lambda_handler(event, _): 19 | """ 20 | This function adds a new hourly partition to athena table. 21 | It runs every hour, triggered by a CloudWatch event rule. 22 | """ 23 | logger.debug('[add-athena-partition lambda_handler] Start') 24 | try: 25 | # ---------------------------------------------------------- 26 | # Process event 27 | # ---------------------------------------------------------- 28 | athena_client = create_client('athena') 29 | database_name = event['glueAccessLogsDatabase'] 30 | access_log_bucket = event['accessLogBucket'] 31 | waf_log_bucket = event['wafLogBucket'] 32 | athena_work_group = event['athenaWorkGroup'] 33 | 34 | try: 35 | # Add athena partition for cloudfront or alb logs 36 | if len(access_log_bucket) > 0: 37 | execute_athena_query(access_log_bucket, database_name, event['glueAppAccessLogsTable'], athena_client, 38 | athena_work_group) 39 | except Exception as error: 40 | logger.error('[add-athena-partition lambda_handler] App access log Athena query execution failed: %s'%str(error)) 41 | 42 | try: 43 | # Add athena partition for waf logs 44 | if len(waf_log_bucket) > 0: 45 | execute_athena_query(waf_log_bucket, database_name, event['glueWafAccessLogsTable'], athena_client, 46 | athena_work_group) 47 | except Exception as error: 48 | logger.error('[add-athena-partition lambda_handler] WAF access log Athena query execution failed: %s'%str(error)) 49 | 50 | except Exception as error: 51 | logger.error(str(error)) 52 | raise 53 | 54 | logger.debug('[add-athena-partition lambda_handler] End') 55 | 56 | 57 | def build_athena_query(database_name, table_name): 58 | """ 59 | This function dynamically builds the alter table athena query 60 | to add partition to athena table. 61 | 62 | Args: 63 | database_name: string. The Athena/Glue database name 64 | table_name: string. The Athena/Glue table name 65 | 66 | Returns: 67 | string. Athena query string 68 | """ 69 | 70 | current_timestamp = datetime.datetime.utcnow() 71 | year = current_timestamp.year 72 | month = current_timestamp.month 73 | day = current_timestamp.day 74 | hour = current_timestamp.hour 75 | 76 | query_string = "ALTER TABLE " \ 77 | + database_name + "." + table_name \ 78 | + "\nADD IF NOT EXISTS\n" \ 79 | "PARTITION (\n" \ 80 | "\tyear = " + str(year) + ",\n" \ 81 | "\tmonth = " + str(month).zfill(2) + ",\n" \ 82 | "\tday = " + str(day).zfill(2) + ",\n" \ 83 | "\thour = " + str(hour).zfill(2) + ");" 84 | 85 | logger.debug( 86 | "[build_athena_query] Query string:\n%s\n" 87 | %query_string) 88 | 89 | return query_string 90 | 91 | 92 | def execute_athena_query(log_bucket, database_name, table_name, athena_client, athena_work_group): 93 | """ 94 | This function executes the alter table athena query to 95 | add partition to athena table. 96 | 97 | Args: 98 | log_bucket: s3 bucket for logs(cloudfront, alb or waf logs) 99 | database_name: string. The Athena/Glue database name 100 | table_name: string. The Athena/Glue table name 101 | athena_client: object. Athena client object 102 | 103 | Returns: 104 | None 105 | """ 106 | 107 | s3_output = "s3://%s/athena_results/"%log_bucket 108 | 109 | query_string = build_athena_query(database_name, table_name) 110 | 111 | logger.info("[execute_athena_query] Query string:\n%s \ 112 | \nAthena S3 Output Bucket: %s\n" % (query_string, s3_output)) 113 | 114 | response = athena_client.start_query_execution( 115 | QueryString=query_string, 116 | QueryExecutionContext={'Database': database_name}, 117 | ResultConfiguration={'OutputLocation': s3_output, 118 | 'EncryptionConfiguration': { 119 | 'EncryptionOption': 'SSE_S3' 120 | } 121 | }, 122 | WorkGroup=athena_work_group 123 | ) 124 | 125 | logger.info("[execute_athena_query] Query execution response:\n%s" % response) 126 | -------------------------------------------------------------------------------- /source/log_parser/athena_log_parser.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | import csv 15 | import datetime 16 | from os import environ, remove 17 | from build_athena_queries import build_athena_query_for_app_access_logs, \ 18 | build_athena_query_for_waf_logs 19 | from lib.boto3_util import create_client 20 | from lib.s3_util import S3 21 | from lambda_log_parser import LambdaLogParser 22 | 23 | 24 | class AthenaLogParser(object): 25 | """ 26 | This class includes functions to process WAF and App access logs using Athena parser 27 | """ 28 | 29 | def __init__(self, log): 30 | self.log = log 31 | self.s3_util = S3(log) 32 | self.lambda_log_parser = LambdaLogParser(log) 33 | 34 | 35 | def process_athena_scheduler_event(self, event): 36 | self.log.debug("[athena_log_parser: process_athena_scheduler_event] Start") 37 | 38 | log_type = str(environ['LOG_TYPE'].upper()) 39 | 40 | # Execute athena query for CloudFront or ALB logs 41 | if event['resourceType'] == 'LambdaAthenaAppLogParser' \ 42 | and (log_type == 'CLOUDFRONT' or log_type == 'ALB'): 43 | self.execute_athena_query(log_type, event) 44 | 45 | # Execute athena query for WAF logs 46 | if event['resourceType'] == 'LambdaAthenaWAFLogParser': 47 | self.execute_athena_query('WAF', event) 48 | 49 | self.log.debug("[athena_log_parser: process_athena_scheduler_event] End") 50 | 51 | 52 | def execute_athena_query(self, log_type, event): 53 | self.log.debug("[athena_log_parser: execute_athena_query] Start") 54 | 55 | athena_client = create_client('athena') 56 | s3_output = "s3://%s/athena_results/" % event['accessLogBucket'] 57 | database_name = event['glueAccessLogsDatabase'] 58 | 59 | # Dynamically build query string using partition 60 | # for CloudFront or ALB logs 61 | if log_type == 'CLOUDFRONT' or log_type == 'ALB': 62 | query_string = build_athena_query_for_app_access_logs( 63 | self.log, 64 | log_type, 65 | event['glueAccessLogsDatabase'], 66 | event['glueAppAccessLogsTable'], 67 | datetime.datetime.utcnow(), 68 | int(environ['WAF_BLOCK_PERIOD']), 69 | int(environ['ERROR_THRESHOLD']) 70 | ) 71 | else: # Dynamically build query string using partition for WAF logs 72 | query_string = build_athena_query_for_waf_logs( 73 | self.log, 74 | event['glueAccessLogsDatabase'], 75 | event['glueWafAccessLogsTable'], 76 | datetime.datetime.utcnow(), 77 | int(environ['WAF_BLOCK_PERIOD']), 78 | int(environ['REQUEST_THRESHOLD']), 79 | environ['REQUEST_THRESHOLD_BY_COUNTRY'], 80 | environ['HTTP_FLOOD_ATHENA_GROUP_BY'], 81 | int(environ['ATHENA_QUERY_RUN_SCHEDULE']) 82 | ) 83 | 84 | response = athena_client.start_query_execution( 85 | QueryString=query_string, 86 | QueryExecutionContext={'Database': database_name}, 87 | ResultConfiguration={ 88 | 'OutputLocation': s3_output, 89 | 'EncryptionConfiguration': { 90 | 'EncryptionOption': 'SSE_S3' 91 | } 92 | }, 93 | WorkGroup=event['athenaWorkGroup'] 94 | ) 95 | 96 | self.log.info("[athena_log_parser: execute_athena_query] Query Execution Response: {}".format(response)) 97 | self.log.info("[athena_log_parser: execute_athena_query] End") 98 | 99 | 100 | def read_athena_result_file(self, local_file_path): 101 | self.log.debug("[athena_log_parser: read_athena_result_file] Start") 102 | 103 | outstanding_requesters = { 104 | 'general': {}, 105 | 'uriList': {} 106 | } 107 | utc_now_timestamp_str = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S %Z%z") 108 | with open(local_file_path, 'r') as csvfile: 109 | reader = csv.DictReader(csvfile) 110 | for row in reader: 111 | # max_counter_per_min is set as 1 just to reuse lambda log parser data structure 112 | # and reuse update_ip_set. 113 | outstanding_requesters['general'][row['client_ip']] = { 114 | "max_counter_per_min": row['max_counter_per_min'], 115 | "updated_at": utc_now_timestamp_str 116 | } 117 | remove(local_file_path) 118 | 119 | self.log.debug("[athena_log_parser: read_athena_result_file] local_file_path: %s", 120 | local_file_path) 121 | self.log.debug("[athena_log_parser: read_athena_result_file] End") 122 | 123 | return outstanding_requesters 124 | 125 | 126 | def process_athena_result(self, bucket_name, key_name, ip_set_type): 127 | self.log.debug("[athena_log_parser: process_athena_result] Start") 128 | 129 | try: 130 | # -------------------------------------------------------------------------------------------------------------- 131 | self.log.info("[athena_log_parser: process_athena_result] Download file from S3") 132 | # -------------------------------------------------------------------------------------------------------------- 133 | local_file_path = '/tmp/' + key_name.split('/')[-1] #NOSONAR tmp use for an insensitive workspace 134 | self.s3_util.download_file_from_s3(bucket_name, key_name, local_file_path) 135 | 136 | # -------------------------------------------------------------------------------------------------------------- 137 | self.log.info("[athena_log_parser: process_athena_result] Read file content") 138 | # -------------------------------------------------------------------------------------------------------------- 139 | outstanding_requesters = self.read_athena_result_file(local_file_path) 140 | 141 | # -------------------------------------------------------------------------------------------------------------- 142 | self.log.info("[athena_log_parser: process_athena_result] Update WAF IP Sets") 143 | # -------------------------------------------------------------------------------------------------------------- 144 | self.lambda_log_parser.update_ip_set(ip_set_type, outstanding_requesters) 145 | 146 | except Exception as e: 147 | self.log.error("[athena_log_parser: process_athena_result] Error to read input file") 148 | self.log.error(e) 149 | 150 | self.log.debug("[athena_log_parser: process_athena_result] End") 151 | -------------------------------------------------------------------------------- /source/log_parser/partition_s3_logs.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import re 5 | from os import environ, getenv 6 | from lib.boto3_util import create_client 7 | from aws_lambda_powertools import Logger 8 | 9 | logger = Logger( 10 | level=getenv('LOG_LEVEL') 11 | ) 12 | 13 | @logger.inject_lambda_context 14 | def lambda_handler(event, _): 15 | """ 16 | This function is triggered by S3 event to move log files 17 | (upon their arrival in s3) from their original location 18 | to a partitioned folder structure created per timestamps 19 | in file names, hence allowing the usage of partitioning 20 | within AWS Athena. 21 | 22 | Sample partitioned folder structure: 23 | AWSLogs-Partitioned/year=2020/month=04/day=09/hour=23/ 24 | 25 | """ 26 | logger.debug('[partition_s3_logs lambda_handler] Start') 27 | try: 28 | # ---------------------------------------------------------- 29 | # Process event 30 | # ---------------------------------------------------------- 31 | 32 | 33 | keep_original_data = str(environ['KEEP_ORIGINAL_DATA'].upper()) 34 | endpoint = str(environ['ENDPOINT'].upper()) 35 | logger.info("\n[partition_s3_logs lambda_handler] KEEP ORIGINAL DATA: %s; End POINT: %s." 36 | %(keep_original_data, endpoint)) 37 | 38 | s3 = create_client('s3') 39 | 40 | count = 0 41 | 42 | # Iterate through all records in the event 43 | for record in event['Records']: 44 | # Get S3 bucket 45 | bucket = record['s3']['bucket']['name'] 46 | 47 | # Get source S3 object key 48 | key = record['s3']['object']['key'] 49 | 50 | # Get file name, which should be the last one in the string 51 | filename = "" 52 | number = len(key.split('/')) 53 | if number >= 1: 54 | number = number - 1 55 | filename = key.split('/')[number] 56 | 57 | if endpoint == 'CLOUDFRONT': 58 | dest = parse_cloudfront_logs(key, filename) 59 | else: # ALB endpoint 60 | dest = parse_alb_logs(key, filename) 61 | 62 | source_path = bucket + '/' + key 63 | dest_path = bucket + '/' + dest 64 | 65 | # Copy S3 object to destination 66 | s3.copy_object(Bucket=bucket, Key=dest, CopySource=source_path) 67 | 68 | logger.info("\n[partition_s3_logs lambda_handler] Copied file %s to destination %s"%(source_path, dest_path)) 69 | 70 | # Only delete source S3 object from its original folder if keeping original data is no 71 | if keep_original_data == 'NO': 72 | s3.delete_object(Bucket=bucket, Key=key) 73 | logger.info("\n[partition_s3_logs lambda_handler] Removed file %s"%source_path) 74 | 75 | count = count + 1 76 | 77 | logger.info("\n[partition_s3_logs lambda_handler] Successfully partitioned %s file(s)."%(str(count))) 78 | 79 | except Exception as error: 80 | logger.error(str(error)) 81 | raise 82 | 83 | logger.debug('[partition_s3_logs lambda_handler] End') 84 | 85 | 86 | def parse_cloudfront_logs(key, filename): 87 | # Get year, month, day and hour 88 | time_stamp = re.search('(\\d{4})-(\\d{2})-(\\d{2})-(\\d{2})', key) 89 | year, month, day, hour = time_stamp.group(0).split('-') 90 | 91 | # Create destination path 92 | dest = 'AWSLogs-Partitioned/year={}/month={}/day={}/hour={}/{}' \ 93 | .format(year, month, day, hour, filename) 94 | 95 | return dest 96 | 97 | 98 | def parse_alb_logs(key, filename): 99 | # Get year, month and day 100 | time_stamp = re.search('(\\d{4})/(\\d{2})/(\\d{2})', key) 101 | year, month, day = time_stamp.group(0).split('/') 102 | 103 | # Get hour 104 | time_stamp = re.search('(\\d{8})T(\\d{2})', filename) 105 | hour = time_stamp.group(0).split('T')[1] 106 | 107 | # Create destination path 108 | dest = 'AWSLogs-Partitioned/year={}/month={}/day={}/hour={}/{}' \ 109 | .format(year, month, day, hour, filename) 110 | 111 | return dest 112 | -------------------------------------------------------------------------------- /source/log_parser/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "log_parser" 3 | package-mode = false 4 | 5 | [tool.poetry.dependencies] 6 | python = "~3.12" 7 | requests = "^2.32.3" 8 | backoff = "^2.2.1" 9 | pyparsing = "^3.1.4" 10 | aws-lambda-powertools = "~3.2.0" 11 | 12 | 13 | [tool.poetry.group.dev.dependencies] 14 | moto = "^4.1.4" 15 | pytest = "^7.2.2" 16 | pytest-mock = "^3.12.0" 17 | pytest-runner = "^6.0.0" 18 | pytest-cov = "^4.0.0" 19 | pytest-env = "^0.8.1" 20 | boto3 = "^1.35.30" 21 | freezegun = "^1.2.2" 22 | 23 | [build-system] 24 | requires = ["poetry-core"] 25 | build-backend = "poetry.core.masonry.api" 26 | -------------------------------------------------------------------------------- /source/log_parser/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/log_parser/test/__init__.py -------------------------------------------------------------------------------- /source/log_parser/test/test_add_athena_partitions.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from types import SimpleNamespace 5 | 6 | from add_athena_partitions import lambda_handler 7 | 8 | context = SimpleNamespace(**{ 9 | 'function_name': 'foo', 10 | 'memory_limit_in_mb': '512', 11 | 'invoked_function_arn': ':::invoked_function_arn', 12 | 'log_group_name': 'log_group_name', 13 | 'log_stream_name': 'log_stream_name', 14 | 'aws_request_id': 'baz' 15 | }) 16 | 17 | def test_add_athena_partitions(athena_partitions_test_event_setup): 18 | try: 19 | event = athena_partitions_test_event_setup 20 | result = False 21 | lambda_handler(event, context) 22 | result = True 23 | except Exception: 24 | raise 25 | assert result == True 26 | 27 | 28 | def test_add_athena_partitions(athena_partitions_non_existent_work_group_test_event_setup): 29 | try: 30 | event = athena_partitions_non_existent_work_group_test_event_setup 31 | result = False 32 | lambda_handler(event, context) 33 | result = True 34 | except Exception: 35 | assert result == False 36 | -------------------------------------------------------------------------------- /source/log_parser/test/test_build_athena_queries.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | import datetime 15 | import logging 16 | import build_athena_queries, add_athena_partitions 17 | from datetime import datetime 18 | from freezegun import freeze_time 19 | 20 | log_level = 'DEBUG' 21 | logging.getLogger().setLevel(log_level) 22 | log = logging.getLogger('test_build_athena_queries') 23 | database_name = 'testdb' 24 | table_name = 'testtable' 25 | end_timestamp = datetime.strptime('May 7 2020 1:33PM', '%b %d %Y %I:%M%p') 26 | waf_block_period = 240 27 | error_threshold = 2000 28 | request_threshold = 50 29 | request_threshold_by_country = '{"TR":30,"CN":100,"SE":150}' 30 | no_request_threshold_by_country = '' 31 | group_by_country = 'country' 32 | group_by_uri = 'uri' 33 | group_by_country_uri = 'country and uri' 34 | no_group_by = 'none' 35 | athena_query_run_schedule = 5 36 | cloudfront_log_type = 'CLOUDFRONT' 37 | alb_log_type = 'ALB' 38 | waf_log_type = 'WAF' 39 | log_bucket = 'LogBucket' 40 | 41 | 42 | def test_build_athena_queries_for_cloudfront_logs(): 43 | query_string = build_athena_queries.build_athena_query_for_app_access_logs( 44 | log, cloudfront_log_type, database_name, table_name, 45 | end_timestamp, waf_block_period, error_threshold) 46 | 47 | with open('./test/test_data/cloudfront_logs_query.txt', 'r') as file: 48 | cloudfront_logs_query = file.read() 49 | assert type(query_string) is str 50 | assert query_string == cloudfront_logs_query 51 | 52 | 53 | def test_build_athena_queries_for_alb_logs(): 54 | query_string = build_athena_queries.build_athena_query_for_app_access_logs( 55 | log, alb_log_type, database_name, table_name, 56 | end_timestamp, waf_block_period, error_threshold) 57 | 58 | with open('./test/test_data/alb_logs_query.txt', 'r') as file: 59 | alb_logs_query = file.read() 60 | assert type(query_string) is str 61 | assert query_string == alb_logs_query 62 | 63 | 64 | def test_build_athena_queries_for_waf_logs_one(): 65 | # test original waf log query one - no group by; no threshold by country 66 | query_string = build_athena_queries.build_athena_query_for_waf_logs( 67 | log, database_name, table_name,end_timestamp, waf_block_period, 68 | request_threshold, no_request_threshold_by_country, no_group_by, 69 | athena_query_run_schedule 70 | ) 71 | 72 | with open('./test/test_data/waf_logs_query_1.txt', 'r') as file: 73 | waf_logs_query = file.read() 74 | assert type(query_string) is str 75 | assert query_string == waf_logs_query 76 | 77 | def test_build_athena_queries_for_waf_logs_two(): 78 | # test waf log query two - group by country; no threshold by country 79 | query_string = build_athena_queries.build_athena_query_for_waf_logs( 80 | log, database_name, table_name,end_timestamp, waf_block_period, 81 | request_threshold, no_request_threshold_by_country, group_by_country, 82 | athena_query_run_schedule 83 | ) 84 | 85 | with open('./test/test_data/waf_logs_query_2.txt', 'r') as file: 86 | waf_logs_query = file.read() 87 | assert type(query_string) is str 88 | assert query_string == waf_logs_query 89 | 90 | def test_build_athena_queries_for_waf_logs_three(): 91 | # test waf log query three - group by uri; no threshold by country 92 | query_string = build_athena_queries.build_athena_query_for_waf_logs( 93 | log, database_name, table_name,end_timestamp, waf_block_period, 94 | request_threshold, no_request_threshold_by_country, group_by_uri, 95 | athena_query_run_schedule 96 | ) 97 | 98 | with open('./test/test_data/waf_logs_query_3.txt', 'r') as file: 99 | waf_logs_query = file.read() 100 | assert type(query_string) is str 101 | assert query_string == waf_logs_query 102 | 103 | def test_build_athena_queries_for_waf_logs_four(): 104 | # test waf log query four - group by country and uri; no threshold by country 105 | query_string = build_athena_queries.build_athena_query_for_waf_logs( 106 | log, database_name, table_name,end_timestamp, waf_block_period, 107 | request_threshold, no_request_threshold_by_country, group_by_country_uri, 108 | athena_query_run_schedule 109 | ) 110 | 111 | with open('./test/test_data/waf_logs_query_4.txt', 'r') as file: 112 | waf_logs_query = file.read() 113 | assert type(query_string) is str 114 | assert query_string == waf_logs_query 115 | 116 | def test_build_athena_queries_for_waf_logs_five(): 117 | # test waf log query five - no group by; has threshold by country 118 | query_string = build_athena_queries.build_athena_query_for_waf_logs( 119 | log, database_name, table_name,end_timestamp, waf_block_period, 120 | request_threshold, request_threshold_by_country, no_group_by, 121 | athena_query_run_schedule 122 | ) 123 | 124 | with open('./test/test_data/waf_logs_query_5.txt', 'r') as file: 125 | waf_logs_query = file.read() 126 | assert type(query_string) is str 127 | assert query_string == waf_logs_query 128 | 129 | def test_build_athena_queries_for_waf_logs_six(): 130 | # test waf log query six - group by country; has threshold by country 131 | query_string = build_athena_queries.build_athena_query_for_waf_logs( 132 | log, database_name, table_name,end_timestamp, waf_block_period, 133 | request_threshold, request_threshold_by_country, group_by_country, 134 | athena_query_run_schedule 135 | ) 136 | 137 | with open('./test/test_data/waf_logs_query_5.txt', 'r') as file: 138 | waf_logs_query = file.read() 139 | assert type(query_string) is str 140 | assert query_string == waf_logs_query 141 | 142 | def test_build_athena_queries_for_waf_logs_seven(): 143 | # test waf log query seven - group by uri; has threshold by country 144 | query_string = build_athena_queries.build_athena_query_for_waf_logs( 145 | log, database_name, table_name,end_timestamp, waf_block_period, 146 | request_threshold, request_threshold_by_country, group_by_uri, 147 | athena_query_run_schedule 148 | ) 149 | 150 | with open('./test/test_data/waf_logs_query_6.txt', 'r') as file: 151 | waf_logs_query = file.read() 152 | assert type(query_string) is str 153 | assert query_string == waf_logs_query 154 | 155 | def test_build_athena_queries_for_waf_logs_eight(): 156 | # test waf log query eight - group by country and uri; has threshold by country 157 | query_string = build_athena_queries.build_athena_query_for_waf_logs( 158 | log, database_name, table_name,end_timestamp, waf_block_period, 159 | request_threshold, request_threshold_by_country, group_by_country_uri, 160 | athena_query_run_schedule 161 | ) 162 | 163 | with open('./test/test_data/waf_logs_query_6.txt', 'r') as file: 164 | waf_logs_query = file.read() 165 | assert type(query_string) is str 166 | assert query_string == waf_logs_query 167 | 168 | @freeze_time("2020-05-08 02:21:34", tz_offset=-4) 169 | def test_add_athena_partitions_build_query_string(): 170 | query_string = add_athena_partitions.build_athena_query(database_name, table_name) 171 | 172 | with open('./test/test_data/athena_partitions_query.txt', 'r') as file: 173 | athena_partitions_query = file.read() 174 | assert type(query_string) is str 175 | assert query_string == athena_partitions_query 176 | -------------------------------------------------------------------------------- /source/log_parser/test/test_data/E3HXCM7PFRG6HT.2023-04-24-21.d740d76bCloudFront.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/log_parser/test/test_data/E3HXCM7PFRG6HT.2023-04-24-21.d740d76bCloudFront.gz -------------------------------------------------------------------------------- /source/log_parser/test/test_data/XXXXXXXXXXXX_elasticloadbalancing_us-east-1_app.ApplicationLoadBalancer.fa87e1db7badc175_20230424T2110Z_X.X.X.X_4c8scnzy.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/log_parser/test/test_data/XXXXXXXXXXXX_elasticloadbalancing_us-east-1_app.ApplicationLoadBalancer.fa87e1db7badc175_20230424T2110Z_X.X.X.X_4c8scnzy.log.gz -------------------------------------------------------------------------------- /source/log_parser/test/test_data/alb_logs_query.txt: -------------------------------------------------------------------------------- 1 | SELECT 2 | client_ip, 3 | MAX_BY(counter, counter) as max_counter_per_min 4 | FROM ( 5 | WITH logs_with_concat_data AS ( 6 | SELECT 7 | client_ip, 8 | target_status_code AS status, 9 | parse_datetime(time, 'yyyy-MM-dd''T''HH:mm:ss.SSSSSS''Z') AS datetime 10 | FROM 11 | testdb.testtable 12 | WHERE year = 2020 13 | AND month = 05 14 | AND day = 07 15 | AND hour between 09 and 13 16 | ) 17 | SELECT 18 | client_ip, 19 | COUNT(*) as counter 20 | FROM 21 | logs_with_concat_data 22 | WHERE 23 | datetime > TIMESTAMP '2020-05-07 09:33:00' 24 | AND status = ANY (VALUES '400', '401', '403', '404', '405') 25 | GROUP BY 26 | client_ip, 27 | date_trunc('minute', datetime) 28 | HAVING 29 | COUNT(*) >= 2000 30 | ) GROUP BY 31 | client_ip 32 | ORDER BY 33 | max_counter_per_min DESC 34 | LIMIT 10000; -------------------------------------------------------------------------------- /source/log_parser/test/test_data/athena_partitions_query.txt: -------------------------------------------------------------------------------- 1 | ALTER TABLE testdb.testtable 2 | ADD IF NOT EXISTS 3 | PARTITION ( 4 | year = 2020, 5 | month = 05, 6 | day = 08, 7 | hour = 02); -------------------------------------------------------------------------------- /source/log_parser/test/test_data/cf-access-log-sample.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/log_parser/test/test_data/cf-access-log-sample.gz -------------------------------------------------------------------------------- /source/log_parser/test/test_data/cloudfront_logs_query.txt: -------------------------------------------------------------------------------- 1 | SELECT 2 | client_ip, 3 | MAX_BY(counter, counter) as max_counter_per_min 4 | FROM ( 5 | WITH logs_with_concat_data AS ( 6 | SELECT 7 | requestip as client_ip, 8 | cast(status as varchar) as status, 9 | parse_datetime( concat( concat( format_datetime(date, 'yyyy-MM-dd'), '-' ), time ), 'yyyy-MM-dd-HH:mm:ss') AS datetime 10 | FROM 11 | testdb.testtable 12 | WHERE year = 2020 13 | AND month = 05 14 | AND day = 07 15 | AND hour between 09 and 13 16 | ) 17 | SELECT 18 | client_ip, 19 | COUNT(*) as counter 20 | FROM 21 | logs_with_concat_data 22 | WHERE 23 | datetime > TIMESTAMP '2020-05-07 09:33:00' 24 | AND status = ANY (VALUES '400', '401', '403', '404', '405') 25 | GROUP BY 26 | client_ip, 27 | date_trunc('minute', datetime) 28 | HAVING 29 | COUNT(*) >= 2000 30 | ) GROUP BY 31 | client_ip 32 | ORDER BY 33 | max_counter_per_min DESC 34 | LIMIT 10000; -------------------------------------------------------------------------------- /source/log_parser/test/test_data/test_athena_query_result.csv: -------------------------------------------------------------------------------- 1 | "client_ip","max_counter_per_min" 2 | "10.x.x.x","2798" -------------------------------------------------------------------------------- /source/log_parser/test/test_data/test_waf_log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/log_parser/test/test_data/test_waf_log.gz -------------------------------------------------------------------------------- /source/log_parser/test/test_data/waf-stack-app_log_out.json: -------------------------------------------------------------------------------- 1 | {"general": {"x.x.0.0": {"max_counter_per_min": 100, "updated_at": "2023-04-21 22:57:59 UTC+0000"}}, "uriList": {}} -------------------------------------------------------------------------------- /source/log_parser/test/test_data/waf_logs_query_1.txt: -------------------------------------------------------------------------------- 1 | SELECT 2 | client_ip, 3 | MAX_BY(counter, counter) as max_counter_per_min 4 | FROM ( 5 | WITH logs_with_concat_data AS ( 6 | SELECT 7 | httprequest.clientip as client_ip, 8 | from_unixtime(timestamp/1000) as datetime 9 | FROM 10 | testdb.testtable 11 | WHERE year = 2020 12 | AND month = 05 13 | AND day = 07 14 | AND hour between 09 and 13 15 | ) 16 | SELECT 17 | client_ip, 18 | COUNT(*) as counter 19 | FROM 20 | logs_with_concat_data 21 | WHERE 22 | datetime > TIMESTAMP '2020-05-07 09:33:00' 23 | GROUP BY 24 | client_ip, 25 | date_trunc('minute', datetime) 26 | HAVING 27 | COUNT(*) >= 10.0 28 | ) GROUP BY 29 | client_ip 30 | ORDER BY 31 | max_counter_per_min DESC 32 | LIMIT 10000; -------------------------------------------------------------------------------- /source/log_parser/test/test_data/waf_logs_query_2.txt: -------------------------------------------------------------------------------- 1 | SELECT 2 | client_ip, country, 3 | MAX_BY(counter, counter) as max_counter_per_min 4 | FROM ( 5 | WITH logs_with_concat_data AS ( 6 | SELECT 7 | httprequest.clientip as client_ip,httprequest.country as country, 8 | from_unixtime(timestamp/1000) as datetime 9 | FROM 10 | testdb.testtable 11 | WHERE year = 2020 12 | AND month = 05 13 | AND day = 07 14 | AND hour between 09 and 13 15 | ) 16 | SELECT 17 | client_ip, country, 18 | COUNT(*) as counter 19 | FROM 20 | logs_with_concat_data 21 | WHERE 22 | datetime > TIMESTAMP '2020-05-07 09:33:00' 23 | GROUP BY 24 | client_ip, country, 25 | date_trunc('minute', datetime) 26 | HAVING 27 | COUNT(*) >= 10.0 28 | ) GROUP BY 29 | client_ip, country 30 | ORDER BY 31 | max_counter_per_min DESC 32 | LIMIT 10000; -------------------------------------------------------------------------------- /source/log_parser/test/test_data/waf_logs_query_3.txt: -------------------------------------------------------------------------------- 1 | SELECT 2 | client_ip, uri, 3 | MAX_BY(counter, counter) as max_counter_per_min 4 | FROM ( 5 | WITH logs_with_concat_data AS ( 6 | SELECT 7 | httprequest.clientip as client_ip,httprequest.uri as uri, 8 | from_unixtime(timestamp/1000) as datetime 9 | FROM 10 | testdb.testtable 11 | WHERE year = 2020 12 | AND month = 05 13 | AND day = 07 14 | AND hour between 09 and 13 15 | ) 16 | SELECT 17 | client_ip, uri, 18 | COUNT(*) as counter 19 | FROM 20 | logs_with_concat_data 21 | WHERE 22 | datetime > TIMESTAMP '2020-05-07 09:33:00' 23 | GROUP BY 24 | client_ip, uri, 25 | date_trunc('minute', datetime) 26 | HAVING 27 | COUNT(*) >= 10.0 28 | ) GROUP BY 29 | client_ip, uri 30 | ORDER BY 31 | max_counter_per_min DESC 32 | LIMIT 10000; -------------------------------------------------------------------------------- /source/log_parser/test/test_data/waf_logs_query_4.txt: -------------------------------------------------------------------------------- 1 | SELECT 2 | client_ip, country, uri, 3 | MAX_BY(counter, counter) as max_counter_per_min 4 | FROM ( 5 | WITH logs_with_concat_data AS ( 6 | SELECT 7 | httprequest.clientip as client_ip,httprequest.country as country, httprequest.uri as uri, 8 | from_unixtime(timestamp/1000) as datetime 9 | FROM 10 | testdb.testtable 11 | WHERE year = 2020 12 | AND month = 05 13 | AND day = 07 14 | AND hour between 09 and 13 15 | ) 16 | SELECT 17 | client_ip, country, uri, 18 | COUNT(*) as counter 19 | FROM 20 | logs_with_concat_data 21 | WHERE 22 | datetime > TIMESTAMP '2020-05-07 09:33:00' 23 | GROUP BY 24 | client_ip, country, uri, 25 | date_trunc('minute', datetime) 26 | HAVING 27 | COUNT(*) >= 10.0 28 | ) GROUP BY 29 | client_ip, country, uri 30 | ORDER BY 31 | max_counter_per_min DESC 32 | LIMIT 10000; -------------------------------------------------------------------------------- /source/log_parser/test/test_data/waf_logs_query_5.txt: -------------------------------------------------------------------------------- 1 | SELECT 2 | client_ip, country, 3 | MAX_BY(counter, counter) as max_counter_per_min 4 | FROM ( 5 | WITH logs_with_concat_data AS ( 6 | SELECT 7 | httprequest.clientip as client_ip,httprequest.country as country, 8 | from_unixtime(timestamp/1000) as datetime 9 | FROM 10 | testdb.testtable 11 | WHERE year = 2020 12 | AND month = 05 13 | AND day = 07 14 | AND hour between 09 and 13 15 | ) 16 | SELECT 17 | client_ip, country, 18 | COUNT(*) as counter 19 | FROM 20 | logs_with_concat_data 21 | WHERE 22 | datetime > TIMESTAMP '2020-05-07 09:33:00' 23 | GROUP BY 24 | client_ip, country, 25 | date_trunc('minute', datetime) 26 | HAVING 27 | (COUNT(*) >= 6.0 AND country = 'TR') OR 28 | (COUNT(*) >= 20.0 AND country = 'CN') OR 29 | (COUNT(*) >= 30.0 AND country = 'SE') OR 30 | (COUNT(*) >= 10.0 AND country NOT IN ('TR','CN','SE')) 31 | ) GROUP BY 32 | client_ip, country 33 | ORDER BY 34 | max_counter_per_min DESC 35 | LIMIT 10000; -------------------------------------------------------------------------------- /source/log_parser/test/test_data/waf_logs_query_6.txt: -------------------------------------------------------------------------------- 1 | SELECT 2 | client_ip, country, uri, 3 | MAX_BY(counter, counter) as max_counter_per_min 4 | FROM ( 5 | WITH logs_with_concat_data AS ( 6 | SELECT 7 | httprequest.clientip as client_ip,httprequest.country as country, httprequest.uri as uri, 8 | from_unixtime(timestamp/1000) as datetime 9 | FROM 10 | testdb.testtable 11 | WHERE year = 2020 12 | AND month = 05 13 | AND day = 07 14 | AND hour between 09 and 13 15 | ) 16 | SELECT 17 | client_ip, country, uri, 18 | COUNT(*) as counter 19 | FROM 20 | logs_with_concat_data 21 | WHERE 22 | datetime > TIMESTAMP '2020-05-07 09:33:00' 23 | GROUP BY 24 | client_ip, country, uri, 25 | date_trunc('minute', datetime) 26 | HAVING 27 | (COUNT(*) >= 6.0 AND country = 'TR') OR 28 | (COUNT(*) >= 20.0 AND country = 'CN') OR 29 | (COUNT(*) >= 30.0 AND country = 'SE') OR 30 | (COUNT(*) >= 10.0 AND country NOT IN ('TR','CN','SE')) 31 | ) GROUP BY 32 | client_ip, country, uri 33 | ORDER BY 34 | max_counter_per_min DESC 35 | LIMIT 10000; -------------------------------------------------------------------------------- /source/log_parser/test/test_data/waf_stack-app_log_conf.json: -------------------------------------------------------------------------------- 1 | {"general": {"errorThreshold": 5, "blockPeriod": 240, "errorCodes": ["400", "401", "403", "404", "405"]}, "uriList": {}} -------------------------------------------------------------------------------- /source/log_parser/test/test_data/waf_stack-waf_log_conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "requestThreshold": 20, 4 | "blockPeriod": 240, 5 | "ignoredSufixes": [".css", ".js", ".jpeg"] 6 | }, 7 | "uriList": { 8 | "/socket.io/": { 9 | "requestThreshold": 5, 10 | "blockPeriod": 100 11 | }, 12 | "/assets/public/images/products/green_smoothie.jpg": { 13 | "requestThreshold": 5, 14 | "blockPeriod": 100 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /source/log_parser/test/test_data/waf_stack-waf_log_out.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "x.0.0.0": { 4 | "max_counter_per_min": 715, 5 | "updated_at": "2023-04-24 22:16:11 UTC+0000" 6 | }, 7 | "x.x.0.0": { 8 | "max_counter_per_min": 8571, 9 | "updated_at": "2023-04-24 22:16:11 UTC+0000" 10 | } 11 | }, 12 | "uriList": {} 13 | } -------------------------------------------------------------------------------- /source/log_parser/test/test_log_parser.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from os import environ 5 | from types import SimpleNamespace 6 | 7 | from log_parser import log_parser 8 | 9 | context = SimpleNamespace(**{ 10 | 'function_name': 'foo', 11 | 'memory_limit_in_mb': '512', 12 | 'invoked_function_arn': ':::invoked_function_arn', 13 | 'log_group_name': 'log_group_name', 14 | 'log_stream_name': 'log_stream_name', 15 | 'aws_request_id': 'baz' 16 | }) 17 | 18 | 19 | UNDEFINED_HANDLER_MESSAGE = "[lambda_handler] undefined handler for this type of event" 20 | ATHENA_LOG_PARSER_PROCESSED_MESSAGE = "[lambda_handler] Athena scheduler event processed." 21 | ATHENA_APP_LOG_QUERY_RESULT_PROCESSED_MESSAGE = "[lambda_handler] Athena app log query result processed." 22 | ATHENA_WAF_LOG_QUERY_RESULT_PROCESSED_MESSAGE = "[lambda_handler] Athena AWS WAF log query result processed." 23 | APP_LOG_LAMBDA_PARSER_PROCESSED_MESSAGE = "[lambda_handler] App access log file processed." 24 | WAF_LOG_LAMBDA_PARSER_PROCESSED_MESSAGE = "[lambda_handler] AWS WAF access log file processed." 25 | TYPE_ERROR_MESSAGE = "TypeError: string indices must be integers" 26 | 27 | 28 | def test_undefined_handler_event(): 29 | event = {"test": "value"} 30 | result = {"message": UNDEFINED_HANDLER_MESSAGE} 31 | assert result == log_parser.lambda_handler(event, context) 32 | 33 | 34 | def test_undefined_handler_records(cloudfront_log_lambda_parser_test_event_setup): 35 | event = cloudfront_log_lambda_parser_test_event_setup 36 | UNDEFINED_HANDLER_MESSAGE = "[lambda_handler] undefined handler for bucket %s" % environ["APP_ACCESS_LOG_BUCKET"] 37 | environ.pop('APP_ACCESS_LOG_BUCKET') 38 | result = {"message": UNDEFINED_HANDLER_MESSAGE} 39 | assert result == log_parser.lambda_handler(event, context) 40 | 41 | 42 | def test_cloudfront_log_athena_parser(app_log_athena_parser_test_event_setup): 43 | environ['LOG_TYPE'] = "CLOUDFRONT" 44 | event = app_log_athena_parser_test_event_setup 45 | result = {"message": ATHENA_LOG_PARSER_PROCESSED_MESSAGE} 46 | assert result == log_parser.lambda_handler(event, context) 47 | environ.pop('LOG_TYPE') 48 | 49 | 50 | def test_alb_log_athena_parser(app_log_athena_parser_test_event_setup): 51 | environ['LOG_TYPE'] = "ALB" 52 | event = app_log_athena_parser_test_event_setup 53 | result = {"message": ATHENA_LOG_PARSER_PROCESSED_MESSAGE} 54 | assert result == log_parser.lambda_handler(event, context) 55 | environ.pop('LOG_TYPE') 56 | 57 | 58 | def test_waf_log_athena_parser(waf_log_athena_parser_test_event_setup): 59 | environ['LOG_TYPE'] = "WAF" 60 | event = waf_log_athena_parser_test_event_setup 61 | result = {"message": ATHENA_LOG_PARSER_PROCESSED_MESSAGE} 62 | assert result == log_parser.lambda_handler(event, context) 63 | environ.pop('LOG_TYPE') 64 | 65 | 66 | def test_app_log_athena_result_processor(app_log_athena_query_result_test_event_setup): 67 | event = app_log_athena_query_result_test_event_setup 68 | result = {"message": ATHENA_APP_LOG_QUERY_RESULT_PROCESSED_MESSAGE} 69 | assert result == log_parser.lambda_handler(event, context) 70 | environ.pop('APP_ACCESS_LOG_BUCKET') 71 | 72 | 73 | def test_waf_log_athena_result_processor(waf_log_athena_query_result_test_event_setup): 74 | event = waf_log_athena_query_result_test_event_setup 75 | result = {"message": ATHENA_WAF_LOG_QUERY_RESULT_PROCESSED_MESSAGE} 76 | assert result == log_parser.lambda_handler(event, context) 77 | environ.pop('WAF_ACCESS_LOG_BUCKET') 78 | 79 | 80 | def test_cloudfront_log_lambda_parser(cloudfront_log_lambda_parser_test_event_setup): 81 | environ['LOG_TYPE'] = "cloudfront" 82 | event = cloudfront_log_lambda_parser_test_event_setup 83 | result = {"message": APP_LOG_LAMBDA_PARSER_PROCESSED_MESSAGE} 84 | assert result == log_parser.lambda_handler(event, context) 85 | environ.pop('APP_ACCESS_LOG_BUCKET') 86 | environ.pop('LOG_TYPE') 87 | 88 | 89 | def test_alb_log_lambda_parser(alb_log_lambda_parser_test_event_setup): 90 | environ['LOG_TYPE'] = "alb" 91 | event = alb_log_lambda_parser_test_event_setup 92 | result = {"message": APP_LOG_LAMBDA_PARSER_PROCESSED_MESSAGE} 93 | assert result == log_parser.lambda_handler(event, context) 94 | environ.pop('APP_ACCESS_LOG_BUCKET') 95 | environ.pop('LOG_TYPE') 96 | 97 | def test_alb_log_lambda_parser_over_ip_range_limit(alb_log_lambda_parser_test_event_setup): 98 | environ['LOG_TYPE'] = "alb" 99 | environ['LIMIT_IP_ADDRESS_RANGES_PER_IP_MATCH_CONDITION'] = '1' 100 | event = alb_log_lambda_parser_test_event_setup 101 | result = {"message": APP_LOG_LAMBDA_PARSER_PROCESSED_MESSAGE} 102 | assert result == log_parser.lambda_handler(event, context) 103 | environ.pop('APP_ACCESS_LOG_BUCKET') 104 | environ.pop('LOG_TYPE') 105 | environ.pop('LIMIT_IP_ADDRESS_RANGES_PER_IP_MATCH_CONDITION') 106 | 107 | 108 | def test_waf_lambda_parser(waf_log_lambda_parser_test_event_setup): 109 | environ['LOG_TYPE'] = "waf" 110 | event = waf_log_lambda_parser_test_event_setup 111 | result = {"message": WAF_LOG_LAMBDA_PARSER_PROCESSED_MESSAGE} 112 | assert result == log_parser.lambda_handler(event, context) 113 | environ.pop('WAF_ACCESS_LOG_BUCKET') 114 | environ.pop('LOG_TYPE') 115 | 116 | 117 | def test_waf_lambda_parser_over_ip_range_limit(waf_log_lambda_parser_test_event_setup): 118 | environ['LOG_TYPE'] = "waf" 119 | environ['LIMIT_IP_ADDRESS_RANGES_PER_IP_MATCH_CONDITION'] = '1' 120 | event = waf_log_lambda_parser_test_event_setup 121 | result = {"message": WAF_LOG_LAMBDA_PARSER_PROCESSED_MESSAGE} 122 | assert result == log_parser.lambda_handler(event, context) 123 | environ.pop('WAF_ACCESS_LOG_BUCKET') 124 | environ.pop('LOG_TYPE') 125 | environ.pop('LIMIT_IP_ADDRESS_RANGES_PER_IP_MATCH_CONDITION') 126 | 127 | 128 | def test_lambda_parser_unsupported_log_type(cloudfront_log_lambda_parser_test_event_setup): 129 | try: 130 | environ['LOG_TYPE'] = "unsupported" 131 | event = cloudfront_log_lambda_parser_test_event_setup 132 | except Exception as e: 133 | assert str(e) == TYPE_ERROR_MESSAGE 134 | finally: 135 | environ.pop('APP_ACCESS_LOG_BUCKET') 136 | environ.pop('LOG_TYPE') -------------------------------------------------------------------------------- /source/log_parser/test/test_partition_s3_logs.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from os import environ 5 | from types import SimpleNamespace 6 | 7 | from partition_s3_logs import lambda_handler 8 | 9 | context = SimpleNamespace(**{ 10 | 'function_name': 'foo', 11 | 'memory_limit_in_mb': '512', 12 | 'invoked_function_arn': ':::invoked_function_arn', 13 | 'log_group_name': 'log_group_name', 14 | 'log_stream_name': 'log_stream_name', 15 | 'aws_request_id': 'baz' 16 | }) 17 | 18 | def test_partition_s3_cloudfront_log(partition_s3_cloudfront_log_test_event_setup): 19 | try: 20 | event = partition_s3_cloudfront_log_test_event_setup 21 | result = False 22 | lambda_handler(event, context) 23 | result = True 24 | environ.pop('KEEP_ORIGINAL_DATA') 25 | environ.pop('ENDPOINT') 26 | except Exception: 27 | raise 28 | assert result == True 29 | 30 | 31 | def test_partition_s3_alb_log(partition_s3_alb_log_test_event_setup): 32 | try: 33 | event = partition_s3_alb_log_test_event_setup 34 | result = False 35 | lambda_handler(event, context) 36 | result = True 37 | environ.pop('KEEP_ORIGINAL_DATA') 38 | environ.pop('ENDPOINT') 39 | except Exception: 40 | raise 41 | assert result == True -------------------------------------------------------------------------------- /source/log_parser/test/test_solution_metrics.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). # 5 | # You may not use this file except in compliance with the License. 6 | # A copy of the License is located at # 7 | # # 8 | # http://www.apache.org/licenses/LICENSE-2.0 # 9 | # # 10 | # or in the "license" file accompanying this file. This file is distributed # 11 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express # 12 | # or implied. See the License for the specific language governing permissions# 13 | # and limitations under the License. # 14 | ############################################################################### 15 | 16 | from lib.solution_metrics import send_metrics 17 | 18 | 19 | def test_send_solution_metrics(): 20 | uuid = "waf_test_00001" 21 | solution_id = "waf_test" 22 | data = { 23 | "test_string1": "waf_test", 24 | "test_string2": "test_1" 25 | } 26 | url = "https://testurl.com/generic" 27 | response = send_metrics(data, uuid, solution_id, url) 28 | assert response is not None 29 | -------------------------------------------------------------------------------- /source/reputation_lists_parser/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | test/* 4 | */__init__.py 5 | **/__init__.py 6 | backoff/* 7 | bin/* 8 | boto3/* 9 | botocore/* 10 | certifi/* 11 | charset*/* 12 | crhelper* 13 | chardet* 14 | dateutil/* 15 | idna/* 16 | jmespath/* 17 | lib/* 18 | package* 19 | python_* 20 | requests/* 21 | s3transfer/* 22 | six* 23 | tenacity* 24 | tests 25 | urllib3/* 26 | yaml 27 | PyYAML-* 28 | source = 29 | . -------------------------------------------------------------------------------- /source/reputation_lists_parser/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/reputation_lists_parser/__init__.py -------------------------------------------------------------------------------- /source/reputation_lists_parser/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "reputation_lists_parser" 3 | package-mode = false 4 | 5 | [tool.poetry.dependencies] 6 | python = "~3.12" 7 | requests = "^2.32.3" 8 | backoff = "^2.2.1" 9 | aws-lambda-powertools = "~3.2.0" 10 | 11 | 12 | [tool.poetry.group.dev.dependencies] 13 | moto = "^4.1.4" 14 | pytest = "^7.2.2" 15 | pytest-mock = "^3.12.0" 16 | pytest-runner = "^6.0.0" 17 | pytest-cov = "^4.0.0" 18 | pytest-env = "^0.8.1" 19 | boto3 = "^1.35.30" 20 | 21 | [build-system] 22 | requires = ["poetry-core"] 23 | build-backend = "poetry.core.masonry.api" 24 | -------------------------------------------------------------------------------- /source/reputation_lists_parser/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/reputation_lists_parser/test/__init__.py -------------------------------------------------------------------------------- /source/reputation_lists_parser/test/conftest.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). # 5 | # You may not use this file except in compliance with the License. 6 | # A copy of the License is located at # 7 | # # 8 | # http://www.apache.org/licenses/LICENSE-2.0 # 9 | # # 10 | # or in the "license" file accompanying this file. This file is distributed # 11 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express # 12 | # or implied. See the License for the specific language governing permissions# 13 | # and limitations under the License. # 14 | ############################################################################### 15 | 16 | import pytest 17 | from os import environ 18 | 19 | 20 | @pytest.fixture(scope='module', autouse=True) 21 | def test_environment_vars_setup(): 22 | environ['IP_SET_NAME_REPUTATIONV4'] = 'test_ReputationListsSetIPV4' 23 | environ['IP_SET_NAME_REPUTATIONV6'] = 'test_ReputationListsSetIPV6' 24 | environ['IP_SET_ID_REPUTATIONV4'] = 'arn:aws:wafv2:us-east-1:11111111111:regional/ipset/test' 25 | environ['IP_SET_ID_REPUTATIONV6'] = 'arn:aws:wafv2:us-east-1:11111111111:regional/ipset/test' 26 | environ['SCOPE'] = 'REGIONAL' 27 | environ['SEND_ANONYMIZED_USAGE_DATA'] = 'Yes' 28 | environ['LOG_LEVEL'] = 'INFO' 29 | environ['UUID'] = 'test_uuid' 30 | environ['SOLUTION_ID'] = 'SO0006' 31 | environ['METRICS_URL'] = 'https://testurl.com/generic' -------------------------------------------------------------------------------- /source/reputation_lists_parser/test/test_data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/reputation_lists_parser/test/test_data/__init__.py -------------------------------------------------------------------------------- /source/reputation_lists_parser/test/test_data/test_data.txt: -------------------------------------------------------------------------------- 1 | ; Test Project EDROP List 2023/04/28 - (c) 2023 The Test Project 2 | ; https://www.testendpoint.com/testdummyvalues 3 | ; Last-Modified: Fri, 28 Apr 2023 11:06:55 GMT 4 | ; Expires: Sat, 29 Apr 2023 15:39:42 GMT 5 | x.x.x.x/x ; abcd1234 6 | x.x.x.x/x ; abcd1234 7 | x.x.x.x/x ; abcd1234 8 | x.x.x.x/x ; abcd1234 9 | x.x.x.x/x ; abcd1234 10 | -------------------------------------------------------------------------------- /source/reputation_lists_parser/test/test_reputation_lists_parser.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from types import SimpleNamespace 5 | 6 | from reputation_lists_parser import reputation_lists 7 | from lib.cw_metrics_util import WAFCloudWatchMetrics 8 | from os import environ 9 | import pytest 10 | import requests 11 | 12 | context = SimpleNamespace(**{ 13 | 'function_name': 'foo', 14 | 'memory_limit_in_mb': '512', 15 | 'invoked_function_arn': ':::invoked_function_arn', 16 | 'log_group_name': 'log_group_name', 17 | 'log_stream_name': 'log_stream_name', 18 | 'aws_request_id': 'baz' 19 | }) 20 | 21 | def test_lambda_handler_raises_exception_if_env_variable_not_present(mocker): 22 | event = {} 23 | mocker.patch.object(WAFCloudWatchMetrics, 'add_waf_cw_metric_to_usage_data') 24 | with pytest.raises(TypeError): 25 | reputation_lists.lambda_handler(event, context) 26 | 27 | 28 | def test_lambda_handler_returns_error_when_populate_ip_sets_function_fails(mocker): 29 | event = {} 30 | environ['URL_LIST'] = '[{"url":"https://www.testmocketenvtest.com"},' \ 31 | '{"url":"https://www.testmocketenvagaintest.com"}] ' 32 | mocker.patch.object(reputation_lists, 'populate_ipsets', side_effect=Exception('mocked error')) 33 | mocker.patch.object(requests, 'get') 34 | response = reputation_lists.lambda_handler(event, context) 35 | assert response == '{"statusCode": "400", "body": {"message": "mocked error"}}' 36 | 37 | 38 | def test_lambda_handler_returns_success(mocker): 39 | event = {} 40 | environ['URL_LIST'] = '[{"url":"https://www.testmocketenvtest.com"},' \ 41 | '{"url":"https://www.testmocketenvagaintest.com"}] ' 42 | mocker.patch.object(requests, 'get') 43 | with open('./test/test_data/test_data.txt', 'r') as file: 44 | test_data = file.read() 45 | requests.get.return_value = test_data 46 | ip_set = {'IPSet': 47 | { 48 | 'Name': 'prodIPReputationListsSetIPV6', 49 | 'Id': '4342423-d428-4e9d-ba3a-376737347db', 50 | 'ARN': 'arn:aws:wafv2:us-east-1:111111111:regional/ipset/ptestvalue', 51 | 'Description': 'Block Reputation List IPV6 addresses', 52 | 'IPAddressVersion': 'IPV6', 53 | 'Addresses': [] 54 | }, 55 | 'LockToken': 'test-token', 56 | 'ResponseMetadata': { 57 | 'RequestId': 'test-id', 58 | 'HTTPStatusCode': 200, 59 | 'HTTPHeaders': 60 | {'x-amzn-requestid': 'test-id', 61 | 'content-type': 'application/x-amz-json-1.1', 62 | 'content-length': 'test', 63 | 'date': 'Thu, 27 Apr 2023 03:50:24 GMT'}, 64 | 'RetryAttempts': 0 65 | } 66 | } 67 | mocker.patch.object(reputation_lists.waflib, 'update_ip_set') 68 | mocker.patch.object(reputation_lists.waflib, 'get_ip_set') 69 | mocker.patch.object(reputation_lists.waflib, 'update_ip_set') 70 | mocker.patch.object(reputation_lists.waflib, 'get_ip_set') 71 | mocker.patch.object(WAFCloudWatchMetrics, 'add_waf_cw_metric_to_usage_data') 72 | reputation_lists.waflib.get_ip_set.return_value = ip_set 73 | response = reputation_lists.lambda_handler(event, context) 74 | assert response == '{"StatusCode": "200", "Body": {"message": "success"}}' 75 | -------------------------------------------------------------------------------- /source/timer/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | test/* 4 | */__init__.py 5 | **/__init__.py 6 | backoff/* 7 | bin/* 8 | boto3/* 9 | botocore/* 10 | certifi/* 11 | charset*/* 12 | crhelper* 13 | chardet* 14 | dateutil/* 15 | idna/* 16 | jmespath/* 17 | lib/* 18 | package* 19 | python_* 20 | requests/* 21 | s3transfer/* 22 | six* 23 | tenacity* 24 | tests 25 | urllib3/* 26 | yaml 27 | PyYAML-* 28 | source = 29 | . -------------------------------------------------------------------------------- /source/timer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/timer/__init__.py -------------------------------------------------------------------------------- /source/timer/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "timer" 3 | package-mode = false 4 | 5 | [tool.poetry.dependencies] 6 | python = "~3.12" 7 | requests = "^2.32.3" 8 | backoff = "^2.2.1" 9 | aws-lambda-powertools = "~3.2.0" 10 | 11 | 12 | [tool.poetry.group.dev.dependencies] 13 | moto = "^4.1.4" 14 | pytest = "^7.2.2" 15 | pytest-mock = "^3.12.0" 16 | pytest-runner = "^6.0.0" 17 | pytest-cov = "^4.0.0" 18 | pytest-env = "^0.8.1" 19 | boto3 = "^1.35.30" 20 | 21 | [build-system] 22 | requires = ["poetry-core"] 23 | build-backend = "poetry.core.masonry.api" 24 | -------------------------------------------------------------------------------- /source/timer/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/aws-waf-security-automations/321d3bf8c8fc1b8523ac425d57c13cd60561c2ba/source/timer/test/__init__.py -------------------------------------------------------------------------------- /source/timer/test/conftest.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | import pytest 15 | 16 | class Context: 17 | def __init__(self, invoked_function_arn, log_group_name, log_stream_name): 18 | self.invoked_function_arn = invoked_function_arn 19 | self.log_group_name = log_group_name 20 | self.log_stream_name = log_stream_name 21 | 22 | 23 | @pytest.fixture(scope="session") 24 | def example_context(): 25 | return Context(':::invoked_function_arn', 'log_group_name', 'log_stream_name') 26 | 27 | 28 | @pytest.fixture(scope="session") 29 | def timer_event(): 30 | return { 31 | 'LogicalResourceId': 'Timer', 32 | 'RequestId': '25d75d10-c5fa-48da-a79a-d827bfe0a465', 33 | 'RequestType': 'Create', 34 | 'ResourceProperties': { 35 | 'DeliveryStreamArn': 'arn:aws:firehose:us-east-2:XXXXXXXXXXXX:deliverystream/aws-waf-logs-wafohio_xToOQk', 36 | 'ServiceToken': 'arn:aws:lambda:us-east-2:XXXXXXXXXXXX:function:wafohio-CustomResource-WnfNLnBqtXPF', 37 | 'WAFWebACLArn': 'arn:aws:wafv2:us-east-2:XXXXXXXXXXXX:regional/webacl/wafohio/c2e77a1b-6bb3-4d9d-86f9-0bfd9b6fdcaf' 38 | }, 39 | 'ResourceType': 'Custom::Timer', 40 | 'ResponseURL': 'https://cloudformation-custom-resource-response-useast2.s3.us-east-2.amazonaws.com/', 41 | 'ServiceToken': 'arn:aws:lambda:us-east-2:XXXXXXXXXXXX:function:wafohio-CustomResource-WnfNLnBqtXPF', 42 | 'StackId': 'arn:aws:cloudformation:us-east-2:XXXXXXXXXXXX:stack/wafohio/70c177d0-e2c7-11ed-9e83-02ff465f0e71' 43 | } -------------------------------------------------------------------------------- /source/timer/test/test_timer.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | from timer.timer import lambda_handler 15 | 16 | def test_timer(timer_event, example_context): 17 | result = lambda_handler(timer_event, example_context) 18 | expected = '{"StatusCode": "200", "Body": {"message": "success"}}' 19 | assert result == expected -------------------------------------------------------------------------------- /source/timer/timer.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | ###################################################################################################################### 5 | ###################################################################################################################### 6 | 7 | import time 8 | import os 9 | import json 10 | from lib.cfn_response import send_response 11 | from aws_lambda_powertools import Logger 12 | 13 | logger = Logger( 14 | level=os.getenv('LOG_LEVEL') 15 | ) 16 | 17 | # ====================================================================================================================== 18 | # Lambda Entry Point 19 | # ====================================================================================================================== 20 | def lambda_handler(event, context): 21 | logger.info('[lambda_handler] Start') 22 | 23 | response_status = 'SUCCESS' 24 | reason = None 25 | response_data = {} 26 | result = { 27 | 'StatusCode': '200', 28 | 'Body': {'message': 'success'} 29 | } 30 | 31 | try: 32 | count = 3 33 | SECONDS = os.getenv('SECONDS') 34 | if (SECONDS != None): 35 | count = int(SECONDS) 36 | time.sleep(count) 37 | logger.info(count) 38 | except Exception as error: 39 | logger.error(str(error)) 40 | response_status = 'FAILED' 41 | reason = str(error) 42 | result = { 43 | 'statusCode': '400', 44 | 'body': {'message': reason} 45 | } 46 | finally: 47 | logger.info('[lambda_handler] End') 48 | if 'ResponseURL' in event: 49 | resource_id = event['PhysicalResourceId'] if 'PhysicalResourceId' in event else event['LogicalResourceId'] 50 | logger.info("ResourceId %s", resource_id) 51 | send_response(logger, event, context, response_status, response_data, resource_id, reason) 52 | 53 | return json.dumps(result) #NOSONAR needed to send a response of the result 54 | --------------------------------------------------------------------------------