├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── cloudformation └── template.yml ├── lambda ├── services.json ├── test_event.json └── update_aws_ip_ranges.py └── terraform └── main.tf /.gitignore: -------------------------------------------------------------------------------- 1 | *.DS_Store 2 | __pycache__/ 3 | *.zip 4 | .vscode 5 | internal/* 6 | *.bak 7 | _*.* 8 | *.zip 9 | lambda_return.json 10 | ip-ranges.json 11 | *.sh 12 | probeout 13 | 14 | ### Terraform ### 15 | # Local .terraform directories 16 | **/.terraform/* 17 | 18 | # .tfstate files 19 | *.tfstate 20 | *.tfstate.* 21 | 22 | # Crash log files 23 | crash.log 24 | crash.*.log 25 | 26 | # Exclude all .tfvars files, which are likely to contain sensitive data, such as 27 | # password, private keys, and other secrets. These should not be part of version 28 | # control as they are data points which are potentially sensitive and subject 29 | # to change depending on the environment. 30 | *.tfvars 31 | *.tfvars.json 32 | 33 | # Ignore override files as they are usually used to override resources locally and so 34 | # are not checked in 35 | override.tf 36 | override.tf.json 37 | *_override.tf 38 | *_override.tf.json 39 | 40 | # Include override files you do wish to add to version control using negated pattern 41 | # !example_override.tf 42 | 43 | # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan 44 | # example: *tfplan* 45 | 46 | # Ignore CLI configuration files 47 | .terraformrc 48 | terraform.rc 49 | 50 | *.hcl -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Automatically update AWS resources with AWS IP Ranges 2 | 3 | This project creates Lambda function that automatically create or update AWS resource with AWS service's IP ranges from the [ip-ranges.json](https://docs.aws.amazon.com/general/latest/gr/aws-ip-ranges.html) file. 4 | You can configure which service and region to get range. You can also configure to which resources you want to create or update with those ranges. 5 | Use cases include allowing CloudFront requests, API Gateway requests, Route53 health checker and EC2 IP range (which includes AWS Lambda and CloudWatch Synthetics). 6 | The resources are created or updated in the region where the CloudFormation stack is created. 7 | 8 | 9 | > **NOTE ABOUT CloudFront:** 10 | > There is already a managed VPC Prefix List for CloudFront. 11 | > So, doesn't make sense to use this code to create Prefix List for CloudFront. Please, use the managed one. 12 | > It does make sense to use this code to handle WAF IPSet for CloudFront. 13 | > https://aws.amazon.com/about-aws/whats-new/2022/02/amazon-cloudfront-managed-prefix-list/ 14 | 15 | 16 | 17 | > **NOTE ABOUT Route 53 health checks:** 18 | > There is already a managed VPC Prefix List for Route 53 health checks. 19 | > So, doesn't make sense to use this code to create Prefix List for Route 53 health checks. Please, use the managed ones. 20 | > It does make sense to use this code to handle WAF IPSet for Route 53 health checks. 21 | https://aws.amazon.com/about-aws/whats-new/2023/09/amazon-route-53-managed-prefix-lists-health-checks/ 22 | 23 | 24 | 25 | > **NOTE** 26 | > This is an upgraded version of the repository below. This repo add support for VPC Prefix List and allow you to have resources for more than one service. 27 | > https://github.com/aws-samples/aws-waf-ipset-auto-update-aws-ip-ranges 28 | 29 | 30 | ## Overview 31 | 32 | The CloudFormation template `cloudformation/template.yml` creates a stack with the following resources: 33 | 34 | 1. AWS Lambda function with customizable config file called `services.json`. The function's code is in `lambda/update_aws_ip_ranges.py` and is written in Python compatible with version 3.12. 35 | 1. Lambda function's execution role. 36 | 1. SNS subscription and Lambda invocation permissions for the `arn:aws:sns:us-east-1:806199016981:AmazonIpSpaceChanged` SNS topic. 37 | 38 | ``` 39 | +-----------------+ +---------------------+ 40 | | Lambda | | | 41 | | Execution Role | +--->+AWS WAF IPv4/IPv6 Set| 42 | +--------+--------+ | | | 43 | | | +---------------------+ 44 | | | 45 | +--------------------+ +--------+--------+ | 46 | |SNS Topic +--->+ Lambda function +----+ 47 | |AmazonIpSpaceChanged| +--------+--------+ | 48 | +--------------------+ | | +-------------------+ 49 | | | | | 50 | v +--->+AWS VPC Prefix List| 51 | +--------+--------+ | | 52 | | CloudWatch Logs | +-------------------+ 53 | +-----------------+ 54 | ``` 55 | 56 | ## Supported resources 57 | 58 | It supports to create or update the following resource: 59 | * WAF IPSet (only WAFv2, WAF Classic is not supported) 60 | * VPC Prefix List 61 | 62 | > **NOTE** 63 | > If you miss some AWS resource that should be supported, feel free to open an issue or contribute with a pull request. 64 | > Keep in mind that you can reference a prefix list in the following AWS resources: 65 | > * VPC security groups 66 | > * Subnet route tables 67 | > * Transit gateway route tables 68 | > * AWS Network Firewall rule groups 69 | > 70 | > It will create or update AWS resource, but it will NEVER remove it after it is created. 71 | > As soon it is created it can only be updated by this solution. 72 | > If you need to remove any resource created by solution you need to do it manually. 73 | 74 | ### Considerations 75 | 76 | * Lambda code MUST have a config file called `services.json` in the root path. See below more details about its format. 77 | * WAF IPSet will just be updated if there are entries to remove or to add. 78 | * VPC Prefix List will just be updated if there are entries to remove or to add. 79 | * When VPC Prefix List is created, the `max entries` configuration will be the length of current IP ranges for that service plus 10. 80 | * When VPC Prefix List is updated, if current `max entries` configuration is lower than the length of current IP ranges for that service, it will change the `max entries` to the length of current IP ranges. If it fails to update, due to size restriction where Prefix List is used, it will NOT update the IP ranges. 81 | * If it fails to create or update resource for any service, the code will not stop, it will continue to handle the other resource and services. 82 | * It only creates resource for service and IP version if there is at least one IP range. Otherwise, it will not create. 83 | * Resources are named as `aws-ip-ranges--`. 84 | Where: 85 | * `` is the service name inside `ip-ranges.json` file. Converted to lower case and replaced `_` with `-`. 86 | * `` is `ipv4` or `ipv6`. 87 | 88 | Examples: 89 | * `aws-ip-ranges-api-gateway-ipv4` 90 | * `aws-ip-ranges-route53-healthchecks-ipv4` 91 | * `aws-ip-ranges-route53-healthchecks-ipv6` 92 | 93 | > **NOTE ABOUT REGIONS DEPLOY** 94 | > There is no reason to deploy this solution twice inside the same region. 95 | > If you have a reason for doing it, please open an issue and let's talk about it. 96 | 97 | ## Lambda configuration 98 | 99 | To configure which service lambda should handle IP ranges or which region, you need to change the file `services.json`. 100 | 101 | To see the list of possible service names inside `ip-ranges.json` file, run the command below: 102 | ```shell 103 | curl -s 'https://ip-ranges.amazonaws.com/ip-ranges.json' | jq -r '.prefixes[] | .service' | sort -u 104 | ``` 105 | 106 | To see the list of possible region names inside `ip-ranges.json` file, run the command below: 107 | ```shell 108 | curl -s 'https://ip-ranges.amazonaws.com/ip-ranges.json' | jq -r '.prefixes[] | .region' | sort -u 109 | ``` 110 | 111 | See below the file commented. 112 | 113 | ```shell 114 | { 115 | "Services": [ 116 | { 117 | # Service name. MUST match the service name inside ip-ranges.json file. 118 | # Case is sensitive. 119 | "Name": "API_GATEWAY", 120 | 121 | # Region name. It is an array, so you can specify more than one region. MUST match the region name inside ip-ranges.json file. 122 | # Case is sensitive. 123 | # 124 | # Please not that there is one region called GLOBAL inside ip-ranges.json file. 125 | # If you want to get IP ranges from all region keep the array empty. 126 | # 127 | # If you specify more than one region, or keep it empty, it will aggregate the IP ranges from those regions inside the resource at the region where Lambda function is running. 128 | # It will NOT create the resources on each region specified. 129 | "Regions": ["sa-east-1"], 130 | 131 | "PrefixList": { 132 | # Indicate if VPC Prefix List should be create for IP ranges from this service. It will be created in the same region where Lambda function is running. 133 | "Enable": true, 134 | # Indicate if VPC Prefix List IP ranges should be summarized or not for this specific service. 135 | "Summarize": true 136 | }, 137 | 138 | "WafIPSet": { 139 | # Indicate if WAF IPSet should be create for IP ranges from this service. It will be created in the same region where Lambda function is running. 140 | "Enable": true, 141 | # Indicate if WAF IPSet IP ranges should be summarized or not for this specific service. 142 | "Summarize": true, 143 | # WAF IPSet scope to create or update resources. Possible values are ONLY "CLOUDFRONT" and "REGIONAL". 144 | # Case is sensitive. 145 | # 146 | # Note that "CLOUDFRONT" can ONLY be used in North Virginia (us-east-1) region. So, you MUST deploy it on North Virginia (us-east-1) region. 147 | "Scopes": ["CLOUDFRONT", "REGIONAL"] 148 | } 149 | } 150 | ] 151 | } 152 | ``` 153 | 154 | Example: 155 | 156 | ```json 157 | { 158 | "Services": [ 159 | { 160 | "Name": "API_GATEWAY", 161 | "Regions": ["sa-east-1"], 162 | "PrefixList": { 163 | "Enable": true, 164 | "Summarize": true 165 | }, 166 | "WafIPSet": { 167 | "Enable": true, 168 | "Summarize": true, 169 | "Scopes": ["REGIONAL"] 170 | } 171 | }, 172 | { 173 | "Name": "CLOUDFRONT_ORIGIN_FACING", 174 | "Regions": [], 175 | "PrefixList": { 176 | "Enable": false, 177 | "Summarize": false 178 | }, 179 | "WafIPSet": { 180 | "Enable": true, 181 | "Summarize": false, 182 | "Scopes": ["REGIONAL"] 183 | } 184 | }, 185 | { 186 | "Name": "EC2_INSTANCE_CONNECT", 187 | "Regions": ["sa-east-1"], 188 | "PrefixList": { 189 | "Enable": true, 190 | "Summarize": false 191 | }, 192 | "WafIPSet": { 193 | "Enable": true, 194 | "Summarize": false, 195 | "Scopes": ["REGIONAL"] 196 | } 197 | }, 198 | { 199 | "Name": "ROUTE53_HEALTHCHECKS", 200 | "Regions": [], 201 | "PrefixList": { 202 | "Enable": false, 203 | "Summarize": false 204 | }, 205 | "WafIPSet": { 206 | "Enable": true, 207 | "Summarize": false, 208 | "Scopes": ["REGIONAL"] 209 | } 210 | } 211 | ] 212 | } 213 | ``` 214 | 215 | ## Setup 216 | 217 | These are the overall steps to deploy: 218 | 219 | **Setup using CloudFormation** 220 | 1. Validate CloudFormation template file. 221 | 1. Create the CloudFormation stack. 222 | 1. Package the Lambda code into a `.zip` file. 223 | 1. Update Lambda function with the packaged code. 224 | 225 | **Setup using Terraform** 226 | 1. Initialize Terraform state 227 | 1. Validate Terraform template. 228 | 1. Apply Terraform template. 229 | 230 | **After setup** 231 | 1. Trigger a test Lambda invocation. 232 | 1. Reference resources 233 | 1. Clean-up 234 | 235 | 236 | 237 | 238 | 239 | ## Setup using CloudFormation 240 | To simplify setup and deployment, assign the values to the following variables. Replace the values according to your deployment options. 241 | 242 | ```bash 243 | export AWS_REGION="sa-east-1" 244 | export CFN_STACK_NAME="update-aws-ip-ranges" 245 | ``` 246 | 247 | > **IMPORTANT:** Please, use AWS CLI v2 248 | 249 | ### 1. Validate CloudFormation template 250 | 251 | Ensure the CloudFormation template is valid before use it. 252 | 253 | ```bash 254 | aws cloudformation validate-template --template-body file://cloudformation/template.yml 255 | ``` 256 | 257 | ### 2. Create CloudFormation stack 258 | 259 | At this point it will create Lambda function with a dummy code. 260 | You will update it later. 261 | 262 | ```bash 263 | aws cloudformation create-stack --stack-name "${CFN_STACK_NAME}" \ 264 | --capabilities CAPABILITY_IAM \ 265 | --template-body file://cloudformation/template.yml && { 266 | ### Wait for stack to be created 267 | aws cloudformation wait stack-create-complete --stack-name "${CFN_STACK_NAME}" 268 | } 269 | ``` 270 | 271 | If the stack creation fails, troubleshoot by reviewing the stack events. The typical failure reasons are insufficient IAM permissions. 272 | 273 | ### 3. Create the packaged code 274 | 275 | Before you package it, please change `lambda/services.json` file according to your requirement. For more information read the section `Lambda configuration` above. 276 | 277 | ```bash 278 | zip --junk-paths update_aws_ip_ranges.zip lambda/update_aws_ip_ranges.py lambda/services.json 279 | ``` 280 | 281 | ### 4. Update lambda package code 282 | 283 | ```bash 284 | FUNCTION_NAME=$(aws cloudformation describe-stack-resources --stack-name "${CFN_STACK_NAME}" --query "StackResources[?LogicalResourceId=='LambdaUpdateIPRanges'].PhysicalResourceId" --output text) 285 | aws lambda update-function-code --function-name "${FUNCTION_NAME}" --zip-file fileb://update_aws_ip_ranges.zip --publish 286 | ``` 287 | 288 | > **NOTE:** Every time you change Lambda function configuration file `services.json` you need to execute steps 3 and 4 again. 289 | 290 | 291 | 292 | 293 | 294 | ## Setup using Terraform 295 | 296 | Terraform template uses the following providers: 297 | * aws 298 | * archive 299 | 300 | > **IMPORTANT:** Please, use Terraform version 1.3.7 or higher 301 | 302 | ### 1. Initialize Terraform state 303 | ```bash 304 | cd terraform/ 305 | terraform init 306 | ``` 307 | 308 | ### 2. Validate Terraform template 309 | 310 | Ensure Terraform template is valid before use it. 311 | 312 | ```bash 313 | terraform validate 314 | ``` 315 | 316 | ### 3. Apply Terraform template 317 | 318 | Before you apply, please change `lambda/services.json` file according to your requirement. For more information read the section `Lambda configuration` above. 319 | 320 | ```bash 321 | terraform apply 322 | ``` 323 | 324 | > **NOTE:** Every time you change Lambda function configuration file `services.json` you need to execute this step again. 325 | 326 | 327 | 328 | 329 | 330 | ## After setup 331 | 332 | ### 1a. Trigger a test Lambda invocation with the AWS CLI 333 | 334 | After the stack is created, AWS resources are not created or updated until a new SNS message is received. To test the function and create or update AWS resources with the current IP ranges for the first time, do a test invocation with the AWS CLI command below: 335 | 336 | **CloudFormation** 337 | ```bash 338 | aws lambda invoke \ 339 | --function-name "${FUNCTION_NAME}" \ 340 | --cli-binary-format 'raw-in-base64-out' \ 341 | --payload file://lambda/test_event.json lambda_return.json 342 | ``` 343 | 344 | **Terraform** 345 | ```bash 346 | FUNCTION_NAME=$(terraform output | grep 'lambda_name' | cut -d ' ' -f 3 | tr -d '"') 347 | aws lambda invoke \ 348 | --function-name "${FUNCTION_NAME}" \ 349 | --cli-binary-format 'raw-in-base64-out' \ 350 | --payload file://lambda/test_event.json lambda_return.json 351 | ``` 352 | 353 | After successful invocation, you should receive the response below with no errors. 354 | 355 | ```json 356 | { 357 | "StatusCode": 200, 358 | "ExecutedVersion": "$LATEST" 359 | } 360 | ``` 361 | 362 | The content of the `lambda_return.json` will list all AWS resources created or updated by the Lambda function with IP ranges from configured services. 363 | 364 | ### 1b. Trigger a test Lambda invocation with the AWS Console 365 | 366 | Alternatively, you can invoke the test event in the AWS Lambda console with sample event below. This event uses a `test-hash` md5 string that the function parses as a test event. 367 | 368 | ```json 369 | { 370 | "Records": [ 371 | { 372 | "EventVersion": "1.0", 373 | "EventSubscriptionArn": "arn:aws:sns:EXAMPLE", 374 | "EventSource": "aws:sns", 375 | "Sns": { 376 | "SignatureVersion": "1", 377 | "Timestamp": "1970-01-01T00:00:00.000Z", 378 | "Signature": "EXAMPLE", 379 | "SigningCertUrl": "EXAMPLE", 380 | "MessageId": "12345678-1234-1234-1234-123456789012", 381 | "Message": "{\"create-time\": \"yyyy-mm-ddThh:mm:ss+00:00\", \"synctoken\": \"0123456789\", \"md5\": \"test-hash\", \"url\": \"https://ip-ranges.amazonaws.com/ip-ranges.json\"}", 382 | "Type": "Notification", 383 | "UnsubscribeUrl": "EXAMPLE", 384 | "TopicArn": "arn:aws:sns:EXAMPLE", 385 | "Subject": "TestInvoke" 386 | } 387 | } 388 | ] 389 | } 390 | ``` 391 | 392 | ### 2. Reference resources 393 | 394 | For WAF IPSet, see [Using an IP set in a rule group or Web ACL](https://docs.aws.amazon.com/waf/latest/developerguide/waf-ip-set-using.html). 395 | For VPC Prefix List, see [Reference prefix lists in your AWS resources](https://docs.aws.amazon.com/vpc/latest/userguide/managed-prefix-lists-referencing.html). 396 | 397 | ### 3. Clean-up 398 | 399 | Remove the temporary files, remove CloudFormation stack and destroy Terraform resources. 400 | 401 | **CloudFormation** 402 | ```bash 403 | rm update_aws_ip_ranges.zip 404 | rm lambda_return.json 405 | aws cloudformation delete-stack --stack-name "${CFN_STACK_NAME}" 406 | unset AWS_REGION 407 | unset CFN_STACK_NAME 408 | ``` 409 | 410 | **Terraform** 411 | ```bash 412 | rm lambda_return.json 413 | terraform destroy 414 | ``` 415 | 416 | 417 | > **ATTENTION** 418 | > When you remove CloudFormation stack, or destroy Terraform resources, it will NOT remove WAF IPSet or VPC Prefix List created by this solution. 419 | > If you want to remove it, you need to do it manually. 420 | 421 | ## Lambda function customization 422 | 423 | After the stack is created, you can customize the Lambda function's execution log level by editing the function's [environment variables](https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html). 424 | 425 | * `LOG_LEVEL`: **Optional**. Set log level to increase or reduce verbosity. The default value is `INFO`. Possible values are: 426 | * CRITICAL 427 | * ERROR 428 | * WARNING 429 | * INFO 430 | * DEBUG 431 | 432 | ## Troubleshooting 433 | 434 | **Wrong WAF IPSet Scope** 435 | 436 | > An error occurred (WAFInvalidParameterException) when calling the ListIPSets operation: Error reason: The scope is not valid., field: SCOPE_VALUE, parameter: CLOUDFRONT 437 | 438 | Scope name `CLOUDFRONT` is correct, but it MUST be running on North Virginia (us-east-1) region. If it runs outside North Virginia, you will see the error above. 439 | Please make sure it is running on North Virginia (us-east-1) region. 440 | 441 | ## Security 442 | 443 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 444 | 445 | ## License 446 | 447 | This library is licensed under the MIT-0 License. See the LICENSE file. 448 | -------------------------------------------------------------------------------- /cloudformation/template.yml: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | AWSTemplateFormatVersion: 2010-09-09 5 | Description: > 6 | Creates Lambda function and subscribe it to ip-ranges.json SNS topic. 7 | Lambda function will create or update AWS resources according to the configuration on services.json file inside it. 8 | 9 | Resources: 10 | LambdaUpdateIPRanges: 11 | Type: AWS::Lambda::Function 12 | # checkov:skip=CKV_AWS_116:Code log errors on CloudWatch logs 13 | # checkov:skip=CKV_AWS_117:Not required to run inside a VPC 14 | # checkov:skip=CKV_AWS_173:Variable is not sensitive 15 | Metadata: 16 | cfn_nag: 17 | rules_to_suppress: 18 | - id: W58 19 | reason: "Permission is defined with much restriction as possible" 20 | - id: W89 21 | reason: "Not required to run inside a VPC" 22 | Properties: 23 | Description: 'This Lambda function, invoked by an incoming SNS message, updates the IPv4 and IPv6 ranges with the addresses from the specified services' 24 | Architectures: 25 | - 'arm64' 26 | Environment: 27 | Variables: 28 | LOG_LEVEL: 'INFO' 29 | Handler: 'update_aws_ip_ranges.lambda_handler' 30 | MemorySize: 256 31 | Role: !GetAtt 'LambdaUpdateIPRangesIamRole.Arn' 32 | Runtime: python3.12 33 | Timeout: 300 34 | ReservedConcurrentExecutions: 2 35 | Code: 36 | ZipFile: | 37 | def lambda_handler(event, context): 38 | print(event) 39 | 40 | LambdaUpdateIPRangesIamRole: 41 | Type: AWS::IAM::Role 42 | # checkov:skip=CKV_AWS_111:CloudWatch Logs doesn't support condition 43 | Metadata: 44 | cfn_nag: 45 | rules_to_suppress: 46 | - id: W11 47 | reason: "Policy has conditions when it is allowed" 48 | Properties: 49 | AssumeRolePolicyDocument: 50 | Version: 2012-10-17 51 | Statement: 52 | - Effect: Allow 53 | Principal: 54 | Service: lambda.amazonaws.com 55 | Action: sts:AssumeRole 56 | Description: IP Ranges auto update Lambda 57 | Policies: 58 | - PolicyName: 'CloudWatchLogsPermissions' 59 | PolicyDocument: 60 | Version: '2012-10-17' 61 | Statement: 62 | - Effect: 'Allow' 63 | Action: 64 | - 'logs:CreateLogGroup' 65 | Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*' 66 | 67 | - Effect: 'Allow' 68 | Action: 69 | - 'logs:CreateLogStream' 70 | - 'logs:PutLogEvents' 71 | Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*LambdaUpdateIPRanges*:*' 72 | 73 | 74 | - PolicyName: 'WAFPermissions' 75 | PolicyDocument: 76 | Version: '2012-10-17' 77 | Statement: 78 | - Effect: 'Allow' 79 | Action: 80 | - 'wafv2:ListIPSets' 81 | Resource: '*' 82 | 83 | - Effect: 'Allow' 84 | Action: 85 | - 'wafv2:CreateIPSet' 86 | - 'wafv2:TagResource' 87 | Resource: '*' 88 | Condition: 89 | StringLike: 90 | 'aws:RequestTag/Name': 91 | - 'aws-ip-ranges-*-ipv4' 92 | - 'aws-ip-ranges-*-ipv6' 93 | StringEquals: 94 | 'aws:RequestTag/ManagedBy': 'update-aws-ip-ranges' 95 | 'aws:RequestTag/UpdatedAt': 'Not yet' 96 | 'ForAllValues:StringEquals': 97 | 'aws:TagKeys': 98 | - 'Name' 99 | - 'ManagedBy' 100 | - 'CreatedAt' 101 | - 'UpdatedAt' 102 | 103 | - Effect: 'Allow' 104 | Action: 105 | - 'wafv2:TagResource' 106 | Resource: 107 | - !Sub 'arn:${AWS::Partition}:wafv2:${AWS::Region}:${AWS::AccountId}:*/ipset/aws-ip-ranges-*-ipv4/*' 108 | - !Sub 'arn:${AWS::Partition}:wafv2:${AWS::Region}:${AWS::AccountId}:*/ipset/aws-ip-ranges-*-ipv6/*' 109 | Condition: 110 | StringLike: 111 | 'aws:ResourceTag/Name': 112 | - 'aws-ip-ranges-*-ipv4' 113 | - 'aws-ip-ranges-*-ipv6' 114 | StringEquals: 115 | 'aws:ResourceTag/ManagedBy': 'update-aws-ip-ranges' 116 | 'ForAllValues:StringEquals': 117 | 'aws:TagKeys': 118 | - 'UpdatedAt' 119 | 120 | - Effect: 'Allow' 121 | Action: 122 | - 'wafv2:ListTagsForResource' 123 | - 'wafv2:GetIPSet' 124 | - 'wafv2:UpdateIPSet' 125 | Resource: 126 | - !Sub 'arn:${AWS::Partition}:wafv2:${AWS::Region}:${AWS::AccountId}:*/ipset/aws-ip-ranges-*-ipv4/*' 127 | - !Sub 'arn:${AWS::Partition}:wafv2:${AWS::Region}:${AWS::AccountId}:*/ipset/aws-ip-ranges-*-ipv6/*' 128 | Condition: 129 | StringLike: 130 | 'aws:ResourceTag/Name': 131 | - 'aws-ip-ranges-*-ipv4' 132 | - 'aws-ip-ranges-*-ipv6' 133 | StringEquals: 134 | 'aws:ResourceTag/ManagedBy': 'update-aws-ip-ranges' 135 | 136 | 137 | - PolicyName: 'EC2Permissions' 138 | PolicyDocument: 139 | Version: '2012-10-17' 140 | Statement: 141 | - Effect: 'Allow' 142 | Action: 143 | - 'ec2:DescribeTags' 144 | - 'ec2:DescribeManagedPrefixLists' 145 | Resource: '*' 146 | Condition: 147 | StringEquals: 148 | 'ec2:Region': !Ref AWS::Region 149 | 150 | - Effect: 'Allow' 151 | Action: 152 | - 'ec2:GetManagedPrefixListEntries' 153 | - 'ec2:ModifyManagedPrefixList' 154 | - 'ec2:CreateTags' 155 | Resource: !Sub 'arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:prefix-list/*' 156 | Condition: 157 | StringEquals: 158 | 'ec2:Region': !Ref AWS::Region 159 | 'aws:ResourceTag/ManagedBy': 'update-aws-ip-ranges' 160 | StringLike: 161 | 'aws:ResourceTag/Name': 162 | - 'aws-ip-ranges-*-ipv4' 163 | - 'aws-ip-ranges-*-ipv6' 164 | 165 | - Effect: 'Allow' 166 | Action: 167 | - 'ec2:CreateManagedPrefixList' 168 | Resource: !Sub 'arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:prefix-list/*' 169 | Condition: 170 | StringEquals: 171 | 'ec2:Region': !Ref AWS::Region 172 | 'aws:RequestTag/ManagedBy': 'update-aws-ip-ranges' 173 | 'aws:RequestTag/UpdatedAt': 'Not yet' 174 | StringLike: 175 | 'aws:RequestTag/Name': 176 | - 'aws-ip-ranges-*-ipv4' 177 | - 'aws-ip-ranges-*-ipv6' 178 | 'ForAllValues:StringEquals': 179 | 'aws:TagKeys': 180 | - 'Name' 181 | - 'ManagedBy' 182 | - 'CreatedAt' 183 | - 'UpdatedAt' 184 | 185 | - Effect: 'Allow' 186 | Action: 187 | - 'ec2:CreateTags' 188 | Resource: !Sub 'arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:prefix-list/*' 189 | Condition: 190 | StringEquals: 191 | 'ec2:Region': !Ref AWS::Region 192 | 'ec2:CreateAction' : 'CreateManagedPrefixList' 193 | 194 | LambdaPermission: 195 | Type: 'AWS::Lambda::Permission' 196 | Properties: 197 | Action: 'lambda:InvokeFunction' 198 | FunctionName: !Ref LambdaUpdateIPRanges 199 | Principal: 'sns.amazonaws.com' 200 | SourceArn: 'arn:aws:sns:us-east-1:806199016981:AmazonIpSpaceChanged' 201 | SourceAccount: '806199016981' 202 | 203 | LambdaSNSSubscription: 204 | Type: 'AWS::SNS::Subscription' 205 | Properties: 206 | Endpoint: !GetAtt 'LambdaUpdateIPRanges.Arn' 207 | Protocol: 'lambda' 208 | Region: 'us-east-1' 209 | TopicArn: 'arn:aws:sns:us-east-1:806199016981:AmazonIpSpaceChanged' 210 | -------------------------------------------------------------------------------- /lambda/services.json: -------------------------------------------------------------------------------- 1 | { 2 | "Services": [ 3 | { 4 | "Name": "API_GATEWAY", 5 | "Regions": ["sa-east-1"], 6 | "PrefixList": { 7 | "Enable": true, 8 | "Summarize": false 9 | }, 10 | "WafIPSet": { 11 | "Enable": true, 12 | "Summarize": false, 13 | "Scopes": ["REGIONAL"] 14 | } 15 | }, 16 | { 17 | "Name": "CLOUDFRONT_ORIGIN_FACING", 18 | "Regions": [], 19 | "PrefixList": { 20 | "Enable": false, 21 | "Summarize": false 22 | }, 23 | "WafIPSet": { 24 | "Enable": true, 25 | "Summarize": false, 26 | "Scopes": ["REGIONAL"] 27 | } 28 | }, 29 | { 30 | "Name": "EC2_INSTANCE_CONNECT", 31 | "Regions": ["sa-east-1"], 32 | "PrefixList": { 33 | "Enable": true, 34 | "Summarize": false 35 | }, 36 | "WafIPSet": { 37 | "Enable": true, 38 | "Summarize": false, 39 | "Scopes": ["REGIONAL"] 40 | } 41 | }, 42 | { 43 | "Name": "ROUTE53_HEALTHCHECKS", 44 | "Regions": [], 45 | "PrefixList": { 46 | "Enable": false, 47 | "Summarize": false 48 | }, 49 | "WafIPSet": { 50 | "Enable": true, 51 | "Summarize": true, 52 | "Scopes": ["REGIONAL"] 53 | } 54 | } 55 | ] 56 | } -------------------------------------------------------------------------------- /lambda/test_event.json: -------------------------------------------------------------------------------- 1 | { 2 | "Records": [ 3 | { 4 | "EventVersion": "1.0", 5 | "EventSubscriptionArn": "arn:aws:sns:EXAMPLE", 6 | "EventSource": "aws:sns", 7 | "Sns": { 8 | "SignatureVersion": "1", 9 | "Timestamp": "1970-01-01T00:00:00.000Z", 10 | "Signature": "EXAMPLE", 11 | "SigningCertUrl": "EXAMPLE", 12 | "MessageId": "12345678-1234-1234-1234-123456789012", 13 | "Message": "{\"create-time\": \"yyyy-mm-ddThh:mm:ss+00:00\", \"synctoken\": \"0123456789\", \"md5\": \"test-hash\", \"url\": \"https://ip-ranges.amazonaws.com/ip-ranges.json\"}", 14 | "Type": "Notification", 15 | "UnsubscribeUrl": "EXAMPLE", 16 | "TopicArn": "arn:aws:sns:EXAMPLE", 17 | "Subject": "TestInvoke" 18 | } 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /lambda/update_aws_ip_ranges.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | SPDX-License-Identifier: MIT-0 4 | """ 5 | 6 | import hashlib 7 | import ipaddress 8 | import json 9 | import logging 10 | import os 11 | import time 12 | from dataclasses import asdict, dataclass, field 13 | from datetime import datetime, timezone 14 | from typing import Any, Union 15 | from urllib import request 16 | 17 | import boto3 # type: ignore 18 | 19 | #### 20 | 21 | DESCRIPTION = 'Managed by update IP ranges Lambda' 22 | RESOURCE_NAME_PREFIX = 'aws-ip-ranges' 23 | MANAGED_BY = 'update-aws-ip-ranges' 24 | 25 | ####### Get values from environment variables ###### 26 | 27 | ## Logging level options in less verbosity order. INFO is the default. 28 | ## If you enable DEBUG, it will log boto3 calls as well. 29 | # CRITICAL 30 | # ERROR 31 | # WARNING 32 | # INFO 33 | # DEBUG 34 | 35 | # Get logging level from environment variable 36 | if (LOG_LEVEL := os.getenv('LOG_LEVEL', '').upper()) not in {'CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'}: 37 | LOG_LEVEL = 'INFO' 38 | 39 | # Set up logging. Set the level if the handler is already configured. 40 | if len(logging.getLogger().handlers) > 0: 41 | logging.getLogger().setLevel(LOG_LEVEL) 42 | else: 43 | logging.basicConfig(level=LOG_LEVEL) 44 | 45 | # Define client for services 46 | waf_client = boto3.client('wafv2') 47 | ec2_client = boto3.client('ec2') 48 | 49 | 50 | #====================================================================================================================== 51 | # Data classes and help functions 52 | #====================================================================================================================== 53 | @dataclass 54 | class IPv4List(): 55 | """List of IPv4 Networks""" 56 | 57 | ip_list: list[str] = field(default_factory=list) 58 | _summarized_ip_list: list[str] = field(default_factory=list) 59 | 60 | def summarized(self) -> list[str]: 61 | """Summarize this list as IPv4 network and sort it""" 62 | if not self._summarized_ip_list: 63 | if len(self.ip_list) == 1: 64 | return self.ip_list 65 | 66 | summarized_sorted = sorted( 67 | ipaddress.collapse_addresses( 68 | [ipaddress.IPv4Network(addr) for addr in self.ip_list] 69 | ) 70 | ) 71 | self._summarized_ip_list = [net.with_prefixlen for net in summarized_sorted] 72 | return self._summarized_ip_list 73 | 74 | def sort(self) -> None: 75 | """Sort this list as IPv4 network""" 76 | if len(self.ip_list) > 1: 77 | self.ip_list = sorted(self.ip_list, key = ipaddress.IPv4Network) 78 | 79 | def asdict(self) -> dict: 80 | """Return a dictionary from this object""" 81 | return asdict(self) 82 | 83 | @dataclass 84 | class IPv6List(): 85 | """List of IPv6 Networks""" 86 | 87 | ip_list: list[str] = field(default_factory=list) 88 | _summarized_ip_list: list[str] = field(default_factory=list) 89 | 90 | def summarized(self) -> list[str]: 91 | """Summarize this list as IPv6 network and sort it""" 92 | if not self._summarized_ip_list: 93 | summarized_sorted = sorted( 94 | ipaddress.collapse_addresses( 95 | [ipaddress.IPv6Network(addr) for addr in self.ip_list] 96 | ) 97 | ) 98 | self._summarized_ip_list = [net.exploded for net in summarized_sorted] 99 | return self._summarized_ip_list 100 | 101 | def sort(self) -> None: 102 | """Sort this list as IPv6 network""" 103 | if len(self.ip_list) > 1: 104 | sorted_list = sorted([ipaddress.IPv6Network(addr) for addr in self.ip_list]) 105 | self.ip_list = [net.exploded for net in sorted_list] 106 | 107 | def asdict(self) -> dict: 108 | """Return a dictionary from this object""" 109 | return asdict(self) 110 | 111 | @dataclass 112 | class ServiceIPRange(): 113 | """Store IPv4 and IPv6 networks""" 114 | ipv4: IPv4List = field(default_factory=IPv4List) 115 | ipv6: IPv6List = field(default_factory=IPv6List) 116 | 117 | def asdict(self) -> dict[str, Union[IPv4List, IPv6List]]: 118 | """Return a dictionary from this object""" 119 | return {'ipv4': self.ipv4, 'ipv6': self.ipv6} 120 | 121 | 122 | ### General functions 123 | def get_ip_groups_json(url: str, expected_hash: str) -> str: 124 | """Get ip-range.json file and check if it mach the expected MD5 hash""" 125 | logging.info('get_ip_groups_json start') 126 | logging.debug(f'Parameter url: {url}') 127 | logging.debug(f'Parameter expected_hash: {expected_hash}') 128 | 129 | # Get ip-ranges.json file 130 | logging.info(f'Updating from "{url}"') 131 | if not url.lower().startswith('http'): 132 | raise Exception(f'Expecting an HTTP protocol URL, got "{url}"') 133 | req = request.Request(url) 134 | with request.urlopen(req) as response: #nosec B310 135 | ip_json = response.read() 136 | logging.info(f'Got "ip-ranges.json" file from "{url}"') 137 | logging.debug(f'File content: {ip_json}') 138 | 139 | # Calculate MD5 hash from current file 140 | m = hashlib.md5() # nosec B303 141 | m.update(ip_json) 142 | current_hash = m.hexdigest() 143 | logging.debug(f'Calculated MD5 file hash "{current_hash}"') 144 | 145 | # If the hash provided is 'test-hash', returns the JSON without checking the hash 146 | if expected_hash == 'test-hash': 147 | logging.info('Running in test mode') 148 | return ip_json 149 | 150 | # Current file hash MUST match the one expected 151 | if current_hash != expected_hash: 152 | raise Exception(f'MD5 Mismatch: got "{current_hash}" expected "{expected_hash}"') 153 | 154 | logging.debug(f'Function return: {ip_json}') 155 | logging.info('get_ip_groups_json end') 156 | return ip_json 157 | 158 | def get_ranges_for_service(ranges: dict, config_services: dict) -> dict[str, ServiceIPRange]: 159 | """Gets IPv4 and IPv6 prefixes from the matching services""" 160 | logging.info('get_ranges_for_service start') 161 | logging.debug(f'Parameter ranges: {ranges}') 162 | logging.debug(f'Parameter config_services: {config_services}') 163 | 164 | service_ranges: dict[str, ServiceIPRange] = {} 165 | service_control: dict[str, bool] = {} 166 | for config_service in config_services['Services']: 167 | service_name = config_service['Name'] 168 | if len(config_service['Regions']) > 0: 169 | for region in config_service['Regions']: 170 | logging.info(f"Will search for '{service_name}' and region '{region}'") 171 | key = f"{service_name}-{region}" 172 | 173 | service_control[key] = True 174 | service_ranges[service_name] = ServiceIPRange() 175 | else: 176 | logging.info(f"Will search for '{service_name}' without consider a region, so will get all prefixes from all regions") 177 | key = f"{service_name}" 178 | 179 | service_control[key] = True 180 | service_ranges[service_name] = ServiceIPRange() 181 | 182 | # Loop over the IPv4 prefixes and appends the matching services 183 | logging.info('Searching for IPv4 prefixes') 184 | for prefix in ranges['prefixes']: 185 | service_name = prefix['service'] 186 | service_region = prefix['region'] 187 | key = f"{service_name}-{service_region}" 188 | if key in service_control: 189 | logging.info(f"Found service: '{service_name}' region: '{service_region}' range: {prefix['ip_prefix']}") 190 | service_ranges[service_name].ipv4.ip_list.append(prefix['ip_prefix']) 191 | else: 192 | key = f"{service_name}" 193 | if key in service_control: 194 | logging.info(f"Found service: '{service_name}' range: {prefix['ip_prefix']}") 195 | service_ranges[service_name].ipv4.ip_list.append(prefix['ip_prefix']) 196 | 197 | # Loop over the IPv6 prefixes and appends the matching services 198 | logging.info('Searching for IPv6 prefixes') 199 | for ipv6_prefix in ranges['ipv6_prefixes']: 200 | service_name = ipv6_prefix['service'] 201 | service_region = ipv6_prefix['region'] 202 | key = f"{service_name}-{service_region}" 203 | if key in service_control: 204 | logging.info(f"Found service: '{service_name}' region: '{service_region}' range: {ipv6_prefix['ipv6_prefix']}") 205 | service_ranges[service_name].ipv6.ip_list.append(ipv6_prefix['ipv6_prefix']) 206 | else: 207 | key = f"{service_name}" 208 | if key in service_control: 209 | logging.info(f"Found service: '{service_name}' range: {ipv6_prefix['ipv6_prefix']}") 210 | service_ranges[service_name].ipv6.ip_list.append(ipv6_prefix['ipv6_prefix']) 211 | 212 | # Sort all ranges 213 | for service_name in service_ranges.keys(): 214 | service_ranges[service_name].ipv4.sort() 215 | service_ranges[service_name].ipv6.sort() 216 | 217 | logging.debug(f'Function return: {service_ranges}') 218 | logging.info('get_ranges_for_service end') 219 | return service_ranges 220 | 221 | 222 | ### WAF IPSet functions 223 | def manage_waf_ipset(client: Any, waf_ipsets: dict[str, dict], service_name: str, ipset_scope: str, service_ranges: dict[str, ServiceIPRange], should_summarize: bool) -> dict[str, list[str]]: 224 | """Create or Update WAF IPSet""" 225 | logging.info('manage_waf_ipset start') 226 | logging.debug(f'Parameter client: {client}') 227 | logging.debug(f'Parameter waf_ipsets: {waf_ipsets}') 228 | logging.debug(f'Parameter service_name: {service_name}') 229 | logging.debug(f'Parameter ipset_scope: {ipset_scope}') 230 | logging.debug(f'Parameter service_ranges: {service_ranges}') 231 | logging.debug(f'Parameter should_summarize: {should_summarize}') 232 | 233 | # Dictionary to return the IPSet names that will be created or updated 234 | ipset_names: dict[str, list[str]] = {'created': [], 'updated': []} 235 | 236 | for ip_version in ['ipv4', 'ipv6']: 237 | if not service_ranges[service_name].asdict()[ip_version].ip_list: 238 | logging.debug(f'No IP ranges found for service "{service_name}" and IP version "{ip_version}"') 239 | else: 240 | # Found ranges for specific IP version (ipv4 or ipv6) 241 | address_list: list[str] = service_ranges[service_name].asdict()[ip_version].ip_list 242 | logging.debug(f'Summarize: "{should_summarize}" and address list lenght: "{len(address_list)}"') 243 | if should_summarize and (len(address_list) > 1): 244 | address_list = service_ranges[service_name].asdict()[ip_version].summarized() 245 | 246 | # Check if it is to create or update WAF IPSet 247 | ipset_name: str = f"{RESOURCE_NAME_PREFIX}-{service_name.lower().replace('_', '-')}-{ip_version}" 248 | if ipset_name in waf_ipsets: 249 | # IPSets exists, so will update it 250 | logging.debug(f'WAF IPSet "{ipset_name}" found. Will update it.') 251 | updated: bool = update_waf_ipset(client, ipset_name, ipset_scope, waf_ipsets[ipset_name], address_list) 252 | if updated: 253 | ipset_names['updated'].append(ipset_name) 254 | else: 255 | # IPSet not found, so will create it 256 | # IPAddressVersion can be 'IPV4' or 'IPV6' 257 | logging.debug(f'WAF IPSet "{ipset_name}" not found. Will create it.') 258 | create_waf_ipset(client, ipset_name, ipset_scope, ip_version.upper(), address_list) 259 | ipset_names['created'].append(ipset_name) 260 | 261 | logging.debug(f'Function return: {ipset_names}') 262 | logging.info('manage_waf_ipset end') 263 | return ipset_names 264 | 265 | def create_waf_ipset(client: Any, ipset_name: str, ipset_scope: str, ipset_version: str, address_list: list[str]) -> None: 266 | """Create the AWS WAF IP set""" 267 | logging.info('create_waf_ipset start') 268 | logging.debug(f'Parameter client: {client}') 269 | logging.debug(f'Parameter ipset_name: {ipset_name}') 270 | logging.debug(f'Parameter ipset_scope: {ipset_scope}') 271 | logging.debug(f'Parameter ipset_version: {ipset_version}') 272 | logging.debug(f'Parameter address_list: {address_list}') 273 | 274 | logging.info(f'Creating IPSet "{ipset_name}" with scope "{ipset_scope}" with {len(address_list)} CIDRs. List: {address_list}') 275 | response = client.create_ip_set( 276 | Name=ipset_name, 277 | Scope=ipset_scope, 278 | Description=DESCRIPTION, 279 | IPAddressVersion=ipset_version, 280 | Addresses=address_list, 281 | Tags=[ 282 | { 283 | 'Key': 'Name', 284 | 'Value': ipset_name 285 | }, 286 | { 287 | 'Key': 'ManagedBy', 288 | 'Value': MANAGED_BY 289 | }, 290 | { 291 | 'Key': 'CreatedAt', 292 | 'Value': datetime.now(timezone.utc).isoformat() 293 | }, 294 | { 295 | 'Key': 'UpdatedAt', 296 | 'Value': 'Not yet' 297 | }, 298 | ] 299 | ) 300 | logging.info(f'Created IPSet "{ipset_name}"') 301 | logging.debug(f'Response: {response}') 302 | logging.debug('Function return: None') 303 | logging.info('create_waf_ipset end') 304 | 305 | def update_waf_ipset(client: Any, ipset_name: str, ipset_scope: str, waf_ipset: dict[str, Any], address_list: list[str]) -> bool: 306 | """Updates the AWS WAF IP set""" 307 | logging.info('update_waf_ipset start') 308 | logging.debug(f'Parameter client: {client}') 309 | logging.debug(f'Parameter ipset_name: {ipset_name}') 310 | logging.debug(f'Parameter ipset_scope: {ipset_scope}') 311 | logging.debug(f'Parameter waf_ipset: {waf_ipset}') 312 | logging.debug(f'Parameter address_list: {address_list}') 313 | 314 | ipset_id: str = waf_ipset['Id'] 315 | ipset_lock_token: str = waf_ipset['LockToken'] 316 | ipset_description: str = waf_ipset['Description'] 317 | ipset_arn: str = waf_ipset['ARN'] 318 | logging.debug(f'ipset_id: {ipset_id}') 319 | logging.debug(f'ipset_lock_token: {ipset_lock_token}') 320 | logging.debug(f'ipset_description: {ipset_description}') 321 | logging.debug(f'ipset_arn: {ipset_arn}') 322 | 323 | # It uses entries_to_remove and entries_to_add just for control if it needs to update IPSet or not. 324 | # If it needs to update, it will always use the full list of addresses from address_list 325 | 326 | network_list: list[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]] = [ipaddress.ip_network(net) for net in address_list] 327 | # Get current IPSet entries 328 | current_entries: dict[Union[ipaddress.IPv4Network, ipaddress.IPv6Network], str] = get_ip_set_entries(client, ipset_name, ipset_scope, ipset_id) 329 | # Filter to get the list of entries to remove from IPSet 330 | entries_to_remove: list[str] = [cidr.with_prefixlen for cidr in current_entries.keys() if cidr not in network_list] 331 | # Filter to get the list of entries to add into Prefix List 332 | entries_to_add: list[str] = [cidr.with_prefixlen for cidr in network_list if cidr not in current_entries] 333 | logging.debug(f'current_entries: {current_entries}') 334 | logging.debug(f'entries_to_remove: {entries_to_remove}') 335 | logging.debug(f'entries_to_add: {entries_to_add}') 336 | 337 | updated: bool = False 338 | if (not entries_to_add) and (not entries_to_remove): 339 | logging.info(f'Nothing to add or remove at "{ipset_name}"') 340 | else: 341 | # Update IPSet 342 | logging.info(f'Updating IPSet "{ipset_name}" with scope "{ipset_scope}" with lock_token "{ipset_lock_token}" with {len(address_list)} CIDRs. List: {address_list}') 343 | response = client.update_ip_set( 344 | Name=ipset_name, 345 | Scope=ipset_scope, 346 | Id=ipset_id, 347 | Description=ipset_description, 348 | Addresses=address_list, 349 | LockToken=ipset_lock_token 350 | ) 351 | updated = True 352 | logging.info(f'Updated IPSet "{ipset_name}"') 353 | logging.debug(f'Response: {response}') 354 | 355 | # Update IPSet tags 356 | logging.info(f'Updating Tags for "{ipset_name}" with ARN "{ipset_arn}"') 357 | response = client.tag_resource( 358 | ResourceARN=ipset_arn, 359 | Tags=[{'Key': 'UpdatedAt','Value': datetime.now(timezone.utc).isoformat()}] 360 | ) 361 | logging.info(f'Updated Tags for "{ipset_name}" with ARN {ipset_arn}') 362 | logging.debug(f'Response: {response}') 363 | 364 | logging.debug(f'Function return: {updated}') 365 | logging.info('update_waf_ipset end') 366 | return updated 367 | 368 | def list_waf_ipset(client: Any, ipset_scope: str) -> dict[str, dict]: 369 | """List all AWS WAF IP set from specific scope""" 370 | logging.info('list_waf_ipset start') 371 | logging.debug(f'Parameter client: {client}') 372 | logging.debug(f'Parameter ipset_scope: {ipset_scope}') 373 | 374 | # Put all IPSets inside a dictionary 375 | ipsets: dict[str, dict] = {} 376 | 377 | response = client.list_ip_sets(Scope=ipset_scope) 378 | logging.info(f'Listed IPSet with scope "{ipset_scope}"') 379 | logging.debug(f'Response: {response}') 380 | while True: 381 | for ipset in response['IPSets']: 382 | ipsets[ipset['Name']] = ipset 383 | 384 | if not 'NextMarker' in response: 385 | break 386 | 387 | # As there is a NextMarket it needs to perform the list call again 388 | next_marker = response['NextMarker'] 389 | logging.info(f'Found NextMarker "{next_marker}"') 390 | response = client.list_ip_sets(Scope=ipset_scope, NextMarker=next_marker) 391 | logging.info(f'Listed IPSet with scope "{ipset_scope}" and NextMarker "{next_marker}"') 392 | logging.debug(f'Response: {response}') 393 | 394 | 395 | logging.debug(f'Function return: {ipsets}') 396 | logging.info('list_waf_ipset end') 397 | return ipsets 398 | 399 | def get_ip_set_entries(client: Any, ipset_name: str, ipset_scope: str, ipset_id: str) -> dict[Union[ipaddress.IPv4Network, ipaddress.IPv6Network], str]: 400 | """Get AWS WAF IP set entries""" 401 | logging.info('get_ip_set_entries start') 402 | logging.debug(f'Parameter client: {client}') 403 | logging.debug(f'Parameter ipset_name: {ipset_name}') 404 | logging.debug(f'Parameter ipset_scope: {ipset_scope}') 405 | logging.debug(f'Parameter ipset_id: {ipset_id}') 406 | 407 | response = client.get_ip_set( 408 | Name=ipset_name, 409 | Scope=ipset_scope, 410 | Id=ipset_id 411 | ) 412 | logging.info(f'Got IPSet with name "{ipset_name}" and scope "{ipset_scope}" and ID "{ipset_id}"') 413 | logging.debug(f'Response: {response}') 414 | 415 | # Add entries in a dictionary 416 | entries: dict[Union[ipaddress.IPv4Network, ipaddress.IPv6Network], str] = {} 417 | for entrie in response['IPSet']['Addresses']: 418 | network = ipaddress.ip_network(entrie) 419 | entries[network] = entrie 420 | logging.debug(f'Function return: {entries}') 421 | logging.info('get_ip_set_entries end') 422 | return entries 423 | 424 | 425 | ### VPC Prefix List 426 | def manage_prefix_list(client: Any, vpc_prefix_lists: dict[str, dict], service_name: str, service_ranges: dict[str, ServiceIPRange], should_summarize: bool) -> dict[str, list[str]]: 427 | """Create or Update VPC Prefix List""" 428 | logging.info('manage_prefix_list start') 429 | logging.debug(f'Parameter client: {client}') 430 | logging.debug(f'Parameter vpc_prefix_lists: {vpc_prefix_lists}') 431 | logging.debug(f'Parameter service_name: {service_name}') 432 | logging.debug(f'Parameter service_ranges: {service_ranges}') 433 | logging.debug(f'Parameter should_summarize: {should_summarize}') 434 | 435 | # Dictionary to return the Prefix List names that will be created or updated 436 | prefix_list_names: dict[str, list[str]] = {'created': [], 'updated': []} 437 | 438 | for ip_version in ['ipv4', 'ipv6']: 439 | if not service_ranges[service_name].asdict()[ip_version].ip_list: 440 | logging.debug(f'No IP ranges found for service "{service_name}" and IP version "{ip_version}"') 441 | else: 442 | # Found ranges for specific IP version (ipv4 or ipv6) 443 | address_list: list[str] = service_ranges[service_name].asdict()[ip_version].ip_list 444 | logging.debug(f'Summarize: "{should_summarize}" and address list lenght: "{len(address_list)}"') 445 | if should_summarize and (len(address_list) > 1): 446 | address_list = service_ranges[service_name].asdict()[ip_version].summarized() 447 | 448 | prefix_list_name: str = f"{RESOURCE_NAME_PREFIX}-{service_name.lower().replace('_', '-')}-{ip_version}" 449 | if prefix_list_name in vpc_prefix_lists: 450 | # Prefix List exists, so will update it 451 | logging.debug(f'VPC Prefix List "{prefix_list_name}" found. Will update it.') 452 | updated: bool = update_prefix_list(client, prefix_list_name, vpc_prefix_lists[prefix_list_name], address_list) 453 | if updated: 454 | prefix_list_names['updated'].append(prefix_list_name) 455 | else: 456 | # Prefix List not found, so will create it 457 | logging.debug(f'VPC Prefix List "{prefix_list_name}" not found. Will create it.') 458 | create_prefix_list(client, prefix_list_name, ip_version.upper(), address_list) 459 | prefix_list_names['created'].append(prefix_list_name) 460 | 461 | logging.debug(f'Function return: {prefix_list_names}') 462 | logging.info('manage_prefix_list end') 463 | return prefix_list_names 464 | 465 | def list_prefix_lists(client: Any) -> dict[str, dict]: 466 | """List all VPC Prefix List""" 467 | logging.info('list_prefix_lists start') 468 | logging.debug(f'Parameter client: {client}') 469 | 470 | # Put all VPC Prefix Lists inside a dictionary 471 | prefix_lists: dict[str, dict] = {} 472 | 473 | response = client.describe_managed_prefix_lists() 474 | logging.info('Listed VPC Prefix Lists') 475 | logging.debug(f'Response: {response}') 476 | while True: 477 | for prefix_list in response['PrefixLists']: 478 | prefix_lists[prefix_list['PrefixListName']] = prefix_list 479 | 480 | if not 'NextToken' in response: 481 | break 482 | 483 | # As there is a NextToken it needs to perform the list call again 484 | next_token = response['NextToken'] 485 | logging.info(f'Found NextToken "{next_token}"') 486 | response = client.describe_managed_prefix_lists(NextToken=next_token) 487 | logging.info(f'Listed VPC Prefix Lists with NextToken "{next_token}"') 488 | logging.debug(f'Response: {response}') 489 | 490 | logging.debug(f'Function return: {prefix_lists}') 491 | logging.info('list_prefix_lists end') 492 | return prefix_lists 493 | 494 | def get_prefix_list_by_id(client: Any, prefix_list_id: str) -> dict[str, Any]: 495 | """Get VPC Prefix List by ID""" 496 | logging.info('get_prefix_list_by_id start') 497 | logging.debug(f'Parameter client: {client}') 498 | logging.debug(f'Parameter prefix_list_id: {prefix_list_id}') 499 | 500 | response = client.describe_managed_prefix_lists(PrefixListIds=[prefix_list_id]) 501 | logging.info(f'Got VPC Prefix Lists with ID: {prefix_list_id}') 502 | logging.debug(f'Response: {response}') 503 | 504 | prefix_list: dict[str, Any] = response['PrefixLists'][0] 505 | logging.debug(f'Function return: {prefix_list}') 506 | logging.info('get_prefix_list_by_id end') 507 | return prefix_list 508 | 509 | def create_prefix_list(client: Any, prefix_list_name: str, prefix_list_ip_version: str, address_list: list[str]) -> None: 510 | """Create the VPC Prefix List""" 511 | logging.info('create_prefix_list start') 512 | logging.debug(f'Parameter client: {client}') 513 | logging.debug(f'Parameter prefix_list_name: {prefix_list_name}') 514 | logging.debug(f'Parameter prefix_list_ip_version: {prefix_list_ip_version}') 515 | logging.debug(f'Parameter address_list: {address_list}') 516 | 517 | # Create the list of Prefix List entries to create 518 | prefix_list_entries: list[dict] = [] 519 | for addr in address_list: 520 | prefix_list_entries.append( 521 | { 522 | 'Cidr': addr, 523 | 'Description': DESCRIPTION 524 | } 525 | ) 526 | logging.debug(f'Prefix List entries: {prefix_list_entries}') 527 | 528 | # Add 10 enties extra when create Prefix List for future expansion 529 | max_entries: int = len(prefix_list_entries) + 10 530 | 531 | logging.info(f'Creating VPC Prefix List "{prefix_list_name}" with max entries "{max_entries}" with address family "{prefix_list_ip_version}" with {len(address_list)} CIDRs. List: {address_list}') 532 | logging.info(f'VPC Prefix List entries in this call: {prefix_list_entries[0:100]}') 533 | response = client.create_managed_prefix_list( 534 | PrefixListName=prefix_list_name, 535 | Entries=prefix_list_entries[0:100], 536 | MaxEntries=max_entries, 537 | TagSpecifications=[ 538 | { 539 | 'ResourceType': 'prefix-list', 540 | 'Tags': [ 541 | { 542 | 'Key': 'Name', 543 | 'Value': prefix_list_name 544 | }, 545 | { 546 | 'Key': 'ManagedBy', 547 | 'Value': MANAGED_BY 548 | }, 549 | { 550 | 'Key': 'CreatedAt', 551 | 'Value': datetime.now(timezone.utc).isoformat() 552 | }, 553 | { 554 | 'Key': 'UpdatedAt', 555 | 'Value': 'Not yet' 556 | }, 557 | ] 558 | }, 559 | ], 560 | AddressFamily=prefix_list_ip_version 561 | ) 562 | logging.info(f'Created VPC Prefix List "{prefix_list_name}"') 563 | logging.debug(f'Response: {response}') 564 | 565 | # Boto3: The number of entries per request cannot exceeds limit (100). 566 | # So, if it is greater than 100, it needs to be split in multiple requests 567 | for index in range(100, len(prefix_list_entries), 100): 568 | 569 | if response['PrefixList']['State'] in {'create-in-progress', 'modify-in-progress'}: 570 | logging.info('Creating VPC Prefix List is in progress. Will wait.') 571 | for count in range(5): 572 | seconds_to_wait: int = count + (count + 1) 573 | logging.info(f'Waiting {seconds_to_wait} seconds') 574 | time.sleep(seconds_to_wait) 575 | wait_prefix_list: dict[str, Any] = get_prefix_list_by_id(client, response['PrefixList']['PrefixListId']) 576 | if wait_prefix_list['State'] in {'create-complete', 'modify-complete'}: 577 | break 578 | else: 579 | # Else doesn't execute if exit via break 580 | raise Exception("Error creating/updating VPC Prefix List. Can't wait anymore.") 581 | 582 | logging.info(f'Updating VPC Prefix List "{prefix_list_name}" with max entries "{max_entries}" with address family "{prefix_list_ip_version}" with {len(address_list)} CIDRs. Starting from index "{index}".') 583 | logging.info(f'VPC Prefix List entries in this call: {prefix_list_entries[index:index+100]}') 584 | response = client.modify_managed_prefix_list( 585 | PrefixListId=response['PrefixList']['PrefixListId'], 586 | CurrentVersion=response['PrefixList']['Version'], 587 | PrefixListName=response['PrefixList']['PrefixListName'], 588 | AddEntries=prefix_list_entries[index:index+100] 589 | ) 590 | logging.info(f'Updated VPC Prefix List "{prefix_list_name}"') 591 | logging.debug(f'Response: {response}') 592 | 593 | logging.debug('Function return: None') 594 | logging.info('create_prefix_list end') 595 | 596 | def update_prefix_list(client: Any, prefix_list_name: str, prefix_list: dict[str, Any], address_list: list[str]) -> bool: 597 | """Updates the AWS VPC Prefix List""" 598 | logging.info('update_prefix_list start') 599 | logging.debug(f'Parameter client: {client}') 600 | logging.debug(f'Parameter prefix_list_name: {prefix_list_name}') 601 | logging.debug(f'Parameter prefix_list: {prefix_list}') 602 | logging.debug(f'Parameter address_list: {address_list}') 603 | 604 | prefix_list_id: str = prefix_list['PrefixListId'] 605 | prefix_list_max_entries: int = prefix_list['MaxEntries'] 606 | prefix_list_version: int = prefix_list['Version'] 607 | logging.debug(f'prefix_list_id: {prefix_list_id}') 608 | logging.debug(f'prefix_list_max_entries: {prefix_list_max_entries}') 609 | logging.debug(f'prefix_list_version: {prefix_list_version}') 610 | 611 | network_list = [ipaddress.ip_network(net) for net in address_list] 612 | # Get current Prefix List entries 613 | current_entries: dict[Union[ipaddress.IPv4Network, ipaddress.IPv6Network], str] = get_prefix_list_entries(client, prefix_list_name, prefix_list_id, prefix_list_version) 614 | # Filter to get the list of entries to remove from Prefix List 615 | entries_to_remove: list[dict] = [{'Cidr': cidr.with_prefixlen} for cidr in current_entries.keys() if cidr not in network_list] 616 | # Filter to get the list of entries to add into Prefix List 617 | entries_to_add: list[dict] = [{'Cidr': cidr.with_prefixlen, 'Description': DESCRIPTION} for cidr in network_list if cidr not in current_entries] 618 | logging.debug(f'current_entries: {current_entries}') 619 | logging.debug(f'entries_to_remove: {entries_to_remove}') 620 | logging.debug(f'entries_to_add: {entries_to_add}') 621 | 622 | updated: bool = False 623 | if (not entries_to_add) and (not entries_to_remove): 624 | logging.info(f'Nothing to add or remove at "{prefix_list_name}"') 625 | else: 626 | # Only change max entries if there is entries to add and current max entries is less than len of new address list 627 | if entries_to_add: 628 | logging.debug('Exist entries to add') 629 | if prefix_list_max_entries < len(address_list): 630 | logging.debug(f'Current Prefix List max entries "{prefix_list_max_entries}" is lower than new range length "{len(address_list)}", so will change it to increase') 631 | # Update Prefix List to change max entries first 632 | # You cannot modify the entries of a prefix list and modify the size of a prefix list at the same time. 633 | prefix_list_max_entries = len(address_list) 634 | logging.debug(f'New max entries value "{prefix_list_max_entries}"') 635 | 636 | logging.info(f'Updating VPC Prefix List "{prefix_list_name}" with id "{prefix_list_id}" with max entries "{prefix_list_max_entries}" with version "{prefix_list_version}"') 637 | response = client.modify_managed_prefix_list( 638 | PrefixListId=prefix_list_id, 639 | PrefixListName=prefix_list_name, 640 | MaxEntries=prefix_list_max_entries 641 | ) 642 | logging.info(f'Updated VPC Prefix List "{prefix_list_name}"') 643 | logging.debug(f'Response: {response}') 644 | 645 | prefix_list_version = response['PrefixList']['Version'] 646 | current_state: str = response['PrefixList']['State'] 647 | current_state_message: str = response['PrefixList']['StateMessage'] 648 | logging.debug(f'prefix_list_version: {prefix_list_version}') 649 | logging.debug(f'current_state: {current_state}') 650 | logging.debug(f'current_state_message: {current_state_message}') 651 | 652 | if current_state not in ['modify-in-progress', 'modify-complete', 'modify-failed']: 653 | raise Exception(f'Error updating VPC Prefix List max entries. Invalid state. Expecting "modify-in-progress" or "modify-complete" or "modify-failed". Got "{current_state}". Name: "{prefix_list_name}" ID: "{prefix_list_id}" current version: "{prefix_list_version}" new max entries: "{prefix_list_max_entries}" StateMessage: "{current_state_message}"') 654 | if current_state == 'modify-failed': 655 | raise Exception(f'Error updating VPC Prefix List max entries. State is "modify-failed". Name: "{prefix_list_name}" ID: "{prefix_list_id}" current version: "{prefix_list_version}" new max entries: "{prefix_list_max_entries}" StateMessage: "{current_state_message}"') 656 | if current_state == 'modify-in-progress': 657 | logging.info('Updating VPC Prefix List max entries is in progress. Will wait.') 658 | for count in range(5): 659 | seconds_to_wait: int = count + (count + 1) 660 | logging.info(f'Waiting {seconds_to_wait} seconds') 661 | time.sleep(seconds_to_wait) 662 | wait_prefix_list: dict[str, Any] = get_prefix_list_by_id(client, prefix_list_id) 663 | if wait_prefix_list['State'] == 'modify-complete': 664 | break 665 | else: 666 | # Else doesn't execute if exit via break 667 | raise Exception("Error updating VPC Prefix List max entries. Can't wait anymore.") 668 | 669 | # Update Prefix List entries 670 | logging.info(f'Updating VPC Prefix List "{prefix_list_name}" with id "{prefix_list_id}" with version "{prefix_list_version}" with {len(address_list)} CIDRs.') 671 | logging.info(f'Updating VPC Prefix List "{prefix_list_name}" Entries to add in this call: {entries_to_add[0:100]}') 672 | logging.info(f'Updating VPC Prefix List "{prefix_list_name}" Entries to remove in this call: {entries_to_remove[0:100]}') 673 | logging.info(f'Updating VPC Prefix List "{prefix_list_name}" Full list of entries: {address_list}') 674 | response = client.modify_managed_prefix_list( 675 | PrefixListId=prefix_list_id, 676 | CurrentVersion=prefix_list_version, 677 | PrefixListName=prefix_list_name, 678 | AddEntries=entries_to_add[0:100], 679 | RemoveEntries=entries_to_remove[0:100] 680 | ) 681 | updated = True 682 | logging.info(f'Updated VPC Prefix List "{prefix_list_name}"') 683 | logging.info(f'Response: {response}') 684 | 685 | # Boto3: Request cannot contain more than 100 entry additions or removals 686 | for index in range(100, max(len(entries_to_add), len(entries_to_remove)), 100): 687 | if response['PrefixList']['State'] in {'modify-in-progress'}: 688 | logging.info('Creating VPC Prefix List is in progress. Will wait.') 689 | for count in range(5): 690 | seconds_to_wait: int = count + (count + 1) 691 | logging.info(f'Waiting {seconds_to_wait} seconds') 692 | time.sleep(seconds_to_wait) 693 | wait_prefix_list: dict[str, Any] = get_prefix_list_by_id(client, response['PrefixList']['PrefixListId']) 694 | if wait_prefix_list['State'] in {'modify-complete'}: 695 | break 696 | else: 697 | # Else doesn't execute if exit via break 698 | raise Exception("Error updating VPC Prefix List. Can't wait anymore.") 699 | 700 | logging.info(f'Updating VPC Prefix List "{prefix_list_name}" with id "{prefix_list_id}" with version "{prefix_list_version}" with {len(address_list)} CIDRs. Starting from index "{index}".') 701 | logging.info(f'Updating VPC Prefix List "{prefix_list_name}" Entries to add in this call: {entries_to_add[index:index+100]}') 702 | logging.info(f'Updating VPC Prefix List "{prefix_list_name}" Entries to remove in this call: {entries_to_remove[index:index+100]}') 703 | response = client.modify_managed_prefix_list( 704 | PrefixListId=response['PrefixList']['PrefixListId'], 705 | CurrentVersion=response['PrefixList']['Version'], 706 | PrefixListName=response['PrefixList']['PrefixListName'], 707 | AddEntries=entries_to_add[0:100], 708 | RemoveEntries=entries_to_remove[0:100] 709 | ) 710 | logging.info(f'Updated VPC Prefix List "{prefix_list_name}"') 711 | logging.info(f'Response: {response}') 712 | 713 | # Update VPC Prefix List tags 714 | logging.info(f'Updating Tags for "{prefix_list_name}" with ID "{prefix_list_id}"') 715 | # Adds or overwrites only the specified tags for the specified Amazon EC2 resource or resources. 716 | response = client.create_tags( 717 | Resources=[prefix_list_id], 718 | Tags=[{'Key': 'UpdatedAt', 'Value': datetime.now(timezone.utc).isoformat()}] 719 | ) 720 | logging.info(f'Updated Tags for "{prefix_list_name}" with ID {prefix_list_id}') 721 | logging.debug(f'Response: {response}') 722 | 723 | logging.debug(f'Function return: {updated}') 724 | logging.info('update_prefix_list end') 725 | return updated 726 | 727 | def get_prefix_list_entries(client: Any, prefix_list_name: str, prefix_list_id: str, prefix_list_version: int) -> dict[Union[ipaddress.IPv4Network, ipaddress.IPv6Network], str]: 728 | """Get the AWS VPC Prefix List entries""" 729 | logging.info('get_prefix_list_entries start') 730 | logging.debug(f'Parameter client: {client}') 731 | logging.debug(f'Parameter prefix_list_name: {prefix_list_name}') 732 | logging.debug(f'Parameter prefix_list_id: {prefix_list_id}') 733 | logging.debug(f'Parameter prefix_list_version: {prefix_list_version}') 734 | 735 | logging.info(f'Getting VPC Prefix List entries "{prefix_list_name}" with id "{prefix_list_id}" with version "{prefix_list_version}"') 736 | response = client.get_managed_prefix_list_entries( 737 | PrefixListId=prefix_list_id, 738 | TargetVersion=prefix_list_version 739 | ) 740 | logging.info(f'Got VPC Prefix List entries "{prefix_list_name}"') 741 | logging.debug(f'Response: {response}') 742 | 743 | # Add entries in a dictionary 744 | entries: dict[Union[ipaddress.IPv4Network, ipaddress.IPv6Network], str] = {} 745 | for entrie in response['Entries']: 746 | network: Union[ipaddress.IPv4Network, ipaddress.IPv6Network] = ipaddress.ip_network(entrie['Cidr']) 747 | if 'Description' in entrie: 748 | entries[network] = entrie['Description'] 749 | else: 750 | entries[network] = '' 751 | 752 | while 'NextToken' in response: 753 | logging.info('Getting VPC Prefix List entries with NextToken') 754 | response = client.get_managed_prefix_list_entries( 755 | PrefixListId=prefix_list_id, 756 | TargetVersion=prefix_list_version, 757 | NextToken=response['NextToken'] 758 | ) 759 | logging.info(f'Got VPC Prefix List entries with NextToken "{prefix_list_name}"') 760 | logging.debug(f'Response: {response}') 761 | 762 | for entrie in response['Entries']: 763 | network: Union[ipaddress.IPv4Network, ipaddress.IPv6Network] = ipaddress.ip_network(entrie['Cidr']) 764 | if 'Description' in entrie: 765 | entries[network] = entrie['Description'] 766 | else: 767 | entries[network] = '' 768 | 769 | logging.debug(f'Function return: {entries}') 770 | logging.info('get_prefix_list_entries end') 771 | return entries 772 | 773 | 774 | 775 | #====================================================================================================================== 776 | # Lambda entry point 777 | #====================================================================================================================== 778 | 779 | def lambda_handler(event, context): 780 | """Lambda function handler""" 781 | logging.info('lambda_handler start') 782 | logging.debug(f'Parameter event: {event}') 783 | logging.debug(f'Parameter context: {context}') 784 | try: 785 | # Read config file to get services and regions 786 | with open('services.json', 'r', encoding='utf-8') as services_file: 787 | logging.info('Reading file "services.json"') 788 | config_services: dict[str, Any] = json.loads(services_file.read()) 789 | logging.info(f'Found services: {config_services}') 790 | 791 | message: dict[str, Any] = json.loads(event['Records'][0]['Sns']['Message']) 792 | logging.debug(f'Message from SNS topic: {message}') 793 | 794 | # Load the ip ranges from the url 795 | logging.debug(f'URL: {message["url"]}') 796 | logging.debug(f'MD5: {message["md5"]}') 797 | ip_ranges: dict[str, Any] = json.loads(get_ip_groups_json(message['url'], message['md5'])) 798 | logging.info('Got "ip-ranges.json" file') 799 | logging.info(f'SyncToken: {ip_ranges["syncToken"]}') 800 | logging.info(f'CreateDate: {ip_ranges["createDate"]}') 801 | logging.debug(f'File content: {ip_ranges}') 802 | 803 | # Extract the service ranges 804 | # Each service name from config file will be a key on returned dictionary 805 | # service_ranges => dict[str, ServiceIPRange] 806 | # keys 'ipv4' and 'ipv6' will ALWAYS be a valid list. 807 | # If no IP range is found, it will be an empty list 808 | # Example: 809 | # { 810 | # 'CLOUDFRONT': { 811 | # 'ipv4': [ 812 | # '1.1.1.1/32', 813 | # '2.2.2.2/32' 814 | # ], 815 | # 'ipv6': [ 816 | # 'aaaa::1/56' 817 | # ] 818 | # }, 819 | # 'API_GATEWAY': { 820 | # 'ipv4': [ 821 | # '3.3.3.3/32' 822 | # ], 823 | # 'ipv6': [] 824 | # } 825 | # } 826 | service_ranges: dict[str, ServiceIPRange] = get_ranges_for_service(ip_ranges, config_services) 827 | logging.info(f'Service IP ranges keys: {service_ranges.keys()}') 828 | logging.debug(f'Dictionary with service IP ranges: {service_ranges}') 829 | 830 | # Dictonary to return from this function 831 | resource_names: dict[str, dict[str, list[str]]] = { 832 | 'PrefixList': {'created': [], 'updated':[]}, 833 | 'WafIPSet': {'created': [], 'updated':[]} 834 | } 835 | service_name: str = '' 836 | should_summarize: bool = False 837 | 838 | # Create or Update the appropriate resource for each service range found 839 | ### Prefix List 840 | logging.info('Handling VPC Prefix List') 841 | vpc_prefix_lists: dict[str, dict] = {} 842 | for config_service in config_services['Services']: 843 | service_name = config_service['Name'] 844 | 845 | logging.info(f'Start handle VPC Prefix List for "{service_name}"') 846 | if service_name not in service_ranges: 847 | logging.warning(f'Service name "{service_name}" not found in service ranges variable. This condition should NEVER happens. Possible bug in the code. Please investigate.') 848 | else: 849 | # Handle Prefix List 850 | if 'PrefixList' not in config_service: 851 | logging.info(f'Service "{service_name}" not configured with "PrefixList"') 852 | else: 853 | if not config_service['PrefixList']['Enable']: 854 | logging.info(f'Service "{service_name}" is configured with "PrefixList" but it is not enable') 855 | else: 856 | try: 857 | # Will list VPC Prefix Lists only once 858 | if not vpc_prefix_lists: 859 | logging.info('Will get the list of VPC Prefix List for the first time') 860 | vpc_prefix_lists = list_prefix_lists(ec2_client) 861 | 862 | should_summarize = config_service['PrefixList']['Summarize'] 863 | prefix_list_names: dict[str, list[str]] = manage_prefix_list(ec2_client, vpc_prefix_lists, service_name, service_ranges, should_summarize) 864 | resource_names['PrefixList']['created'] += prefix_list_names['created'] 865 | resource_names['PrefixList']['updated'] += prefix_list_names['updated'] 866 | except Exception as error: 867 | logging.error("Error handling VPC Prefix List. It will not block the execution. It will continue to other services and resources.") 868 | logging.exception(error) 869 | logging.info(f'Finish handle VPC Prefix List for "{service_name}"') 870 | 871 | # Create or Update the appropriate resource for each service range found 872 | ### WAF IPSet 873 | logging.info('Handling WAF IPSet') 874 | waf_ipsets_by_scope: dict[str, dict] = {} 875 | for config_service in config_services['Services']: 876 | service_name = config_service['Name'] 877 | 878 | logging.info(f'Start handle WAF IPSet for "{service_name}"') 879 | if service_name not in service_ranges: 880 | logging.warning(f'Service name "{service_name}" not found in service ranges variable. This condition should NEVER happens. Possible bug in the code. Please investigate.') 881 | else: 882 | # Handle WAF IPSet 883 | if 'WafIPSet' not in config_service: 884 | logging.info(f'Service "{service_name}" not configured with "WafIPSet"') 885 | else: 886 | if not config_service['WafIPSet']['Enable']: 887 | logging.info(f'Service "{service_name}" is configured with "WafIPSet" but it is not enable') 888 | else: 889 | # Scope can be 'CLOUDFRONT' or 'REGIONAL' 890 | for ipset_scope in config_service['WafIPSet']['Scopes']: 891 | logging.info(f'Service "{service_name}" WAF Scope "{ipset_scope}"') 892 | try: 893 | waf_ipsets: dict[str, dict] = {} 894 | # Will list WAF IPSets only once for each scope 895 | if ipset_scope in waf_ipsets_by_scope: 896 | waf_ipsets = waf_ipsets_by_scope[ipset_scope] 897 | else: 898 | logging.info(f'Will get the list of WAF IPSets for the first time for scope "{ipset_scope}"') 899 | waf_ipsets = list_waf_ipset(waf_client, ipset_scope) 900 | waf_ipsets_by_scope[ipset_scope] = waf_ipsets 901 | 902 | should_summarize = config_service['WafIPSet']['Summarize'] 903 | ipset_names: dict[str, list[str]] = manage_waf_ipset(waf_client, waf_ipsets, service_name, ipset_scope, service_ranges, should_summarize) 904 | resource_names['WafIPSet']['created'] += ipset_names['created'] 905 | resource_names['WafIPSet']['updated'] += ipset_names['updated'] 906 | except Exception as error: 907 | logging.error("Error handling WAF IPSet. It will not block the execution. It will continue to other services and resources.") 908 | logging.exception(error) 909 | logging.info(f'Finish handle WAF IPSet for "{service_name}"') 910 | 911 | except Exception as error: 912 | logging.exception(error) 913 | raise error 914 | 915 | logging.info(f'Function return: {resource_names}') 916 | logging.info('lambda_handler end') 917 | return resource_names 918 | 919 | 920 | # if __name__ == '__main__': 921 | # with open('test_event.json', 'r', encoding='utf-8') as event_file: 922 | # event: dict[str, Any] = json.loads(event_file.read()) 923 | # lambda_handler(event, None) 924 | -------------------------------------------------------------------------------- /terraform/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | aws = { 4 | version = "~> 4.51.0" 5 | } 6 | archive = { 7 | version = ">= 2.3.0" 8 | } 9 | } 10 | 11 | required_version = "~> 1.3.7" 12 | } 13 | 14 | data "aws_partition" "current" {} 15 | 16 | data "aws_region" "current" {} 17 | 18 | data "aws_caller_identity" "current" {} 19 | 20 | resource "aws_iam_role" "update_ip_ranges" { 21 | name_prefix = "UpdateIPRanges" 22 | description = "Managed by update IP ranges Lambda" 23 | assume_role_policy = jsonencode( 24 | { 25 | "Version" : "2012-10-17", 26 | "Statement" : [ 27 | { 28 | "Effect" : "Allow", 29 | "Principal" : { 30 | "Service" : "lambda.amazonaws.com" 31 | }, 32 | "Action" : "sts:AssumeRole" 33 | } 34 | ] 35 | } 36 | ) 37 | 38 | inline_policy { 39 | name = "CloudWatchLogsPermissions" 40 | 41 | policy = jsonencode( 42 | { 43 | "Version" : "2012-10-17", 44 | "Statement" : [ 45 | { 46 | "Effect" : "Allow", 47 | "Action" : [ 48 | "logs:CreateLogGroup" 49 | ], 50 | "Resource" : "arn:${data.aws_partition.current.partition}:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*" 51 | }, 52 | { 53 | "Effect" : "Allow", 54 | "Action" : [ 55 | "logs:CreateLogStream", 56 | "logs:PutLogEvents" 57 | ], 58 | "Resource" : "arn:${data.aws_partition.current.partition}:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*LambdaUpdateIPRanges*:*" 59 | } 60 | ] 61 | } 62 | ) 63 | } 64 | 65 | inline_policy { 66 | name = "WAFPermissions" 67 | 68 | policy = jsonencode( 69 | { 70 | "Version" : "2012-10-17", 71 | "Statement" : [ 72 | { 73 | "Effect" : "Allow", 74 | "Action" : [ 75 | "wafv2:ListIPSets" 76 | ], 77 | "Resource" : "*" 78 | }, 79 | { 80 | "Effect" : "Allow", 81 | "Action" : [ 82 | "wafv2:CreateIPSet", 83 | "wafv2:TagResource" 84 | ], 85 | "Resource" : "*", 86 | "Condition" : { 87 | "StringEquals" : { 88 | "aws:RequestTag/UpdatedAt" : "Not yet", 89 | "aws:RequestTag/ManagedBy" : "update-aws-ip-ranges" 90 | }, 91 | "StringLike" : { 92 | "aws:RequestTag/Name" : [ 93 | "aws-ip-ranges-*-ipv4", 94 | "aws-ip-ranges-*-ipv6" 95 | ] 96 | }, 97 | "ForAllValues:StringEquals" : { 98 | "aws:TagKeys" : [ 99 | "Name", 100 | "ManagedBy", 101 | "CreatedAt", 102 | "UpdatedAt" 103 | ] 104 | } 105 | } 106 | }, 107 | { 108 | "Effect" : "Allow", 109 | "Action" : [ 110 | "wafv2:TagResource" 111 | ], 112 | "Resource" : [ 113 | "arn:${data.aws_partition.current.partition}:wafv2:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*/ipset/aws-ip-ranges-*-ipv4/*", 114 | "arn:${data.aws_partition.current.partition}:wafv2:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*/ipset/aws-ip-ranges-*-ipv6/*" 115 | ], 116 | "Condition" : { 117 | "StringEquals" : { 118 | "aws:ResourceTag/ManagedBy" : "update-aws-ip-ranges" 119 | }, 120 | "StringLike" : { 121 | "aws:ResourceTag/Name" : [ 122 | "aws-ip-ranges-*-ipv4", 123 | "aws-ip-ranges-*-ipv6" 124 | ] 125 | }, 126 | "ForAllValues:StringEquals" : { 127 | "aws:TagKeys" : [ 128 | "UpdatedAt" 129 | ] 130 | } 131 | } 132 | }, 133 | { 134 | "Effect" : "Allow", 135 | "Action" : [ 136 | "wafv2:ListTagsForResource", 137 | "wafv2:GetIPSet", 138 | "wafv2:UpdateIPSet" 139 | ], 140 | "Resource" : [ 141 | "arn:${data.aws_partition.current.partition}:wafv2:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*/ipset/aws-ip-ranges-*-ipv4/*", 142 | "arn:${data.aws_partition.current.partition}:wafv2:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*/ipset/aws-ip-ranges-*-ipv6/*" 143 | ], 144 | "Condition" : { 145 | "StringEquals" : { 146 | "aws:ResourceTag/ManagedBy" : "update-aws-ip-ranges" 147 | }, 148 | "StringLike" : { 149 | "aws:ResourceTag/Name" : [ 150 | "aws-ip-ranges-*-ipv4", 151 | "aws-ip-ranges-*-ipv6" 152 | ] 153 | } 154 | } 155 | } 156 | ] 157 | } 158 | ) 159 | } 160 | 161 | inline_policy { 162 | name = "EC2Permissions" 163 | 164 | policy = jsonencode( 165 | { 166 | "Version" : "2012-10-17", 167 | "Statement" : [ 168 | { 169 | "Effect" : "Allow", 170 | "Action" : [ 171 | "ec2:DescribeTags", 172 | "ec2:DescribeManagedPrefixLists" 173 | ], 174 | "Resource" : "*", 175 | "Condition" : { 176 | "StringEquals" : { 177 | "ec2:Region" : "${data.aws_region.current.name}" 178 | } 179 | } 180 | }, 181 | { 182 | "Effect" : "Allow", 183 | "Action" : [ 184 | "ec2:GetManagedPrefixListEntries", 185 | "ec2:ModifyManagedPrefixList", 186 | "ec2:CreateTags" 187 | ], 188 | "Resource" : "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:prefix-list/*", 189 | "Condition" : { 190 | "StringEquals" : { 191 | "aws:ResourceTag/ManagedBy" : "update-aws-ip-ranges", 192 | "ec2:Region" : "${data.aws_region.current.name}" 193 | }, 194 | "StringLike" : { 195 | "aws:ResourceTag/Name" : [ 196 | "aws-ip-ranges-*-ipv4", 197 | "aws-ip-ranges-*-ipv6" 198 | ] 199 | } 200 | } 201 | }, 202 | { 203 | "Effect" : "Allow", 204 | "Action" : [ 205 | "ec2:CreateManagedPrefixList" 206 | ], 207 | "Resource" : "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:prefix-list/*", 208 | "Condition" : { 209 | "StringEquals" : { 210 | "aws:RequestTag/UpdatedAt" : "Not yet", 211 | "aws:RequestTag/ManagedBy" : "update-aws-ip-ranges", 212 | "ec2:Region" : "${data.aws_region.current.name}" 213 | }, 214 | "StringLike" : { 215 | "aws:RequestTag/Name" : [ 216 | "aws-ip-ranges-*-ipv4", 217 | "aws-ip-ranges-*-ipv6" 218 | ] 219 | }, 220 | "ForAllValues:StringEquals" : { 221 | "aws:TagKeys" : [ 222 | "Name", 223 | "ManagedBy", 224 | "CreatedAt", 225 | "UpdatedAt" 226 | ] 227 | } 228 | } 229 | }, 230 | { 231 | "Effect" : "Allow", 232 | "Action" : [ 233 | "ec2:CreateTags" 234 | ], 235 | "Resource" : "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:prefix-list/*", 236 | "Condition" : { 237 | "StringEquals" : { 238 | "ec2:Region" : "${data.aws_region.current.name}", 239 | "ec2:CreateAction" : "CreateManagedPrefixList" 240 | } 241 | } 242 | } 243 | ] 244 | } 245 | ) 246 | } 247 | } 248 | 249 | # Zip lambda source code. 250 | data "archive_file" "lambda_source" { 251 | type = "zip" 252 | #source_dir = var.src_path 253 | output_path = "/tmp/update_aws_ip_ranges.zip" 254 | 255 | source { 256 | filename = "update_aws_ip_ranges.py" 257 | content = file("../lambda/update_aws_ip_ranges.py") 258 | } 259 | 260 | source { 261 | filename = "services.json" 262 | content = file("../lambda/services.json") 263 | } 264 | } 265 | resource "aws_lambda_function" "update_ip_ranges" { 266 | # checkov:skip=CKV_AWS_50:X-ray tracing not required 267 | # checkov:skip=CKV_AWS_116:Code log errors on CloudWatch logs 268 | # checkov:skip=CKV_AWS_117:Not required to run inside a VPC 269 | # checkov:skip=CKV_AWS_173:Variable is not sensitive 270 | # checkov:skip=CKV_AWS_272:Code signer not required 271 | 272 | filename = data.archive_file.lambda_source.output_path 273 | source_code_hash = filebase64sha256(data.archive_file.lambda_source.output_path) 274 | function_name = "UpdateIPRanges" 275 | description = "This Lambda function, invoked by an incoming SNS message, updates the IPv4 and IPv6 ranges with the addresses from the specified services" 276 | role = aws_iam_role.update_ip_ranges.arn 277 | handler = "update_aws_ip_ranges.lambda_handler" 278 | runtime = "python3.12" 279 | timeout = 300 280 | reserved_concurrent_executions = 2 281 | memory_size = 256 282 | architectures = ["arm64"] 283 | 284 | environment { 285 | variables = { 286 | LOG_LEVEL = "INFO" 287 | } 288 | } 289 | } 290 | 291 | resource "aws_lambda_permission" "amazon_ip_space_changed" { 292 | statement_id = "AllowExecutionFromSNS" 293 | action = "lambda:InvokeFunction" 294 | function_name = aws_lambda_function.update_ip_ranges.function_name 295 | principal = "sns.amazonaws.com" 296 | source_arn = "arn:aws:sns:us-east-1:806199016981:AmazonIpSpaceChanged" 297 | source_account = "806199016981" 298 | } 299 | 300 | # provider to manage SNS topics 301 | provider "aws" { 302 | alias = "sns" 303 | region = "us-east-1" 304 | } 305 | resource "aws_sns_topic_subscription" "amazon_ip_space_changed" { 306 | provider = aws.sns 307 | topic_arn = "arn:aws:sns:us-east-1:806199016981:AmazonIpSpaceChanged" 308 | protocol = "lambda" 309 | endpoint = aws_lambda_function.update_ip_ranges.arn 310 | } 311 | 312 | output "lambda_name" { 313 | value = aws_lambda_function.update_ip_ranges.function_name 314 | } --------------------------------------------------------------------------------