├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── deployment ├── build-s3-dist.sh └── ops-automator.template ├── source ├── cloudformation │ ├── AccountForwardEvents.template │ ├── ops-automator.template │ └── scenarios │ │ └── Ec2VerticalScaling.template ├── code │ ├── actions │ │ ├── __init__.py │ │ ├── action_base.py │ │ ├── action_ec2_events_base.py │ │ ├── dynamodb_set_capacity_action.py │ │ ├── ec2_copy_snapshot_action.py │ │ ├── ec2_create_snapshot_action.py │ │ ├── ec2_delete_snapshot_action.py │ │ ├── ec2_replace_instance_action.py │ │ ├── ec2_resize_instance_action.py │ │ ├── ec2_tag_cpu_instance_action.py │ │ ├── scheduler_config_backup_action.py │ │ ├── scheduler_task_cleanup_action.py │ │ └── scheduler_task_export_action.py │ ├── add_me_to_ops_automator_role.py │ ├── boto_retry │ │ ├── __init__.py │ │ ├── aws_service_retry.py │ │ ├── dynamodb_service_retry.py │ │ ├── ec2_service_retry.py │ │ └── logs_service_retry.py │ ├── build-docker-script.py │ ├── build-forward-event-template.py │ ├── build-ops-automator-template.py │ ├── build_task_custom_resource.py │ ├── builders │ │ ├── __init__.py │ │ ├── action_template_builder.py │ │ ├── actions.html │ │ └── cross_account_role_builder.py │ ├── cloudwatch_queue_handler_lambda.py │ ├── configuration │ │ ├── __init__.py │ │ ├── task_admin_api.py │ │ └── task_configuration.py │ ├── forward-events.py │ ├── handlers │ │ ├── __init__.py │ │ ├── cli_request_handler.py │ │ ├── completion_handler.py │ │ ├── configuration_resource_handler.py │ │ ├── custom_resource.py │ │ ├── ebs_snapshot_event_handler.py │ │ ├── ec2_state_event_handler.py │ │ ├── ec2_tag_event_handler.py │ │ ├── event_handler_base.py │ │ ├── execution_handler.py │ │ ├── rds_event_handler.py │ │ ├── rds_tag_event_handler.py │ │ ├── s3_event_handler.py │ │ ├── schedule_handler.py │ │ ├── select_resources_handler.py │ │ ├── setup_helper_handler.py │ │ ├── task_tracking_handler.py │ │ └── task_tracking_table.py │ ├── helpers │ │ ├── __init__.py │ │ ├── dynamodb.py │ │ └── timer.py │ ├── main.py │ ├── makefile │ ├── metrics │ │ ├── __init__.py │ │ ├── anonymous_metrics.py │ │ └── task_metrics.py │ ├── models │ │ ├── ec2 │ │ │ └── 2016-11-15 │ │ │ │ └── service-2.json │ │ ├── rds │ │ │ └── 2014-10-31 │ │ │ │ └── service-2.json │ │ └── ssm │ │ │ └── 2014-11-06 │ │ │ └── service-2.json │ ├── outputs │ │ ├── __init__.py │ │ ├── issues_topic.py │ │ ├── queued_logger.py │ │ ├── report_output_writer.py │ │ └── result_notifications.py │ ├── requirements.txt │ ├── run_ops_automator_local.py │ ├── run_unit_tests.sh │ ├── scheduling │ │ ├── __init__.py │ │ ├── cron_expression.py │ │ ├── hour_setbuilder.py │ │ ├── minute_setbuilder.py │ │ ├── month_setbuilder.py │ │ ├── monthday_setbuilder.py │ │ ├── setbuilder.py │ │ └── weekday_setbuilder.py │ ├── services │ │ ├── __init__.py │ │ ├── aws_service.py │ │ ├── cloudformation_service.py │ │ ├── cloudwatchlogs_service.py │ │ ├── dynamodb_service.py │ │ ├── ec2_service.py │ │ ├── ecs_service.py │ │ ├── elasticache_service.py │ │ ├── elb_service.py │ │ ├── elbv2_service.py │ │ ├── iam_service.py │ │ ├── kms_service.py │ │ ├── lambda_service.py │ │ ├── opsautomatortest_service.py │ │ ├── rds_service.py │ │ ├── route53_service.py │ │ ├── s3_service.py │ │ ├── servicecatalog_service.py │ │ ├── storagegateway_service.py │ │ ├── tagging_service.py │ │ └── time_service.py │ ├── tagging │ │ ├── __init__.py │ │ ├── tag_filter_expression.py │ │ └── tag_filter_set.py │ ├── testing │ │ ├── __init__.py │ │ ├── cloudwatch_metrics.py │ │ ├── console_logger.py │ │ ├── context.py │ │ ├── datetime_provider.py │ │ ├── dynamodb.py │ │ ├── ec2.py │ │ ├── elb.py │ │ ├── elbv2.py │ │ ├── kms.py │ │ ├── rds.py │ │ ├── report_output_writer.py │ │ ├── s3.py │ │ ├── stack.py │ │ ├── storage_gateway.py │ │ ├── sts.py │ │ ├── tags.py │ │ ├── task_test_runner.py │ │ └── task_tracker.py │ ├── tests │ │ ├── __init__.py │ │ ├── action_tests │ │ │ ├── __init__.py │ │ │ ├── dynamodb_set_capacity │ │ │ │ ├── __init__.py │ │ │ │ ├── test_action.py │ │ │ │ └── test_resources.template │ │ │ ├── ec2_copy_snapshot │ │ │ │ ├── __init__.py │ │ │ │ ├── test_action.py │ │ │ │ ├── test_resources.template │ │ │ │ └── test_resources_destination_region.template │ │ │ ├── ec2_create_snapshot │ │ │ │ ├── __init__.py │ │ │ │ ├── test_action.py │ │ │ │ └── test_resources.template │ │ │ ├── ec2_delete_snapshot │ │ │ │ ├── __init__.py │ │ │ │ ├── test_action.py │ │ │ │ └── test_resources.template │ │ │ ├── ec2_replace_instance │ │ │ │ ├── __init__.py │ │ │ │ ├── test_action.py │ │ │ │ └── test_resources.template │ │ │ ├── ec2_resize_instance │ │ │ │ ├── __init__.py │ │ │ │ ├── test_action.py │ │ │ │ └── test_resources.template │ │ │ └── ec2_tag_cpu_instance │ │ │ │ ├── __init__.py │ │ │ │ ├── test_action.py │ │ │ │ └── test_resources.template │ │ └── configuration_tests │ │ │ ├── test_hour_setbuilder.py │ │ │ ├── test_minute_setbuilder.py │ │ │ ├── test_month_setbuilder.py │ │ │ ├── test_monthday_setbuilder.py │ │ │ ├── test_setbuilder.py │ │ │ ├── test_tag_expression.py │ │ │ ├── test_tag_filter_set.py │ │ │ └── test_weekday_setbuilder.py │ └── update-build-number.py ├── ecs │ ├── Dockerfile │ ├── build-and-deploy-image.sh │ └── ops-automator-ecs-runner.py └── version.txt └── version.txt /.gitignore: -------------------------------------------------------------------------------- 1 | **/dist 2 | **/open-source 3 | **/*.zip 4 | **/.DS_Store 5 | **/*.pyc 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [2.2.0] - 2020-08-27 8 | ### Added 9 | - (installed from source) Instructions for using ECS/Fargate rather than Lambda for Automation. See GitHub https://github.com/awslabs/aws-ops-automator/tree/master/source/ecs/README.md 10 | - S3 access logging to aws-opsautomator-s3-access-logs-\-\ 11 | 12 | ### Changed 13 | - README.md now contains instructions on upgrading Ops Automator 2.x to the latest release. 14 | - ECS/Fargate option updated to use Python3 15 | - ECS/Fargate option now uses OpsAutomatorLambdaRole (previously had no role assigned) 16 | - Updated all Lambda runtimes to Python 3.8 17 | - Encryption is now enabled by default in Mappings->Settings->Resources->EncryptResourceData. All SNS topics, SQS queue, and DynamoDB tables are encrypted by this setting. 18 | - S3 buckets are now encrypted using SSE AES256 19 | 20 | ### Known Issues 21 | - ECS can be used for the Resource Selection process, but may fail when used for the Execution of actions. Customers should test use of ECS for Execution and use Lambda if unsuccessful. 22 | 23 | ## [2.1.0] - 2019-10-06 24 | ### Added 25 | - upgraded the solution to Python 3.7 -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check [existing open](https://github.com/awslabs/aws-ops-automator/issues), or [recently closed](https://github.com/awslabs/aws-ops-automator/issues?q=is%3Aissue+is%3Aclosed), issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/aws-ops-automator/labels/help%20wanted) issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](https://github.com/awslabs/aws-ops-automator/blob/master/LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | AWS Ops Automator 2 | Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except 4 | in compliance with the License. A copy of the License is located at http://www.apache.org/licenses/ 5 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, 6 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the 7 | specific language governing permissions and limitations under the License. 8 | 9 | ********************** 10 | THIRD PARTY COMPONENTS 11 | ********************** 12 | This software includes third party software subject to the following copyrights: 13 | 14 | AWS SDK under the Apache License Version 2.0 15 | pytz under the Massachusetts Institute of Technology (MIT) license 16 | -------------------------------------------------------------------------------- /deployment/build-s3-dist.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # This assumes all of the OS-level configuration has been completed and git repo has already been cloned 4 | # 5 | # This script should be run from the repo's deployment directory 6 | # cd deployment 7 | # ./build-s3-dist.sh source-bucket-base-name trademarked-solution-name version-code 8 | # 9 | # Parameters: 10 | # - source-bucket-base-name: Name for the S3 bucket location where the template will source the Lambda 11 | # code from. The template will append '-[region_name]' to this bucket name. 12 | # For example: ./build-s3-dist.sh solutions my-solution v1.0.0 13 | # The template will then expect the source code to be located in the solutions-[region_name] bucket 14 | # 15 | # - trademarked-solution-name: name of the solution for consistency 16 | # 17 | # - version-code: version of the solution 18 | function do_cmd { 19 | echo "------ EXEC $*" 20 | $* 21 | } 22 | function do_replace { 23 | replace="s/$2/$3/g" 24 | file=$1 25 | do_cmd sed -i -e $replace $file 26 | } 27 | 28 | if [ -z "$1" ] | [ -z "$2" ]; then 29 | echo "Usage: $0 [bucket] [solution-name] {version}" 30 | echo "Please provide the base source bucket name, trademark approved solution name and version where the lambda code will eventually reside." 31 | echo "For example: ./build-s3-dist.sh solutions trademarked-solution-name v1.0.0" 32 | exit 1 33 | fi 34 | 35 | bucket=$1 36 | echo "export DIST_OUTPUT_BUCKET=$bucket" > ./setenv.sh 37 | solution_name=$2 38 | echo "export DIST_SOLUTION_NAME=$solution_name" >> ./setenv.sh 39 | 40 | # Version from the command line is definitive. Otherwise, use version.txt 41 | if [ ! -z "$3" ]; then 42 | version=$3 43 | elif [ -e ../source/version.txt ]; then 44 | version=`cat ../source/version.txt` 45 | else 46 | echo "Version not found. Version must be passed as argument 3 or in version.txt in the format vn.n.n" 47 | fi 48 | 49 | if [[ ! "$version" =~ ^v.*? ]]; then 50 | version=v$version 51 | fi 52 | echo "export DIST_VERSION=$version" >> ./setenv.sh 53 | 54 | echo "==========================================================================" 55 | echo "Building $solution_name version $version for bucket $bucket" 56 | echo "==========================================================================" 57 | 58 | # Get reference for all important folders 59 | template_dir="$PWD" # /deployment 60 | template_dist_dir="$template_dir/global-s3-assets" 61 | build_dist_dir="$template_dir/regional-s3-assets" 62 | source_dir="$template_dir/../source" 63 | dist_dir="$template_dir/dist" 64 | 65 | echo "------------------------------------------------------------------------------" 66 | echo "[Init] Clean old dist folders" 67 | echo "------------------------------------------------------------------------------" 68 | do_cmd rm -rf $template_dist_dir 69 | do_cmd mkdir -p $template_dist_dir 70 | do_cmd rm -rf $build_dist_dir 71 | do_cmd mkdir -p $build_dist_dir 72 | do_cmd rm -rf $dist_dir 73 | do_cmd mkdir -p $dist_dir 74 | 75 | # Copy the source tree to deployment/dist 76 | do_cmd cp -r $source_dir/* $dist_dir 77 | 78 | do_cmd pip install --upgrade pip 79 | # awscli will also install the compatible version of boto3 and botocore 80 | do_cmd pip install --upgrade awscli 81 | do_cmd pip install -r $source_dir/code/requirements.txt -t $dist_dir/code 82 | 83 | echo "------------------------------------------------------------------------------" 84 | echo "[Make] Set up and call make from deployment/dist/code" 85 | echo "------------------------------------------------------------------------------" 86 | cp $source_dir/version.txt $dist_dir/code 87 | cd $dist_dir/code 88 | do_cmd make bucket=$bucket solution=$solution_name version=$version 89 | cd $template_dir 90 | # rm -rf dist 91 | chmod +x setenv.sh 92 | echo "Completed building distribution" 93 | -------------------------------------------------------------------------------- /source/code/add_me_to_ops_automator_role.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import json 14 | import sys 15 | 16 | import boto3 17 | 18 | 19 | def add_me_to_role(stack, principal): 20 | role_resource = boto3.client("cloudformation").describe_stack_resource( 21 | StackName=stack, LogicalResourceId="OpsAutomatorLambdaRole").get("StackResourceDetail", None) 22 | 23 | role_name = role_resource["PhysicalResourceId"] 24 | 25 | role = boto3.client("iam").get_role(RoleName=role_name).get("Role", {}) 26 | assume_role_policy_document = role.get("AssumeRolePolicyDocument", {}) 27 | statement = assume_role_policy_document.get("Statement", []) 28 | 29 | for s in statement: 30 | if s["Principal"].get("AWS", "") == principal: 31 | break 32 | else: 33 | statement.append({"Action": "sts:AssumeRole", "Effect": "Allow", "Principal": {"AWS": principal}}) 34 | boto3.client("iam").update_assume_role_policy( 35 | RoleName=role_name, 36 | PolicyDocument=json.dumps(assume_role_policy_document) 37 | ) 38 | print(("Principal {} can now assume role {}".format(principal, role_name))) 39 | 40 | 41 | if __name__ == "__main__": 42 | 43 | if len(sys.argv) < 1: 44 | print("No stack name argument passed as first parameter") 45 | exit(1) 46 | 47 | principal = boto3.client("sts").get_caller_identity()["Arn"] 48 | print(("Adds {} to AssumeRolePolicyDocument of Ops Automator role defined in stack {} for local debugging".format( 49 | principal, sys.argv[1]))) 50 | 51 | add_me_to_role(sys.argv[1], principal) 52 | 53 | print("Done...") 54 | exit(0) 55 | -------------------------------------------------------------------------------- /source/code/boto_retry/dynamodb_service_retry.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from boto_retry.aws_service_retry import AwsApiServiceRetry 14 | 15 | 16 | class DynamoDbServiceRetry(AwsApiServiceRetry): 17 | """ 18 | Class that extends retry logic with DynamoDB specific logic 19 | """ 20 | 21 | def __init__(self, context=None, logger=None, timeout=None, wait_strategy=None, lambda_time_out_margin=10): 22 | AwsApiServiceRetry.__init__( 23 | self, 24 | call_retry_strategies=None, 25 | wait_strategy=wait_strategy, 26 | context=context, 27 | timeout=timeout, 28 | logger=logger, 29 | lambda_time_out_margin=lambda_time_out_margin) 30 | 31 | self._call_retry_strategies += [ 32 | self.dynamo_throughput_exceeded, 33 | self.dynamo_resource_in_use, 34 | self.dynamo_connection_reset_by_peer 35 | ] 36 | 37 | @classmethod 38 | def dynamo_throughput_exceeded(cls, ex): 39 | """ 40 | Adds retry logic on top of the retry logic already done by boto3 if max throughput is exceeded for a table or index 41 | :param ex: Exception to test 42 | :return: 43 | """ 44 | return type(ex).__name__ == "ProvisionedThroughputExceededException" 45 | 46 | @classmethod 47 | def dynamo_resource_in_use(cls, ex): 48 | return type(ex).__name__ == "ResourceInUseException" 49 | 50 | @classmethod 51 | def dynamo_connection_reset_by_peer(cls, ex): 52 | return "Connection reset by peer" in str(ex) 53 | -------------------------------------------------------------------------------- /source/code/boto_retry/ec2_service_retry.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from botocore.exceptions import ClientError, ParamValidationError 14 | 15 | from boto_retry.aws_service_retry import AwsApiServiceRetry 16 | 17 | 18 | class Ec2ServiceRetry(AwsApiServiceRetry): 19 | """ 20 | Class that extends retry logic with Ec2 specific logic 21 | """ 22 | 23 | def __init__(self, context=None, logger=None, timeout=None, wait_strategy=None, lambda_time_out_margin=10): 24 | AwsApiServiceRetry.__init__( 25 | self, 26 | call_retry_strategies=None, 27 | wait_strategy=wait_strategy, 28 | context=context, 29 | timeout=timeout, 30 | logger=logger, 31 | lambda_time_out_margin=lambda_time_out_margin) 32 | 33 | self._call_retry_strategies += [ 34 | self.snapshot_creation_per_volume_throttles, 35 | self.resource_limit_exceeded, 36 | self.request_limit_exceeded 37 | ] 38 | 39 | @classmethod 40 | def snapshot_creation_per_volume_throttles(cls, ex): 41 | """ 42 | Retries in case the snapshot creation rate is exceeded for a volume 43 | :param ex: Exception to test 44 | :return: 45 | """ 46 | return type(ex) == ClientError and \ 47 | ex.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 0) == 400 and \ 48 | "SnapshotCreationPerVolumeRateExceeded" == ex.response.get("Error", {}).get("Code", "") 49 | 50 | @classmethod 51 | def resource_limit_exceeded(cls, ex): 52 | """ 53 | Retries in case resource limits are exceeded. 54 | :param ex: 55 | :return: 56 | """ 57 | return type(ex) == ClientError and \ 58 | ex.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 0) == 400 and \ 59 | "ResourceLimitExceeded" == ex.response.get("Error", {}).get("Code", "") 60 | 61 | @classmethod 62 | def request_limit_exceeded(cls, ex): 63 | """ 64 | Retries in case requests limits are exceeded. 65 | :param ex: 66 | :return: 67 | """ 68 | return type(ex) == ClientError and \ 69 | ex.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 0) == 503 and \ 70 | "RequestLimitExceeded" == ex.response.get("Error", {}).get("Code", "") 71 | 72 | def can_retry(self, ex): 73 | """ 74 | Tests if a retry can be done based on the exception of an earlier call 75 | :param ex: Execution raise by earlier call of the boto3 method 76 | :return: True if any of the call_retry_strategy returns True, else False 77 | """ 78 | if type(ex) == ParamValidationError: 79 | return False 80 | return AwsApiServiceRetry.can_retry(self, ex) 81 | -------------------------------------------------------------------------------- /source/code/boto_retry/logs_service_retry.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from boto_retry.aws_service_retry import AwsApiServiceRetry 14 | 15 | 16 | class CloudWatchLogsServiceRetry(AwsApiServiceRetry): 17 | 18 | # noinspection PyUnusedLocal 19 | def __init__(self, logger=None, context=None, timeout=15, wait_strategy=None, lambda_time_out_margin=10): 20 | """ 21 | Initializes retry logic 22 | :param wait_strategy: Wait strategy that returns retry wait periods 23 | :param context: Lambda context that is used to calculate remaining execution time 24 | :param timeout: Timeout for method call. This time can not exceed the remaining time if a method is called 25 | within the context of a lambda function. 26 | :param lambda_time_out_margin: If called within the context of a Lambda function this time should at least be 27 | remaining before making a retry. This is to allow possible cleanup and logging actions in the remaining time 28 | """ 29 | AwsApiServiceRetry.__init__( 30 | self, 31 | call_retry_strategies=None, 32 | wait_strategy=wait_strategy, 33 | context=context, 34 | timeout=timeout, 35 | logger=None, 36 | lambda_time_out_margin=lambda_time_out_margin) 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /source/code/build-docker-script.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import sys 14 | 15 | import boto3 16 | 17 | FIVE_YEARS = 5 * 365 * 24 * 3600 18 | 19 | 20 | def get_signed_url(bucket, key): 21 | s3 = boto3.client("s3") 22 | 23 | params = { 24 | "Bucket": bucket, 25 | "Key": key 26 | } 27 | 28 | return s3.generate_presigned_url("get_object", Params=params, ExpiresIn=FIVE_YEARS, HttpMethod="GET") 29 | 30 | 31 | def build_script(script, bucket, version, prefix): 32 | ecs_runner_script_url = get_signed_url(bucket, prefix + "ecs/ops-automator-ecs-runner.py") 33 | docker_file_url = get_signed_url(bucket, prefix + "ecs/Dockerfile") 34 | 35 | with open(script, mode="rt") as f: 36 | return "".join(f.readlines()) \ 37 | .replace("%ecs_runner_script%", ecs_runner_script_url) \ 38 | .replace("%docker_file%", docker_file_url) \ 39 | .replace("%version%", version) 40 | 41 | 42 | if __name__ == "__main__": 43 | try: 44 | script = sys.argv[1] 45 | bucket = sys.argv[2] 46 | version = sys.argv[3] 47 | if len(sys.argv) > 4: 48 | prefix = sys.argv[4] 49 | else: 50 | prefix = "" 51 | 52 | print(build_script(script, bucket, version, prefix)) 53 | 54 | except Exception as ex: 55 | print(ex) 56 | raise ex 57 | exit(1) 58 | -------------------------------------------------------------------------------- /source/code/build-forward-event-template.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import sys 14 | 15 | from builders import build_events_forward_template 16 | 17 | if __name__ == "__main__": 18 | print((build_events_forward_template(template_filename=sys.argv[1], 19 | script_filename=sys.argv[2], 20 | ops_automator_topic_arn="arn:topic", 21 | event_role_arn=sys.argv[3], 22 | version=sys.argv[4]))) 23 | -------------------------------------------------------------------------------- /source/code/build_task_custom_resource.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import decimal 14 | import json 15 | import os.path 16 | import sys 17 | from datetime import datetime 18 | from math import trunc 19 | 20 | import boto3 21 | 22 | 23 | class CustomCfnJsonEncoder(json.JSONEncoder): 24 | 25 | def default(self, o): 26 | if isinstance(o, set): 27 | return list(o) 28 | if isinstance(o, datetime): 29 | return o.isoformat() 30 | if isinstance(o, decimal.Decimal): 31 | return str(trunc(o)) 32 | if isinstance(o, Exception): 33 | return str(o) 34 | return json.JSONEncoder.default(self, o) 35 | 36 | 37 | if __name__ == '__main__': 38 | 39 | if len(sys.argv) < 2: 40 | print(("Syntax is {} taskname [optional profile name]".format(os.path.basename(sys.argv[0])))) 41 | 42 | stack_name = "%stack%" 43 | table_name = "%config_table%" 44 | 45 | task_name = sys.argv[1] 46 | 47 | session = boto3.Session(profile_name=sys.argv[2]) if (len(sys.argv)) > 2 else boto3.Session() 48 | 49 | service_token = "arn:aws:lambda:%region%:%account%:function:%stack%-OpsAutomator-Standard" 50 | 51 | db = session.resource("dynamodb").Table(table_name) 52 | config_item = db.get_item( 53 | TableName=table_name, 54 | Key={ 55 | "Name": task_name 56 | }).get("Item") 57 | 58 | if config_item is None: 59 | print(("Task {} not found in table {}".format(task_name, table_name))) 60 | exit(1) 61 | 62 | config_item.update({"Name": task_name, "ServiceToken": service_token}) 63 | if "StackId" in config_item: 64 | del config_item["StackId"] 65 | 66 | for p in list(config_item.keys()): 67 | if config_item[p] is None: 68 | del config_item[p] 69 | 70 | for p in list(config_item.get("Parameters",{}).keys()): 71 | if config_item["Parameters"][p] is None: 72 | del config_item["Parameters"][p] 73 | 74 | custom_resource = { 75 | "Type": "Custom::TaskConfig", 76 | "Properties": config_item 77 | } 78 | 79 | result = json.dumps(custom_resource, cls=CustomCfnJsonEncoder, indent=3, sort_keys=True) 80 | result = result.replace(': true', ': "True"') 81 | result = result.replace(': false', ': "False"') 82 | 83 | print(result) 84 | -------------------------------------------------------------------------------- /source/code/builders/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import json 14 | import os 15 | import urllib.request, urllib.parse, urllib.error 16 | from collections import OrderedDict 17 | from os import path 18 | 19 | import actions 20 | import handlers 21 | import services 22 | 23 | CFN_CONSOLE_URL_TEMPLATE = \ 24 | "https://{}.console.aws.amazon.com/cloudformation/home?region={}#/stacks/create/review?" \ 25 | "param_Description={}&templateURL=https:%2F%2Fs3-{}.amazonaws.com%2F{}%2FTaskConfiguration%2F{}.template" 26 | 27 | HTML_ACTION_LIST_ITEM = \ 28 | "\t\t\t
  • \n\t\t\t\t{}\n\t\t\t
    • \n" 29 | HTML_ACTIONS_GROUPS_LISTS = \ 30 | "" 31 | HTML_GROUP_LIST_ITEM = \ 32 | "\n\t
  • \n\t\t
    {}
    \n\t\t
      \n{}\n\t\t
    \n\t
    • " 33 | 34 | 35 | def build_events_forward_template(template_filename, script_filename, stack, event_role_arn, ops_automator_topic_arn, version): 36 | with open(script_filename, "rt") as f: 37 | script_text = f.readlines() 38 | 39 | with open(template_filename, "rt") as f: 40 | template = json.loads("".join(f.readlines()), object_pairs_hook=OrderedDict) 41 | 42 | code = template["Resources"]["EventsForwardFunction"]["Properties"]["Code"] 43 | code["ZipFile"]["Fn::Join"][1] = script_text 44 | 45 | return json.dumps(template, indent=3) \ 46 | .replace("%version%", version) \ 47 | .replace("%ops-automator-stack%", stack) \ 48 | .replace("%ops-automator-region%", services.get_session().region_name) \ 49 | .replace("%ops-automator-account%", services.get_aws_account()) \ 50 | .replace("%ops-automator-topic-arn%", ops_automator_topic_arn) \ 51 | .replace("%event-forward-role%", event_role_arn) 52 | 53 | 54 | def build_scenario_templates(templates_dir, stack): 55 | for template_name in os.listdir(templates_dir): 56 | with open(path.join(templates_dir, template_name), "rt") as f: 57 | template = json.loads("".join(f.readlines()), object_pairs_hook=OrderedDict) 58 | 59 | yield template_name, json.dumps(template, indent=3).replace("%ops-automator-stack%", stack) 60 | 61 | 62 | def group_name_from_action_name(action_name): 63 | i = 1 64 | while i < len(action_name) and (action_name[i].islower() or action_name[i].isdigit()): 65 | i += 1 66 | group_name = action_name[0:i].upper() 67 | return group_name 68 | 69 | 70 | def generate_html_actions_page(html_file, region): 71 | with open(html_file) as f: 72 | html_template = "".join(f.readlines()) 73 | 74 | bucket = os.getenv(handlers.ENV_CONFIG_BUCKET) 75 | stack = os.getenv(handlers.ENV_STACK_NAME) 76 | 77 | action_groups = {} 78 | for a in actions.all_actions(): 79 | ap = actions.get_action_properties(a) 80 | if ap.get(actions.ACTION_INTERNAL): 81 | continue 82 | href = CFN_CONSOLE_URL_TEMPLATE.format(region, region, urllib.parse.quote(ap.get(actions.PARAM_DESCRIPTION, "")), region, bucket, 83 | a) 84 | 85 | group_name = group_name_from_action_name(a) 86 | if group_name not in action_groups: 87 | action_groups[group_name] = {} 88 | 89 | action_groups[group_name][a] = (href, ap.get(actions.ACTION_TITLE)) 90 | 91 | action_list = "" 92 | for g in sorted(action_groups.keys()): 93 | actions_list = "" 94 | for a in sorted(action_groups[g].keys()): 95 | actions_list += HTML_ACTION_LIST_ITEM.format(action_groups[g][a][0], 96 | action_groups[g][a][1]) 97 | action_list += HTML_GROUP_LIST_ITEM.format(g, actions_list) 98 | action_list = HTML_ACTIONS_GROUPS_LISTS.format(action_list) 99 | 100 | return html_template.replace("%actions%", action_list).replace("%stack%", stack) 101 | -------------------------------------------------------------------------------- /source/code/builders/actions.html: -------------------------------------------------------------------------------- 1 | 13 | 14 | 15 | 16 | 17 | Available actions for Ops Automator stack %stack% 18 | 19 | 20 | 76 |

    Available actions for Ops Automator stack %stack%

    77 | %actions% 78 | 79 | 80 | -------------------------------------------------------------------------------- /source/code/configuration/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | BOOLEAN_FALSE_VALUES = [ 14 | "false", 15 | "no", 16 | "disabled", 17 | "off", 18 | "0" 19 | ] 20 | 21 | BOOLEAN_TRUE_VALUES = [ 22 | "true", 23 | "yes", 24 | "enabled", 25 | "on", 26 | "1" 27 | ] 28 | 29 | # name of environment variable that holds the name of the configuration table 30 | ENV_CONFIG_TABLE = "CONFIG_TABLE" 31 | ENV_CONFIG_BUCKET = "CONFIG_BUCKET" 32 | 33 | TASKS_OBJECTS = "TaskConfigurationObjects" 34 | 35 | # names of attributes in configuration 36 | # name of the action 37 | CONFIG_ACTION_NAME = "Action" 38 | # debug parameter 39 | CONFIG_DEBUG = "Debug" 40 | # notifications for started/ended tasks 41 | CONFIG_TASK_NOTIFICATIONS = "TaskNotifications" 42 | # list of cross account roles 43 | CONFIG_ACCOUNTS = "Accounts" 44 | # name of alternative cross account role 45 | CONFIG_TASK_CROSS_ACCOUNT_ROLE_NAME = "CrossAccountRole" 46 | # description 47 | CONFIG_DESCRIPTION = "Description" 48 | # Switch to enable/disable task 49 | CONFIG_ENABLED = "Enabled" 50 | # tag filter for tags of source resource of an event 51 | CONFIG_EVENT_SOURCE_TAG_FILTER = "SourceEventTagFilter" 52 | # cron expression interval for time/date based tasks 53 | CONFIG_INTERVAL = "Interval" 54 | # internal task 55 | CONFIG_INTERNAL = "Internal" 56 | # name of the task 57 | CONFIG_TASK_NAME = "Name" 58 | # parameters of a task 59 | CONFIG_PARAMETERS = "Parameters" 60 | # switch to indicate if resource in the account of the scheduler should be processed 61 | CONFIG_THIS_ACCOUNT = "ThisAccount" 62 | # timezone for time/date scheduled task 63 | CONFIG_TIMEZONE = "Timezone" 64 | # tag filter to select resources processed by the task 65 | CONFIG_TAG_FILTER = "TagFilter" 66 | # regions where to select/process resources 67 | CONFIG_REGIONS = "Regions" 68 | # dryrun switch, passed to the tasks action 69 | CONFIG_DRYRUN = "Dryrun" 70 | # events that trigger the task 71 | CONFIG_EVENTS = "Events" 72 | # event scopes 73 | CONFIG_EVENT_SCOPES = "EventScopes" 74 | # stack id if created from cloudformation stack 75 | CONFIG_STACK_ID = "StackId" 76 | # action timeout 77 | CONFIG_TASK_TIMEOUT = "TaskTimeout" 78 | # action select memory 79 | CONFIG_TASK_SELECT_SIZE = "SelectSize" 80 | # action select memory 81 | CONFIG_TASK_EXECUTE_SIZE = "ExecuteSize" 82 | # action completion memory 83 | CONFIG_TASK_COMPLETION_SIZE = "CompletionSize" 84 | # action completion memory when running in ECS 85 | CONFIG_ECS_COMPLETION_MEMORY = "CompletionEcsMemoryValue" 86 | # action select memory when running in ECS 87 | CONFIG_ECS_SELECT_MEMORY = "SelectEcsMemoryValueValue" 88 | # action select memory when running in ECS 89 | CONFIG_ECS_EXECUTE_MEMORY = "ExecuteEcsMemoryValue" 90 | 91 | # Task metrics 92 | CONFIG_TASK_METRICS = "TaskMetrics" 93 | -------------------------------------------------------------------------------- /source/code/forward-events.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import json 14 | import os 15 | 16 | import boto3 17 | 18 | FORWARDED_EVENTS = { 19 | "aws.ec2": [ 20 | "EC2 Instance State-change Notification", 21 | "EBS Snapshot Notification" 22 | ], 23 | "aws.tag": [ 24 | "Tag Change on Resource" 25 | ], 26 | "aws.rds": [ 27 | "AWS API Call via CloudTrail" 28 | ] 29 | } 30 | 31 | INF_FORWARDED = "Event from source \"{}\", type \"{}\" forwarded to region {}, account {}, topic {}\n{}" 32 | INF_EVENT_ALREADY_IN_REGION = "Event from source \"{}\", type \"{}\" already in forward region {} or is a non-forwarded event" 33 | ERR_FAILED_FORWARD = "Failed to forward event {}, {}" 34 | 35 | 36 | def lambda_handler(event, _): 37 | print("Ops Automator Events Forwarder (version %version%)") 38 | destination_region = os.getenv("OPS_AUTOMATOR_REGION", "") 39 | destination_account = os.getenv("OPS_AUTOMATOR_ACCOUNT") 40 | source = event.get("source", "") 41 | detail_type = event.get("detail-type", "") 42 | if ((event.get("region", "") != destination_region) or (event.get("account", "") != destination_account)) and \ 43 | detail_type in FORWARDED_EVENTS.get(source, []): 44 | 45 | destination_region_sns_client = boto3.client("sns", region_name=destination_region) 46 | 47 | try: 48 | topic = os.getenv("OPS_AUTOMATOR_TOPIC_ARN") 49 | destination_region_sns_client.publish(TopicArn=topic, Message=json.dumps(event)) 50 | print((INF_FORWARDED.format(source, detail_type, destination_region, destination_account, topic, str(event)))) 51 | return "OK" 52 | except Exception as ex: 53 | raise Exception(ERR_FAILED_FORWARD, str(event), ex) 54 | 55 | else: 56 | print((INF_EVENT_ALREADY_IN_REGION.format(source, detail_type, destination_region))) 57 | -------------------------------------------------------------------------------- /source/code/handlers/rds_tag_event_handler.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | import services.rds_service 15 | from handlers.event_handler_base import * 16 | 17 | ERR_HANDLING_RDS_TAG_EVENT = "Error handling RDS tag event {}" 18 | 19 | RDS_TAG_EVENT_PARAM = "RdsTag{}" 20 | RDS_TAGGING_EVENTS_TITLE = "Rds tag change events" 21 | 22 | RDS_TAG_EVENT_SOURCE = "rds." + handlers.TAG_EVENT_SOURCE 23 | 24 | CHANGED_DBIBSTANCE_TAG_EVENT_DESCRIPTION_TEXT = "Run task when tags changed for instance" 25 | CHANGED_DBIBSTANCE_TAG_EVENT_LABEL_TEXT = "Tags changed for RDS Instance" 26 | 27 | CHANGED_DBCLUSTER_TAG_EVENT_DESCRIPTION_TEXT = "Run task when tags changed for cluster" 28 | CHANGED_DBCLUSTER_TAG_EVENT_LABEL_TEXT = "Tags changed for RDS Cluster" 29 | 30 | CHANGE_PREFIX = "Changed{}Tags" 31 | 32 | RDS_CHANGED_INSTANCE_TAGS_EVENT = CHANGE_PREFIX.format(services.rds_service.DB_INSTANCES[0:-1]) 33 | RDS_CHANGED_CLUSTER_TAGS_EVENT = CHANGE_PREFIX.format(services.rds_service.DB_CLUSTERS[0:-1]) 34 | 35 | HANDLED_RESOURCES = [services.rds_service.DB_INSTANCES] 36 | 37 | RESOURCE_MAPPINGS = { 38 | "db": services.rds_service.DB_INSTANCES[0:-1], 39 | "cluster": services.rds_service.DB_CLUSTERS[0:-1] 40 | } 41 | 42 | HANDLED_EVENTS = { 43 | EVENT_SOURCE_TITLE: RDS_TAGGING_EVENTS_TITLE, 44 | EVENT_SOURCE: RDS_TAG_EVENT_SOURCE, 45 | EVENT_PARAMETER: RDS_TAG_EVENT_PARAM, 46 | EVENT_EVENTS: { 47 | RDS_CHANGED_INSTANCE_TAGS_EVENT: { 48 | EVENT_LABEL: CHANGED_DBIBSTANCE_TAG_EVENT_LABEL_TEXT, 49 | EVENT_DESCRIPTION: CHANGED_DBIBSTANCE_TAG_EVENT_DESCRIPTION_TEXT 50 | }, 51 | RDS_CHANGED_CLUSTER_TAGS_EVENT: { 52 | EVENT_LABEL: CHANGED_DBCLUSTER_TAG_EVENT_LABEL_TEXT, 53 | EVENT_DESCRIPTION: CHANGED_DBCLUSTER_TAG_EVENT_DESCRIPTION_TEXT 54 | } 55 | } 56 | } 57 | 58 | 59 | class RdsTagEventHandler(EventHandlerBase): 60 | def __init__(self, event, context): 61 | EventHandlerBase.__init__(self, event=event, 62 | context=context, 63 | resource="", 64 | handled_event_source=RDS_TAG_EVENT_SOURCE, 65 | handled_event_detail_type=handlers.TAG_CHANGE_EVENT, 66 | is_tag_change_event=True, 67 | event_name_in_detail="") 68 | 69 | @staticmethod 70 | def is_handling_event(event, logger): 71 | try: 72 | if event.get("source", "") != handlers.TAG_EVENT_SOURCE: 73 | return False 74 | 75 | if event.get("detail-type", "") != handlers.TAG_CHANGE_EVENT_SOURCE_DETAIL_TYPE: 76 | return False 77 | 78 | detail = event.get("detail", {}) 79 | 80 | if detail.get("service", "").lower() != "rds": 81 | return False 82 | 83 | return detail.get("resource-type", "") in RESOURCE_MAPPINGS 84 | except Exception as ex: 85 | logger.error(ERR_HANDLING_RDS_TAG_EVENT, ex) 86 | return False 87 | 88 | def handle_request(self, use_custom_select=True): 89 | EventHandlerBase.handle_request(self, use_custom_select=False) 90 | 91 | def _select_parameters(self, event_name, task): 92 | resource_type = self._event["detail"]["resource-type"] 93 | if resource_type not in RESOURCE_MAPPINGS: 94 | raise NotImplementedError 95 | 96 | res = RESOURCE_MAPPINGS[self._event["detail"]["resource-type"]] 97 | res = "DB" + res[2:] 98 | return {res+"Identifier": self._event["resources"][0].split(":")[-1], 99 | "_expected_boto3_exceptions_": [res + "NotFound"] 100 | } 101 | 102 | def _event_name(self): 103 | return CHANGE_PREFIX.format(RESOURCE_MAPPINGS[self._event["detail"]["resource-type"]]) 104 | 105 | def _source_resource_tags(self, session, task): 106 | raise NotImplementedError 107 | -------------------------------------------------------------------------------- /source/code/handlers/s3_event_handler.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import services.s3_service 14 | from handlers.event_handler_base import * 15 | 16 | 17 | EVENT_DESCRIPTION_OBJECT_CREATED = "Run task when S3 Object is created" 18 | EVENT_DESCRIPTION_OBJECT_DELETED = "Run task when S3 Object is deleted" 19 | EVENT_LABEL_OBJECT_CREATED = "Object created" 20 | 21 | EVENT_LABEL_OBJECT_DELETED = "Object deleted" 22 | S3_EVENTS_PARAM = "S3{}" 23 | S3_EVENTS_TITLE = "S3 events" 24 | 25 | S3_EVENT_DETAIL_TYPE = "S3 Object Event" 26 | S3_OBJECT_CREATED = "ObjectCreated" 27 | S3_OBJECT_DELETED = "ObjectDeleted" 28 | 29 | S3_OBJECT_EVENTS = [S3_OBJECT_CREATED, S3_OBJECT_DELETED] 30 | 31 | HANDLED_EVENTS = { 32 | EVENT_SOURCE_TITLE: S3_EVENTS_TITLE, 33 | EVENT_SOURCE: handlers.S3_EVENT_SOURCE, 34 | EVENT_PARAMETER: S3_EVENTS_PARAM, 35 | EVENT_EVENTS: { 36 | S3_OBJECT_CREATED: { 37 | EVENT_LABEL: EVENT_LABEL_OBJECT_CREATED, 38 | EVENT_DESCRIPTION: EVENT_DESCRIPTION_OBJECT_CREATED}, 39 | S3_OBJECT_DELETED: { 40 | EVENT_LABEL: EVENT_LABEL_OBJECT_DELETED, 41 | EVENT_DESCRIPTION: EVENT_DESCRIPTION_OBJECT_DELETED} 42 | } 43 | } 44 | 45 | 46 | class S3EventHandler(EventHandlerBase): 47 | def __init__(self, event, context): 48 | EventHandlerBase.__init__(self, 49 | event=event, 50 | resource=services.s3_service.OBJECT, 51 | context=context, 52 | handled_event_detail_type=S3_EVENT_DETAIL_TYPE, 53 | handled_event_source=handlers.S3_EVENT_SOURCE) 54 | self._event["detail-type"] = S3_EVENT_DETAIL_TYPE 55 | 56 | @staticmethod 57 | def is_handling_event(event, logger): 58 | return len(event.get("Records", [])) == 1 and \ 59 | event["Records"][0].get("eventSource", "") == handlers.S3_EVENT_SOURCE and \ 60 | event["Records"][0].get("eventName", "").split(":")[0] in S3_OBJECT_EVENTS 61 | 62 | def _select_parameters(self, event_name, task): 63 | return {} 64 | 65 | def _event_resources(self): 66 | return [{ 67 | "Bucket": self._event["Records"][0]["s3"]["bucket"]["name"], 68 | "Key": self._event["Records"][0]["s3"]["object"]["key"], 69 | "BucketArn": self._event["Records"][0]["s3"]["bucket"]["arn"], 70 | "Owner": self._event["Records"][0]["s3"]["bucket"]["ownerIdentity"], 71 | "Size": self._event["Records"][0]["s3"]["object"]["size"], 72 | "AwsAccount": self._event_account(), 73 | "Region": self._event_region() 74 | }] 75 | 76 | def _event_name(self): 77 | return self._event["Records"][0]["eventName"].split(":")[0] 78 | 79 | def _event_region(self): 80 | return self._event["Records"][0]["awsRegion"] 81 | 82 | def _event_account(self): 83 | return None 84 | 85 | def _event_time(self): 86 | return self._event.get("Records", [{}])[0].get("eventTime") 87 | 88 | def _source_resource_tags(self, session, task): 89 | raise NotImplementedError 90 | 91 | -------------------------------------------------------------------------------- /source/code/helpers/dynamodb.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from datetime import datetime 14 | from decimal import Decimal 15 | 16 | 17 | def unpack_record(record): 18 | def get_data(item): 19 | data_type = list(item.keys())[0] 20 | data = item[data_type] 21 | if data_type == "M": 22 | return {i: get_data(data[i]) for i in data} 23 | if data_type == "L": 24 | return [get_data(i) for i in data] 25 | return item[data_type] 26 | 27 | result = {i: get_data(record[i]) for i in record} 28 | return result 29 | 30 | 31 | def build_record(item): 32 | def build_typed_item(o, dict_as_map=True): 33 | if isinstance(o, datetime): 34 | return {"S": o.isoformat()} 35 | if isinstance(o, bool): 36 | return {"BOOL": o} 37 | if isinstance(o, int) or isinstance(o, float) or isinstance(o, Decimal): 38 | return {"N": str(o)} 39 | if isinstance(o, dict): 40 | return {"M": {i: build_typed_item(o[i]) for i in o if o[i] not in [None, ""]}} if dict_as_map else o 41 | if isinstance(o, list): 42 | return {"L": [build_typed_item(i) for i in o if i not in [None, ""]]} 43 | return {"S": str(o)} 44 | 45 | return {attr: build_typed_item(item[attr]) for attr in item if item[attr] not in [None, ""]} 46 | 47 | 48 | def as_dynamo_safe_types(data): 49 | def check_attributes(d): 50 | for attr in list(d.keys()): 51 | if isinstance(d[attr], datetime): 52 | d[attr] = d[attr].isoformat() 53 | continue 54 | 55 | if isinstance(d[attr], str) and d[attr].strip() == "": 56 | del d[attr] 57 | continue 58 | 59 | if isinstance(d[attr], dict): 60 | d[attr] = as_dynamo_safe_types(d[attr]) 61 | continue 62 | 63 | if isinstance(data, list): 64 | for i in data: 65 | check_attributes(i) 66 | else: 67 | check_attributes(data) 68 | 69 | return data 70 | -------------------------------------------------------------------------------- /source/code/helpers/timer.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import threading 14 | 15 | 16 | class Timer(object): 17 | def __init__(self, timeout_seconds, start=True): 18 | self.timeout = False 19 | self._timer = threading.Timer(interval=timeout_seconds, function=self.fn) 20 | if timeout_seconds > 0 and start: 21 | self.start() 22 | 23 | def fn(self): 24 | self._timer.cancel() 25 | self.timeout = True 26 | 27 | def start(self): 28 | self.timeout = False 29 | self._timer.start() 30 | 31 | def stop(self): 32 | self._timer.cancel() 33 | 34 | def __enter__(self): 35 | return self 36 | 37 | def __exit__(self, exc_type, exc_val, exc_tb): 38 | self._timer.cancel() 39 | -------------------------------------------------------------------------------- /source/code/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from datetime import datetime 14 | 15 | import actions 16 | import handlers 17 | from metrics.task_metrics import TaskMetrics 18 | 19 | ENV_METRICS_URL = "METRICS_URL" 20 | ENV_SOLUTION_ID = "SOLUTION_ID" 21 | ENV_SEND_METRICS = "SEND_METRICS" 22 | 23 | METRICS_STATUS_NAMES = { 24 | handlers.STATUS_PENDING: "Submitted", 25 | handlers.STATUS_STARTED: "Executing", 26 | handlers.STATUS_WAIT_FOR_COMPLETION: "Waiting to complete", 27 | handlers.STATUS_COMPLETED: "Completed", 28 | handlers.STATUS_TIMED_OUT: "Timed out", 29 | handlers.STATUS_FAILED: "Failed", 30 | handlers.STATUS_WAITING: "Waiting for execution" 31 | } 32 | 33 | 34 | def put_task_state_metrics(task_name, metric_state_name, task_level, count=1, logger=None, data=None, context=None, ): 35 | with TaskMetrics(datetime.utcnow(), context=context, logger=logger) as metrics: 36 | metrics.put_task_state_metrics(task_name=task_name, 37 | metric_state_name=metric_state_name, 38 | task_level=task_level, 39 | count=count, 40 | data=data) 41 | 42 | 43 | def put_task_select_data(task_name, items, selected_items, selection_time, logger=None, context=None): 44 | with TaskMetrics(datetime.utcnow(), logger=logger, context=context) as metrics: 45 | metrics.put_task_select_data(task_name=task_name, items=items, selected_items=selected_items, selection_time=selection_time) 46 | 47 | 48 | def put_general_errors_and_warnings(error_count=0, warning_count=0, logger=None, context=None): 49 | with TaskMetrics(datetime.utcnow(), logger=logger, context=context) as metrics: 50 | metrics.put_general_errors_and_warnings(error_count=error_count, warning_count=warning_count) 51 | 52 | 53 | def setup_tasks_metrics(task, action_name, task_level_metrics, logger=None, context=None): 54 | with TaskMetrics(dt=datetime.utcnow(), logger=logger, context=context)as metrics: 55 | 56 | task_class = actions.get_action_class(action_name) 57 | 58 | # number of submitted task instances for task 59 | metrics.put_task_state_metrics(task_name=task, 60 | metric_state_name=METRICS_STATUS_NAMES[handlers.STATUS_PENDING], 61 | count=0, 62 | task_level=task_level_metrics) 63 | 64 | # init metrics for results 65 | for s in [handlers.STATUS_STARTED, handlers.STATUS_COMPLETED, handlers.STATUS_FAILED]: 66 | metrics.put_task_state_metrics(task_name=task, 67 | metric_state_name=METRICS_STATUS_NAMES[s], 68 | count=0, 69 | task_level=task_level_metrics) 70 | 71 | # init metrics for tasks with completion handling 72 | if getattr(task_class, handlers.COMPLETION_METHOD, None) is not None: 73 | metrics.put_task_state_metrics(task_name=task, 74 | metric_state_name=METRICS_STATUS_NAMES[handlers.STATUS_WAIT_FOR_COMPLETION], 75 | count=0, 76 | task_level=task_level_metrics) 77 | metrics.put_task_state_metrics(task_name=task, 78 | metric_state_name=METRICS_STATUS_NAMES[handlers.STATUS_TIMED_OUT], 79 | count=0, 80 | task_level=task_level_metrics) 81 | 82 | # init metrics for tasks with concurrency handling 83 | if getattr(task_class, handlers.ACTION_CONCURRENCY_KEY_METHOD, None) is not None: 84 | metrics.put_task_state_metrics(task_name=task, 85 | metric_state_name=METRICS_STATUS_NAMES[handlers.STATUS_WAITING], 86 | count=0, 87 | task_level=task_level_metrics) 88 | -------------------------------------------------------------------------------- /source/code/metrics/anonymous_metrics.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import os 14 | import uuid 15 | from datetime import datetime 16 | 17 | import requests 18 | 19 | import metrics 20 | from helpers import safe_json 21 | 22 | INF_METRICS_DATA = "Sending anonymous metrics data\n {}" 23 | INF_METRICS_DATA_SENT = "Metrics data send, status code is {}, message is {}" 24 | INF_SENDING_METRICS_FAILED = "Failed send metrics data ({})" 25 | WARN_ENV_METRICS_URL_NOT_SET = "Environment variable {} is not set, metrics dat is not sent" 26 | WARN_SOLUTION_ID_NOT_SET = "Solution id is not set, metrics are not sent" 27 | 28 | 29 | def allow_send_metrics(): 30 | """ 31 | Tests if anonymous metrics can be send 32 | :return: True if metrics can be send 33 | """ 34 | return str(os.getenv(metrics.ENV_SEND_METRICS, "false")).lower() == "true" 35 | 36 | 37 | def send_metrics_data(metrics_data, logger): 38 | url = os.getenv(metrics.ENV_METRICS_URL, None) 39 | if url is None: 40 | logger.warning(WARN_ENV_METRICS_URL_NOT_SET, metrics.ENV_METRICS_URL) 41 | return 42 | 43 | solution_id = os.getenv(metrics.ENV_SOLUTION_ID, None) 44 | if solution_id is None: 45 | logger.warning(WARN_SOLUTION_ID_NOT_SET) 46 | return 47 | 48 | data_dict = { 49 | "TimeStamp": str(datetime.utcnow().isoformat()), 50 | "UUID": str(uuid.uuid4()), 51 | "Data": metrics_data, 52 | "Solution": solution_id, 53 | } 54 | 55 | data_json = safe_json(data_dict, indent=3) 56 | logger.info(INF_METRICS_DATA, data_json) 57 | 58 | headers = { 59 | 'content-type': 'application/json', 60 | "content-length": str(len(data_json)) 61 | } 62 | 63 | try: 64 | response = requests.post(url, data=data_json, headers=headers) 65 | response.raise_for_status() 66 | logger.info(INF_METRICS_DATA_SENT, response.status_code, response.text) 67 | except Exception as exc: 68 | logger.info(INF_SENDING_METRICS_FAILED, str(exc)) 69 | -------------------------------------------------------------------------------- /source/code/outputs/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | import inspect 15 | import sys 16 | 17 | 18 | def get_error_constant_name(scope, message, prefix): 19 | for g in [n for n in scope if n.startswith(prefix)]: 20 | if isinstance(scope[g], str) and scope[g] == message: 21 | return g 22 | return None 23 | 24 | 25 | def get_extended_info(error_message, prefix): 26 | # noinspection PyProtectedMember 27 | caller_stack_frame = sys._getframe(2) 28 | caller = caller_stack_frame.f_code.co_name 29 | line = caller_stack_frame.f_lineno 30 | module = inspect.getmodule(caller_stack_frame) 31 | error_code = get_error_constant_name(module.__dict__, error_message, prefix) 32 | if error_code is None: 33 | error_code = get_error_constant_name(caller_stack_frame.f_globals, error_message, prefix) 34 | 35 | result = { 36 | "Caller": caller, 37 | "Module": module.__name__, 38 | "Line": line 39 | } 40 | if error_code is not None: 41 | result["Code"] = error_code 42 | 43 | return result 44 | 45 | 46 | def raise_value_error(msg, *args): 47 | s = msg if len(args) == 0 else msg.format(*args) 48 | ext_error_info = get_extended_info(msg, "ERR") 49 | code = ext_error_info.get("Code", None) 50 | if code is not None: 51 | s = "{} : {}".format(code, s) 52 | raise ValueError(s) 53 | 54 | 55 | def raise_exception(msg, *args): 56 | s = msg if len(args) == 0 else msg.format(*args) 57 | ext_error_info = get_extended_info(msg, "ERR") 58 | code = ext_error_info.get("Code", None) 59 | if code is not None: 60 | s = "{} : {}".format(code, s) 61 | raise Exception(s) 62 | -------------------------------------------------------------------------------- /source/code/outputs/issues_topic.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto_retry 4 | from helpers import safe_json 5 | 6 | ENV_SNS_ISSUE_TOPIC = "SNS_ISSUES_TOPIC_ARN" 7 | 8 | 9 | class IssuesTopic(object): 10 | 11 | def __init__(self, log_group, log_stream, context): 12 | self._sns_client = None 13 | self._loggroup = log_group 14 | self._logstream = log_stream 15 | self._context = context 16 | 17 | @property 18 | def sns_client(self): 19 | if self._sns_client is None: 20 | self._sns_client = boto_retry.get_client_with_retries("sns", ["publish"], context=self._context) 21 | return self._sns_client 22 | 23 | def publish(self, level, msg, ext_info): 24 | 25 | sns_arn = os.getenv(ENV_SNS_ISSUE_TOPIC, None) 26 | if sns_arn is not None: 27 | message = { 28 | "log-group": self._loggroup, 29 | "log-stream": self._logstream, 30 | "level": level, 31 | "message": msg 32 | } 33 | if ext_info not in [None, {}]: 34 | for i in ext_info: 35 | message[i.lower()] = ext_info[i] 36 | 37 | topic_msg = safe_json({"default": safe_json(message, indent=3), "lambda": message}) 38 | resp = self.sns_client.publish_with_retries(TopicArn=sns_arn, 39 | Message=topic_msg, 40 | MessageStructure="json") 41 | print(resp) 42 | -------------------------------------------------------------------------------- /source/code/outputs/report_output_writer.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import os 14 | from datetime import datetime 15 | 16 | from boto_retry import get_client_with_retries 17 | 18 | ENV_REPORT_BUCKET = "REPORTING_BUCKET" 19 | 20 | 21 | def create_output_writer(context=None, logger=None): 22 | return ReportOutputWriter(context=context, logger=logger) 23 | 24 | 25 | class ReportOutputWriter(object): 26 | 27 | def __init__(self, **kwargs): 28 | self._context = kwargs.get("context") 29 | self._logger = kwargs.get("logger") 30 | 31 | def write(self, data, key): 32 | s3_client = get_client_with_retries("s3", ["put_object"], context=self._context, logger=self._logger) 33 | s3_client.put_object_with_retries(Bucket=os.getenv(ENV_REPORT_BUCKET), Key=key, Body=data) 34 | 35 | 36 | def report_key_name(action, account=None, region=None, subject=None, with_task_id=True, ext="csv"): 37 | return "{}/{}/{}-{}{}-{}{}{}".format(action.__class__.__name__[0:-len("Action")], 38 | action.get("task"), 39 | account if account is not None else action.get("account"), 40 | region if region is not None else action.get("region"), 41 | "-" + subject if subject is not None else "", 42 | datetime.now().strftime("%Y%m%d%H%M"), 43 | ("-" + action.get("task_id")) if with_task_id is not None else "", 44 | (("." if ext.startswith(".") else "") + ext) if ext not in ["", None] else "") 45 | 46 | -------------------------------------------------------------------------------- /source/code/outputs/result_notifications.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import os 14 | from datetime import datetime 15 | 16 | import handlers 17 | from boto_retry import get_client_with_retries 18 | from helpers import safe_json 19 | 20 | MESSAGE_TYPE_ENDED = "task-ended" 21 | MESSAGE_TYPE_STARTED = "task-started" 22 | 23 | ERR_SEND_NOTIFICATION = "Cannot send notification to topic {}, {}" 24 | 25 | ENV_RESULT_TOPIC = "SNS_RESULT_TOPIC_ARN" 26 | MAX_SIZE = 262143 27 | 28 | 29 | class ResultNotifications(object): 30 | 31 | def __init__(self, context, logger): 32 | self._sns_client = None 33 | self._context = context 34 | self._logger = logger 35 | 36 | @property 37 | def sns_client(self): 38 | if self._sns_client is None: 39 | self._sns_client = get_client_with_retries("sns", methods=["publish"], context=self._context) 40 | return self._sns_client 41 | 42 | @property 43 | def topic_arn(self): 44 | return os.getenv(ENV_RESULT_TOPIC, None) 45 | 46 | @classmethod 47 | def _build_common_attributes(cls, task): 48 | message = {a: task.get(a, "") for a in [ 49 | handlers.TASK_TR_ID, 50 | handlers.TASK_TR_NAME, 51 | handlers.TASK_TR_ACTION, 52 | handlers.TASK_TR_ACCOUNT, 53 | handlers.TASK_TR_RESOURCES, 54 | handlers.TASK_TR_PARAMETERS 55 | ]} 56 | 57 | message["Time"] = datetime.now().isoformat() 58 | return message 59 | 60 | def _publish(self, message): 61 | self.sns_client.publish_with_retries(TopicArn=self.topic_arn, Message=safe_json(message)[0:MAX_SIZE]) 62 | 63 | def publish_started(self, task): 64 | try: 65 | if task.get(handlers.TASK_TR_NOTIFICATIONS, False): 66 | message = self._build_common_attributes(task) 67 | message["Type"] = MESSAGE_TYPE_STARTED 68 | self._publish(message) 69 | except Exception as ex: 70 | self._logger.error(ERR_SEND_NOTIFICATION, self.topic_arn, ex) 71 | 72 | def publish_ended(self, task): 73 | try: 74 | if task.get(handlers.TASK_TR_NOTIFICATIONS, False): 75 | message = self._build_common_attributes(task) 76 | message["Type"] = MESSAGE_TYPE_ENDED 77 | message[handlers.TASK_TR_STATUS] = task.get(handlers.TASK_TR_STATUS, "") 78 | if task[handlers.TASK_TR_STATUS] == handlers.STATUS_COMPLETED: 79 | message[handlers.TASK_TR_RESULT] = task.get(handlers.TASK_TR_RESULT) 80 | else: 81 | message[handlers.TASK_TR_ERROR] = task.get(handlers.TASK_TR_ERROR, "") 82 | self._publish(message) 83 | except Exception as ex: 84 | self._logger.error(ERR_SEND_NOTIFICATION, self.topic_arn, ex) 85 | -------------------------------------------------------------------------------- /source/code/requirements.txt: -------------------------------------------------------------------------------- 1 | python_version > '3.6' 2 | boto3 3 | requests 4 | pytz 5 | -------------------------------------------------------------------------------- /source/code/run_unit_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # *** INTERNAL DOCUMENT --- NOT FOR DISTRIBUTION *** 3 | function run_test() { 4 | if [ -e "tests/action_tests/$1/test_action.py" ]; then 5 | if [ -z $2 ]; then 6 | echo Running test $1 7 | python -m unittest tests.action_tests.$1.test_action > test_$1.out 8 | else 9 | echo Running test $1 - $specific_test 10 | python -m unittest tests.action_tests.$1.test_action.TestAction.$specific_test > test_$1.out 11 | fi 12 | else 13 | echo "ERROR: Test $1 not found" 14 | fi 15 | } 16 | 17 | if [ ! -z "$1" ]; then 18 | specific_test="" 19 | if [ ! -z "$2" ]; then 20 | specific_test=$2 21 | fi 22 | run_test $1 $specific_test 23 | else 24 | ls tests/action_tests | while read file; do 25 | if [[ $file == "__"* ]]; then 26 | continue 27 | fi 28 | if [ -d "tests/action_tests/${file}" ]; then 29 | run_test $file 30 | fi 31 | done 32 | fi 33 | -------------------------------------------------------------------------------- /source/code/scheduling/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/aws-ops-automator/215ff25181b5d9193f92fa0eb84a6036d8212163/source/code/scheduling/__init__.py -------------------------------------------------------------------------------- /source/code/scheduling/hour_setbuilder.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from scheduling.setbuilder import SetBuilder 14 | 15 | 16 | class HourSetBuilder(SetBuilder): 17 | """ 18 | Class for building set of hour values 0-23 and am/pm 19 | """ 20 | 21 | def __init__(self): 22 | SetBuilder.__init__(self, min_value=0, max_value=23, wrap=False) 23 | 24 | def _get_value_by_name(self, name_str): 25 | # allow usage of am and pm in value strings 26 | hour = SetBuilder._get_value_by_name(self, name_str) 27 | if hour is None: 28 | return self._get_hour_am_pm(name_str) 29 | 30 | def _get_hour_am_pm(self, hour_am_pm_str): 31 | # process times with am and pm 32 | if 2 < len(hour_am_pm_str) <= 4: 33 | s = hour_am_pm_str.lower() 34 | if s[-2:].lower() in ["am", "pm"]: 35 | hour = self._get_value_by_str(s[0:len(hour_am_pm_str) - 2]) 36 | if hour is not None: 37 | ampm = s[-2:] 38 | if ampm == "pm": 39 | # 12pm = 12:00 40 | if hour != 12: 41 | hour += 12 42 | # invalid to use pm if hour > 12 43 | if hour > 23: 44 | raise ValueError("hour {} is not valid".format(hour_am_pm_str)) 45 | # 12am = 00 46 | elif ampm == "am" and hour == 12: 47 | hour = 0 48 | return hour 49 | return None 50 | -------------------------------------------------------------------------------- /source/code/scheduling/minute_setbuilder.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from scheduling.setbuilder import SetBuilder 14 | 15 | 16 | class MinuteSetBuilder(SetBuilder): 17 | """ 18 | Class for building builds set of minute values (00-59) 19 | """ 20 | 21 | def __init__(self): 22 | SetBuilder.__init__(self, min_value=0, max_value=59, wrap=False) 23 | -------------------------------------------------------------------------------- /source/code/scheduling/month_setbuilder.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import calendar 14 | 15 | from scheduling.setbuilder import SetBuilder 16 | 17 | 18 | class MonthSetBuilder(SetBuilder): 19 | """ 20 | Class for building month sets, 1-12 ans jan-dec 21 | """ 22 | 23 | def __init__(self, wrap=True, ignore_case=True): 24 | """ 25 | Initializes set builder for month sets 26 | :param wrap: Set to True to allow wrapping at last month of the year 27 | :param ignore_case: Set to True to ignore case when mapping month names 28 | """ 29 | SetBuilder.__init__(self, 30 | names=calendar.month_abbr[1:], 31 | significant_name_characters=3, 32 | offset=1, 33 | ignore_case=ignore_case, 34 | wrap=wrap) 35 | -------------------------------------------------------------------------------- /source/code/scheduling/monthday_setbuilder.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import calendar 14 | 15 | from scheduling.setbuilder import SetBuilder 16 | 17 | 18 | class MonthdaySetBuilder(SetBuilder): 19 | """ 20 | Class for building sets of monthdays, 1-(28-31), ',', '-', '/", "*", W for nearest weekday, L for last day of month 21 | """ 22 | WILDCARD_WEEKDAY = "W" 23 | WILDCARD_LAST_WEEKDAY = "L" 24 | 25 | def __init__(self, year, month): 26 | """ 27 | Initializes monthday set builder. 28 | :param year: Year of month to build sets for, only required for month aware 'W' and 'L' features in expressions 29 | :param month: Month to build sets for, only required for month aware 'W' and 'L' features in expressions 30 | """ 31 | self.year = year 32 | self.month = month 33 | self._firstweekday, self._lastday = calendar.monthrange(year, month) 34 | 35 | SetBuilder.__init__(self, 36 | min_value=1, 37 | max_value=self._lastday, 38 | offset=1, 39 | ignore_case=False, 40 | wrap=False, 41 | last_item_wildcard=MonthdaySetBuilder.WILDCARD_LAST_WEEKDAY) 42 | 43 | self._post_custom_parsers = [self._parse_weekday] 44 | 45 | def _parse_weekday(self, day_str): 46 | # dayW return working day nearest to day 47 | return self._get_weekday(day_str) 48 | 49 | def _parse_unknown(self, item): 50 | return [] if item in [str(d) for d in range(self.last, 32)] else None 51 | 52 | def _separator_characters(self): 53 | # adding W to separator characters, it should not be formatted 54 | return SetBuilder._separator_characters(self) + self.WILDCARD_WEEKDAY 55 | 56 | def _get_weekday(self, day_str): 57 | # returns working day nearest to day in month, string is in format dayW 58 | if (1 < len(day_str) <= 3) and day_str.endswith(self.WILDCARD_WEEKDAY): 59 | day = self._get_value_by_str(day_str[0:-1]) 60 | if day is not None: 61 | # calculated day of week based on first weekday of month 62 | weekday = ((day % 7) + self._firstweekday - 1) % 7 63 | # for Saturdays use Friday, or Monday if it is the first day of the month 64 | if weekday == 5: 65 | day = day - 1 if day > 1 else day + 2 66 | # for Sundays use next Monday, or Saturday if it is the last day of the month 67 | elif weekday == 6: 68 | day = day + 1 if day < self._lastday else day - 2 69 | # for other days just return the specified day 70 | return [day] 71 | 72 | return None 73 | -------------------------------------------------------------------------------- /source/code/services/cloudformation_service.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from services.aws_service import AwsService 14 | 15 | ACCOUNT_LIMITS = "AccountLimits" 16 | CHANGE_SET = "ChangeSet" 17 | CHANGE_SETS_SUMMARY = "ChangeSetsSummary" 18 | RESOURCES_SUMMARY = "StackResourcesSummary" 19 | STACK_EVENTS = "StackEvents" 20 | STACK_RESOURCE = "StackResource" 21 | STACK_RESOURCES = "StackResources" 22 | STACKS = "Stacks" 23 | STACK_LIST = "StackList" 24 | STACKS_SUMMARY = "StacksSummary" 25 | STACK_POLICY = "StackPolicy" 26 | TEMPLATE = "Template" 27 | TEMPLATE_SUMMARY = "TemplateSummary" 28 | 29 | CUSTOM_RESULT_PATHS = { 30 | CHANGE_SET: "", 31 | STACK_RESOURCE: "StackResourceDetail", 32 | CHANGE_SETS_SUMMARY: "Summaries", 33 | RESOURCES_SUMMARY: "StackResourceSummaries", 34 | STACKS_SUMMARY: "StackSummaries", 35 | STACK_LIST: "StackSummaries", 36 | STACK_POLICY: "{StackPolicy:StackPolicyBody}", 37 | TEMPLATE: "{" + ",".join(['"{}":{}'.format(i, i) for i in [ 38 | "TemplateBody", 39 | "StagesAvailable", 40 | 41 | ]]) + "}", 42 | TEMPLATE_SUMMARY: "{" + ",".join(['"{}":{}'.format(i, i) for i in [ 43 | "Parameters", 44 | "Description", 45 | "Capabilities", 46 | "CapabilitiesReason", 47 | "ResourceTypes", 48 | "Version", 49 | "Metadata", 50 | "DeclaredTransforms" 51 | ]]) + "}", 52 | } 53 | 54 | RESOURCE_NAMES = [ 55 | ACCOUNT_LIMITS, 56 | CHANGE_SET, 57 | STACK_EVENTS, 58 | STACK_RESOURCE, 59 | STACK_RESOURCES, 60 | STACKS, 61 | CHANGE_SETS_SUMMARY, 62 | RESOURCES_SUMMARY, 63 | STACKS_SUMMARY, 64 | STACK_POLICY, 65 | TEMPLATE, 66 | TEMPLATE_SUMMARY 67 | ] 68 | 69 | RESOURCES_WITH_TAGS = [ 70 | STACKS 71 | ] 72 | 73 | 74 | class CloudformationService(AwsService): 75 | """ 76 | CloudFormation service 77 | """ 78 | 79 | def __init__(self, role_arn=None, session=None, tags_as_dict=True, as_named_tuple=False, service_retry_strategy=None): 80 | """ 81 | :param role_arn: Optional (cross account) role to use to retrieve services 82 | :param session: Optional session to use to retrieve services 83 | :param tags_as_dict: Set to True true to convert resource tags to dictionaries 84 | :param as_named_tuple: Set to True to return resources as named tuples instead of a dictionary 85 | :param service_retry_strategy: service retry strategy for making boto api calls 86 | """ 87 | AwsService.__init__(self, 88 | service_name='cloudformation', 89 | resource_names=RESOURCE_NAMES, 90 | resources_with_tags=RESOURCES_WITH_TAGS, 91 | role_arn=role_arn, 92 | session=session, 93 | tags_as_dict=tags_as_dict, 94 | as_named_tuple=as_named_tuple, 95 | custom_result_paths=CUSTOM_RESULT_PATHS, 96 | service_retry_strategy=service_retry_strategy) 97 | 98 | def describe_resources_function_name(self, resource_name): 99 | """ 100 | Returns the name of the boto client method call to retrieve the specified resource. 101 | :param resource_name: 102 | :return: Name of the boto3 client function to retrieve the specified resource type 103 | """ 104 | s = AwsService.describe_resources_function_name(self, resource_name=resource_name) 105 | 106 | if resource_name in [CHANGE_SETS_SUMMARY, RESOURCES_SUMMARY, STACKS_SUMMARY]: 107 | s = s.replace("describe_", "list_")[0:-len("_Summary")] 108 | 109 | elif resource_name in [STACK_POLICY, TEMPLATE, TEMPLATE_SUMMARY]: 110 | s = s.replace("describe_", "get_") 111 | 112 | elif resource_name == STACK_LIST: 113 | s = "list_stacks" 114 | 115 | return s 116 | -------------------------------------------------------------------------------- /source/code/services/cloudwatchlogs_service.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from services.aws_service import AwsService 14 | 15 | DESTINATIONS = "Destinations" 16 | EXPORT_TASKS = "ExportTasks" 17 | LOG_EVENTS = "LogEvents" 18 | LOG_GROUPS = "LogGroups" 19 | LOG_STREAMS = "LogStreams" 20 | METRIC_FILTERS = "MetricFilters" 21 | SUBSCRIPTION_FILTERS = "SubscriptionFilters" 22 | 23 | MAPPED_PARAMETERS = { 24 | "MaxResults": "limit" 25 | } 26 | 27 | NEXT_TOKEN_ARGUMENT = "nextToken" 28 | NEXT_TOKEN_RESULT = NEXT_TOKEN_ARGUMENT 29 | 30 | RESOURCE_NAMES = [ 31 | DESTINATIONS, 32 | EXPORT_TASKS, 33 | LOG_GROUPS, 34 | LOG_STREAMS, 35 | METRIC_FILTERS, 36 | SUBSCRIPTION_FILTERS, 37 | LOG_EVENTS 38 | ] 39 | 40 | 41 | class CloudwatchlogsService(AwsService): 42 | def __init__(self, role_arn=None, session=None, tags_as_dict=True, as_named_tuple=False, service_retry_strategy=None): 43 | """ 44 | :param role_arn: Optional (cross account) role to use to retrieve services 45 | :param session: Optional session to use to retrieve services 46 | :param tags_as_dict: Set to True true to convert resource tags to dictionaries 47 | :param as_named_tuple: Set to True to return resources as named tuples instead of a dictionary 48 | :param service_retry_strategy: Retry strategy for service 49 | :param service_retry_strategy: service retry strategy for making boto api calls 50 | """ 51 | 52 | custom_resource_paths = {r: r[0].lower() + r[1:] for r in RESOURCE_NAMES} 53 | custom_resource_paths[LOG_EVENTS] = "events" 54 | 55 | AwsService.__init__(self, 56 | service_name='logs', 57 | resource_names=RESOURCE_NAMES, 58 | role_arn=role_arn, 59 | session=session, 60 | tags_as_dict=tags_as_dict, 61 | as_named_tuple=as_named_tuple, 62 | custom_result_paths=custom_resource_paths, 63 | mapped_parameters=MAPPED_PARAMETERS, 64 | next_token_argument=NEXT_TOKEN_ARGUMENT, 65 | next_token_result=NEXT_TOKEN_RESULT, 66 | service_retry_strategy=service_retry_strategy) 67 | 68 | def _tuple_name_func(self, name): 69 | """ 70 | Returns the name of the tuple for resources returned as named tuple 71 | :param name: 72 | :return: 73 | """ 74 | return name[0].upper() + name[1:] 75 | 76 | def describe_resources_function_name(self, resource_name): 77 | """ 78 | Returns the name of the boto client method call to retrieve the specified resource. 79 | :param resource_name: 80 | :return: Name of the boto3 client function to retrieve the specified resource type 81 | """ 82 | s = AwsService.describe_resources_function_name(self, resource_name=resource_name) 83 | 84 | if resource_name == LOG_EVENTS: 85 | s = s.replace("describe_", "filter_") 86 | return s 87 | 88 | def _map_describe_function_parameters(self, resources, args): 89 | """ 90 | Maps the parameter names passed to the service class describe call to names used to make the call the boto 91 | service client describe call 92 | :param resources: Name of the resource type 93 | :param args: parameters to be mapped 94 | :return: mapped parameters 95 | """ 96 | if len(args) == 0: 97 | return args 98 | temp = AwsService._map_describe_function_parameters(self, resources, args) 99 | # for this service arguments start with lowercase 100 | translated = {b[0].lower() + b[1:]: temp[b] for b in temp} 101 | return translated 102 | -------------------------------------------------------------------------------- /source/code/services/ecs_service.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from services.aws_service import AwsService 14 | 15 | 16 | CLUSTERS = "Clusters" 17 | CLUSTERS_ARNS = "ClustersArns" 18 | CONTAINER_INSTANCES = "ContainerInstances" 19 | CONTAINER_INSTANCES_ARNS = "ContainerInstancesArns" 20 | SERVICES = "Services" 21 | SERVICES_ARNS = "ServicesArns" 22 | TASK_ARNS = "TaskArns" 23 | TASK_DEFINITION_FAMILIES = "TaskDefinitionFamilies" 24 | TASK_DEFINITIONS = "TaskDefinitions" 25 | TASK_DEFINITIONS_ARNS = "TaskDefinitionsArns" 26 | TASKS = "Tasks" 27 | 28 | CUSTOM_RESULT_PATHS = { 29 | CLUSTERS: "clusters", 30 | CLUSTERS_ARNS: "clusterArns", 31 | CONTAINER_INSTANCES: "containerInstances", 32 | CONTAINER_INSTANCES_ARNS: "containerInstanceArns", 33 | SERVICES: "services", 34 | SERVICES_ARNS: "serviceArns", 35 | TASK_DEFINITION_FAMILIES: "families", 36 | TASK_DEFINITIONS: "taskDefinition", 37 | TASK_DEFINITIONS_ARNS: "taskDefinitionArns", 38 | TASKS: "tasks", 39 | TASK_ARNS: "taskArns" 40 | } 41 | 42 | RESOURCE_NAMES = [ 43 | CLUSTERS, 44 | CLUSTERS_ARNS, 45 | CONTAINER_INSTANCES, 46 | CONTAINER_INSTANCES_ARNS, 47 | SERVICES, 48 | SERVICES_ARNS, 49 | TASK_DEFINITION_FAMILIES, 50 | TASK_DEFINITIONS, 51 | TASK_DEFINITIONS_ARNS, 52 | TASKS, 53 | TASK_ARNS 54 | ] 55 | 56 | NEXT_TOKEN_ARGUMENT = "nextToken" 57 | NEXT_TOKEN_RESULT = "nextToken" 58 | 59 | MAPPED_PARAMETERS = {} 60 | 61 | 62 | class EcsService(AwsService): 63 | def __init__(self, role_arn=None, session=None, tags_as_dict=True, as_named_tuple=False, service_retry_strategy=None): 64 | """ 65 | :param role_arn: Optional (cross account) role to use to retrieve services 66 | :param session: Optional session to use to retrieve services 67 | :param tags_as_dict: Set to True true to convert resource tags to dictionaries 68 | :param as_named_tuple: Set to True to return resources as named tuples instead of a dictionary 69 | :param service_retry_strategy: service retry strategy for making boto api calls 70 | """ 71 | 72 | AwsService.__init__(self, service_name='ecs', 73 | resource_names=RESOURCE_NAMES, 74 | role_arn=role_arn, 75 | session=session, 76 | tags_as_dict=tags_as_dict, 77 | as_named_tuple=as_named_tuple, 78 | custom_result_paths=CUSTOM_RESULT_PATHS, 79 | mapped_parameters=MAPPED_PARAMETERS, 80 | next_token_argument=NEXT_TOKEN_ARGUMENT, 81 | next_token_result=NEXT_TOKEN_RESULT, 82 | service_retry_strategy=service_retry_strategy) 83 | 84 | def describe_resources_function_name(self, resource_name): 85 | """ 86 | Returns the name of the boto client method call to retrieve the specified resource. 87 | :param resource_name: 88 | :return: Name of the boto3 client function to retrieve the specified resource type 89 | """ 90 | s = AwsService.describe_resources_function_name(self, resource_name) 91 | if resource_name in [CLUSTERS_ARNS, 92 | CONTAINER_INSTANCES_ARNS, 93 | SERVICES_ARNS, 94 | TASK_DEFINITION_FAMILIES, 95 | TASK_DEFINITIONS_ARNS, 96 | TASK_ARNS]: 97 | return s.replace("describe_", "list_") 98 | return s 99 | 100 | def _map_describe_function_parameters(self, resources, args): 101 | if len(args) == 0: 102 | return args 103 | temp = AwsService._map_describe_function_parameters(self, resources, args) 104 | # for this service arguments start with lowercase 105 | translated = {b[0].lower() + b[1:]: temp[b] for b in temp} 106 | return translated 107 | 108 | 109 | -------------------------------------------------------------------------------- /source/code/services/tagging_service.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from services.aws_service import AwsService 14 | 15 | 16 | RESOURCES = "Resources" 17 | 18 | 19 | CUSTOM_RESULT_PATHS = { 20 | RESOURCES: "ResourceTagMappingList" 21 | } 22 | 23 | RESOURCE_NAMES = [ 24 | RESOURCES 25 | ] 26 | 27 | NEXT_TOKEN_ARGUMENT = "PaginationToken" 28 | NEXT_TOKEN_RESULT = "PaginationToken" 29 | 30 | MAPPED_PARAMETERS = {} 31 | 32 | 33 | class TaggingService(AwsService): 34 | def __init__(self, role_arn=None, session=None, tags_as_dict=True, as_named_tuple=False, service_retry_strategy=None): 35 | """ 36 | :param role_arn: Optional (cross account) role to use to retrieve services 37 | :param session: Optional session to use to retrieve services 38 | :param tags_as_dict: Set to True true to convert resource tags to dictionaries 39 | :param as_named_tuple: Set to True to return resources as named tuples instead of a dictionary 40 | :param service_retry_strategy: service retry strategy for making boto api calls 41 | """ 42 | 43 | AwsService.__init__(self, service_name='resourcegroupstaggingapi', 44 | resource_names=RESOURCE_NAMES, 45 | role_arn=role_arn, 46 | session=session, 47 | tags_as_dict=tags_as_dict, 48 | as_named_tuple=as_named_tuple, 49 | custom_result_paths=CUSTOM_RESULT_PATHS, 50 | mapped_parameters=MAPPED_PARAMETERS, 51 | next_token_argument=NEXT_TOKEN_ARGUMENT, 52 | next_token_result=NEXT_TOKEN_RESULT, 53 | service_retry_strategy=service_retry_strategy) 54 | 55 | def describe_resources_function_name(self, resource_name): 56 | """ 57 | Returns the name of the boto client method call to retrieve the specified resource. 58 | :param resource_name: 59 | :return: Name of the boto3 client function to retrieve the specified resource type 60 | """ 61 | s = AwsService.describe_resources_function_name(self, resource_name) 62 | return s.replace("describe_", "get_") 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /source/code/services/time_service.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import datetime 14 | 15 | import pytz 16 | import services 17 | from helpers import as_namedtuple 18 | from services.aws_service import AwsService 19 | 20 | RESOURCE_NAMES = [] 21 | 22 | 23 | class TimeService(AwsService): 24 | """ 25 | This is a pseudo service class to let the scheduler use the current UTC time as a resource 26 | """ 27 | 28 | def __init__(self, role_arn=None, session=None, as_named_tuple=False, service_retry_strategy=None): 29 | """ 30 | :param role_arn: Optional (cross account) role to use to retrieve services 31 | :param session: Optional session to use to retrieve services 32 | :param as_named_tuple: Set to True to return resources as named tuples instead of a dictionary 33 | """ 34 | AwsService.__init__(self, service_name='time', 35 | resource_names=[], 36 | role_arn=role_arn, 37 | session=session, 38 | tags_as_dict=False, 39 | as_named_tuple=as_named_tuple, 40 | service_retry_strategy=service_retry_strategy) 41 | 42 | def describe(self, as_tuple=None, **kwargs): 43 | """ 44 | This method is to retrieve a pseudo UTC time resource, method parameters are only used signature compatibility 45 | :param as_tuple: Set to true to return results as immutable named dictionaries instead of dictionaries 46 | :return: Pseudo time resource 47 | """ 48 | 49 | def use_tuple(): 50 | return (as_tuple is not None and as_tuple) or (as_tuple is None and self._as_tuple) 51 | 52 | region = kwargs.get("region") 53 | result = { 54 | "Time": datetime.datetime.now(pytz.timezone("UTC")), 55 | "AwsAccount": self.aws_account, 56 | "Region": region if region else services.get_session().region_name 57 | } 58 | 59 | return [as_namedtuple("Time", result)] if use_tuple() else [result] 60 | 61 | def service_regions(self): 62 | """ 63 | Regions that can be used for this service, return all AWS regions (assuming they all support EC2) 64 | :return: Service regions 65 | """ 66 | return services.get_session().get_available_regions(service_name="ec2") 67 | 68 | def get(self, region=None, as_tuple=None, **kwargs): 69 | """ 70 | Returns a pseudo time resource containing the current UTC time 71 | :param region: Not used, copied to resource 72 | :param as_tuple: Set to true to return results as immutable named dictionaries instead of dictionaries 73 | :return: Service resource of the specified resource type for the service, None if the resource was not available. 74 | """ 75 | return self.describe( 76 | region=region, as_tuple=as_tuple, **kwargs)[0] 77 | -------------------------------------------------------------------------------- /source/code/tagging/tag_filter_expression.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from .tag_filter_set import TagFilterSet 14 | 15 | EXPRESSION_OR = "|" 16 | EXPRESSION_AND = "&" 17 | EXPRESSION_ESCAPE_CHARACTER = '%' 18 | 19 | 20 | class TagFilterExpression(object): 21 | def __init__(self, filter_expression): 22 | self.filter_expression = filter_expression 23 | self.not_matching = None 24 | 25 | @classmethod 26 | def split_expression(cls, s): 27 | 28 | def next_char(): 29 | return s[i] if i < len(s) else None 30 | 31 | group_level = 0 32 | elements = [] 33 | i = 0 34 | element = "" 35 | 36 | c = next_char() 37 | 38 | escape = c == EXPRESSION_ESCAPE_CHARACTER 39 | 40 | while c is not None: 41 | 42 | if not escape: 43 | if c == "(": 44 | group_level += 1 45 | elif c == ")": 46 | group_level -= 1 47 | 48 | if c in [EXPRESSION_AND, EXPRESSION_OR] and group_level == 0: 49 | elements.append(element) 50 | elements.append(s[i]) 51 | element = "" 52 | else: 53 | element += c 54 | else: 55 | element += c 56 | escape = False 57 | 58 | i += 1 59 | c = next_char() 60 | escape = c == EXPRESSION_ESCAPE_CHARACTER and not escape 61 | if escape: 62 | i += 1 63 | c = next_char() 64 | 65 | if len(element) > 0: 66 | elements.append(element) 67 | return elements 68 | 69 | def is_match(self, tags_dict): 70 | 71 | elements = self.split_expression(self.filter_expression) 72 | temp = elements.pop(0) 73 | if temp[0] == "(": 74 | a = TagFilterExpression(temp[1:-1].strip()).is_match(tags_dict=tags_dict) 75 | else: 76 | element_filter = TagFilterSet(temp, filter_sep="|") 77 | if not element_filter.has_not_operator(): 78 | a = len(element_filter.pairs_matching_any_filter(tags_dict)) > 0 79 | else: 80 | a = element_filter.all_pairs_matching_filter(tags_dict) 81 | 82 | while len(elements) > 1: 83 | op = elements.pop(0) 84 | b_expression = elements.pop(0) 85 | b = TagFilterExpression(b_expression).is_match(tags_dict) 86 | if not b: 87 | self.not_matching = b_expression 88 | a = a and b if op == EXPRESSION_AND else a or b 89 | 90 | return a 91 | 92 | def get_filters(self): 93 | result = [] 94 | 95 | elements = self.split_expression(self.filter_expression) 96 | temp = elements.pop(0) 97 | if temp[0] == "(": 98 | a = TagFilterExpression(temp[1:-1].strip()).get_filters() 99 | result += a 100 | else: 101 | result.append(temp) 102 | 103 | while len(elements) > 1: 104 | elements.pop(0) 105 | b = TagFilterExpression(elements.pop(0)).get_filters() 106 | result += b 107 | 108 | return result 109 | 110 | def get_filter_keys(self): 111 | return set([f.split("=")[0] for f in self.get_filters()]) 112 | -------------------------------------------------------------------------------- /source/code/testing/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import os 14 | 15 | import helpers 16 | 17 | OPS_AUTOMATOR_ROLE_NAME = "OpsAutomatorRole" 18 | ACTION_STACK_NAME_TEMPLATE = "{}{}-ActionResources" 19 | ACTION_STACK_ROLE_NAME_TEMPLATE = "{}{}-TestRole" 20 | ENV_TEST_STACK_PREFIX = "TEST_STACK_PREFIX" 21 | 22 | 23 | def action_stack_name(tested_action): 24 | prefix = os.getenv(ENV_TEST_STACK_PREFIX, "") 25 | return helpers.snake_to_pascal_case(ACTION_STACK_NAME_TEMPLATE.format(prefix, tested_action)) 26 | 27 | 28 | def assumed_test_role_name(tested_action): 29 | prefix = os.getenv(ENV_TEST_STACK_PREFIX, "") 30 | return helpers.snake_to_pascal_case(ACTION_STACK_ROLE_NAME_TEMPLATE.format(prefix, tested_action)) 31 | 32 | 33 | TEMPLATE = { 34 | "AWSTemplateFormatVersion": "2010-09-09", 35 | "Resources": { 36 | "OpsAutomatorRole": { 37 | "Type": "AWS::IAM::Role", 38 | "Properties": { 39 | "RoleName": "", 40 | "AssumeRolePolicyDocument": { 41 | "Version": "2012-10-17", 42 | "Statement": [ 43 | { 44 | "Effect": "Allow", 45 | "Principal": { 46 | "AWS": "" 47 | }, 48 | "Action": "sts:AssumeRole" 49 | } 50 | 51 | ] 52 | }, 53 | "Policies": [ 54 | { 55 | "PolicyName": "OpsAutomatorRolePolicy", 56 | "PolicyDocument": {'Statement': []}}] 57 | } 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /source/code/testing/console_logger.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from actions import date_time_provider 14 | 15 | LOG_FORMAT = "{:0>4d}-{:0>2d}-{:0>2d} - {:0>2d}:{:0>2d}:{:0>2d}.{:0>3s} - {:7s} : {}" 16 | 17 | LOG_LEVEL_INFO = "INFO" 18 | LOG_LEVEL_ERROR = "ERROR" 19 | LOG_LEVEL_WARNING = "WARNING" 20 | LOG_LEVEL_DEBUG = "DEBUG" 21 | LOG_LEVEL_TEST = "TEST" 22 | 23 | 24 | # noinspection PyMethodMayBeStatic 25 | class ConsoleLogger(object): 26 | 27 | def __init__(self, debug=False): 28 | self._debug = debug 29 | 30 | def __enter__(self): 31 | return self 32 | 33 | def __exit__(self, exc_type, exc_val, exc_tb): 34 | self.flush() 35 | 36 | def _emit(self, level, msg, *args): 37 | s = msg if len(args) == 0 else msg.format(*args) 38 | dt = date_time_provider().now() 39 | s = LOG_FORMAT.format(dt.year, dt.month, dt.day, dt.hour, dt.minute, 40 | dt.second, str(dt.microsecond)[0:3], level, s) 41 | print(s) 42 | 43 | @property 44 | def debug_enabled(self): 45 | return self._debug 46 | 47 | @debug_enabled.setter 48 | def debug_enabled(self, value): 49 | self._debug = value 50 | 51 | def info(self, msg, *args): 52 | self._emit(LOG_LEVEL_INFO, msg, *args) 53 | 54 | def error(self, msg, *args): 55 | self._emit(LOG_LEVEL_ERROR, msg, *args) 56 | 57 | def warning(self, msg, *args): 58 | self._emit(LOG_LEVEL_WARNING, msg, *args) 59 | 60 | def test(self, msg, *args): 61 | self._emit(LOG_LEVEL_TEST, msg, *args) 62 | 63 | def debug(self, msg, *args): 64 | if self._debug: 65 | self._emit(LOG_LEVEL_DEBUG, msg, *args) 66 | 67 | def flush(self): 68 | pass 69 | -------------------------------------------------------------------------------- /source/code/testing/context.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import os 14 | import time 15 | 16 | import boto3 17 | 18 | import actions 19 | import handlers 20 | 21 | DEFAULT_TIMEOUT = 300 22 | 23 | 24 | class Context(object): 25 | def __init__(self, timeout_seconds=DEFAULT_TIMEOUT): 26 | self._started = time.time() 27 | self._timeout = timeout_seconds 28 | 29 | self.run_local = True 30 | lambda_function = "{}-{}-{}".format(os.getenv(handlers.ENV_STACK_NAME), os.getenv(handlers.ENV_LAMBDA_NAME), 31 | actions.ACTION_SIZE_STANDARD) 32 | 33 | self.log_group_name = "/aws/lambda/" + lambda_function 34 | 35 | self.invoked_function_arn = "arn:aws:lambda:{}:{}:function:{}".format(boto3.Session().region_name, 36 | os.getenv(handlers.ENV_OPS_AUTOMATOR_ACCOUNT), 37 | lambda_function) 38 | 39 | def get_remaining_time_in_millis(self): 40 | return max(self._timeout - (time.time() - self._started), 0) * 1000 41 | -------------------------------------------------------------------------------- /source/code/testing/datetime_provider.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | from datetime import datetime, timedelta 14 | 15 | _delta_ = timedelta() 16 | 17 | 18 | def set_datetime_delta(delta): 19 | global _delta_ 20 | _delta_ = delta 21 | 22 | 23 | # noinspection SpellCheckingInspection 24 | class DatetimeProvider(datetime): 25 | _delta_ = None 26 | 27 | @classmethod 28 | def now(cls, tz=None): 29 | dt = datetime.now(tz) 30 | return dt + _delta_ 31 | 32 | @classmethod 33 | def utcnow(cls, tz=None): 34 | dt = datetime.utcnow() 35 | return dt + _delta_ 36 | -------------------------------------------------------------------------------- /source/code/testing/dynamodb.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import time 14 | 15 | import boto3 16 | 17 | import services.dynamodb_service 18 | from helpers.timer import Timer 19 | 20 | 21 | class DynamoDB(object): 22 | 23 | def __init__(self, region=None, session=None): 24 | 25 | self.region = region if region is not None else boto3.Session().region_name 26 | self.session = session if session is not None else boto3.Session(region_name=self.region) 27 | self.ddb_client = self.session.client("dynamodb", region_name=self.region) 28 | self.ddb_service = services.dynamodb_service.DynamodbService(session=self.session) 29 | 30 | def wait_until_table_backups_available(self, table, timeout_seconds=30 * 60): 31 | with Timer(timeout_seconds=timeout_seconds, start=True) as t: 32 | 33 | while not t.timeout: 34 | try: 35 | # the only way to find out if backups are available for newly created table is to try to create a backup 36 | resp = self.ddb_client.create_backup(TableName=table, BackupName="is-backup-available") 37 | arn = resp.get("BackupDetails", {}).get("BackupArn", None) 38 | self.ddb_client.delete_backup(BackupArn=arn) 39 | return True 40 | except Exception as ex: 41 | if type(ex).__name__ != "ContinuousBackupsUnavailableException": 42 | return False 43 | 44 | time.sleep(30) 45 | 46 | return False 47 | 48 | def get_table(self, table_name): 49 | return self.ddb_service.get(services.dynamodb_service.TABLE, 50 | TableName=table_name, 51 | region=self.region, 52 | tags=True) 53 | 54 | def create_backup(self, table_name, backup_name): 55 | return self.ddb_client.create_backup(TableName=table_name, BackupName=backup_name).get("BackupDetails") 56 | 57 | def delete_backup(self, backup_arn, exception_if_not_exists=False): 58 | try: 59 | self.ddb_client.delete_backup(BackupArn=backup_arn) 60 | except Exception as e: 61 | if e.__class__.__name__ == "BackupNotFoundException": 62 | if exception_if_not_exists: 63 | raise e 64 | return 65 | raise e 66 | 67 | def delete_table_backups(self, table_name): 68 | for backup_arn in [s["BackupArn"] for s in self.get_table_backups(table_name) if s["BackupStatus"] == "AVAILABLE"]: 69 | self.delete_backup(backup_arn=backup_arn) 70 | 71 | def get_table_backups(self, table_name): 72 | return self.ddb_service.describe(services.dynamodb_service.BACKUPS, TableName=table_name) 73 | 74 | def create_tags(self, table_name, tags): 75 | arn = "arn:aws:dynamodb:{}:{}:table/{}".format(self.region, services.get_aws_account(), table_name) 76 | self.ddb_client.tag_resource(ResourceArn=arn, Tags=[{"Key": t, "Value": tags[t]} for t in tags]) 77 | -------------------------------------------------------------------------------- /source/code/testing/elb.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | import boto3 15 | 16 | import services.elb_service 17 | 18 | 19 | class Elb(object): 20 | 21 | def __init__(self, region=None, session=None): 22 | self.region = region if region is not None else boto3.Session().region_name 23 | self.session = session if session is not None else boto3.Session(region_name=self.region) 24 | self.elb_client = self.session.client("elb", region_name=self.region) 25 | self.elb_service = services.elb_service.ElbService(session=self.session) 26 | 27 | def register_instance(self, load_balancer_name, instance_id): 28 | self.elb_client.register_instances_with_load_balancer(LoadBalancerName=load_balancer_name, 29 | Instances=[ 30 | { 31 | "InstanceId": instance_id 32 | }, 33 | ]) 34 | 35 | def get_instance_load_balancers(self, instance_id): 36 | 37 | result = [] 38 | 39 | args = { 40 | "service_resource": services.elb_service.LOAD_BALANCERS, 41 | "region": self.region, 42 | } 43 | 44 | for lb in self.elb_service.describe(**args): 45 | for inst in lb.get("Instances", []): 46 | if inst["InstanceId"] != instance_id: 47 | continue 48 | if instance_id not in result: 49 | result.append(lb["LoadBalancerName"]) 50 | break 51 | 52 | return result 53 | -------------------------------------------------------------------------------- /source/code/testing/elbv2.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | import boto3 15 | 16 | import services.elbv2_service 17 | 18 | 19 | class ElbV2(object): 20 | 21 | def __init__(self, region=None, session=None): 22 | self.region = region if region is not None else boto3.Session().region_name 23 | self.session = session if session is not None else boto3.Session(region_name=self.region) 24 | self.elbv2_client = self.session.client("elbv2", region_name=self.region) 25 | self.elbv2_service = services.elbv2_service.Elbv2Service(session=self.session) 26 | 27 | def register_instance(self, target_group_arn, instance_id, port=None, availability_zone=None): 28 | target = { 29 | "Id": instance_id 30 | } 31 | if port is not None: 32 | target["Port"] = port 33 | 34 | if availability_zone is not None: 35 | target["AvailabilityZone"] = availability_zone 36 | 37 | self.elbv2_client.register_targets(TargetGroupArn=target_group_arn, Targets=[target]) 38 | 39 | def get_instance_target_groups(self, instance_id): 40 | 41 | result = [] 42 | 43 | args = { 44 | "service_resource": services.elbv2_service.TARGET_GROUPS, 45 | "region": self.region, 46 | 47 | } 48 | 49 | target_groups = list(self.elbv2_service.describe(**args)) 50 | for target_group in target_groups: 51 | target_group_healths = list(self.elbv2_service.describe(services.elbv2_service.TARGET_HEALTH, 52 | TargetGroupArn=target_group["TargetGroupArn"])) 53 | for target_group_health in target_group_healths: 54 | target = target_group_health["Target"] 55 | if target["Id"] != instance_id: 56 | continue 57 | result.append(target_group.get("TargetGroupArn")) 58 | 59 | return result 60 | -------------------------------------------------------------------------------- /source/code/testing/kms.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import boto3 14 | 15 | import services.kms_service 16 | 17 | 18 | class Kms(object): 19 | 20 | def __init__(self, region=None, session=None): 21 | self.region = region if region is not None else boto3.Session().region_name 22 | self.session = session if session is not None else boto3.Session(region_name=self.region) 23 | self.kms_client = self.session.client("kms", region_name=self.region) 24 | self.kms_service = services.create_service("kms", session=self.session) 25 | 26 | def get_kms_key(self, keyid): 27 | try: 28 | key = self.kms_service.get(services.kms_service.KEY, 29 | region=self.region, 30 | KeyId=keyid) 31 | return key 32 | except Exception as ex: 33 | if getattr(ex, "response", {}).get("Error", {}).get("Code") == "NotFoundException": 34 | if not keyid.startswith("arn") and not keyid.startswith("alias/"): 35 | return self.get_kms_key("alias/" + keyid) 36 | return None 37 | else: 38 | raise ex 39 | -------------------------------------------------------------------------------- /source/code/testing/report_output_writer.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import io 14 | import csv 15 | 16 | 17 | def create_output_writer(_=None, __=None): 18 | return ReportOutputWriter(_) 19 | 20 | 21 | def csv_to_dict_list(s): 22 | 23 | if s is None: 24 | return None 25 | 26 | result = [] 27 | cols = None 28 | try: 29 | reader = csv.reader(io.StringIO(s)) 30 | cols = next(reader) 31 | row = next(reader) 32 | 33 | while True: 34 | result.append({cols[i]: row[i] for i in list(range(0, len(cols)))}) 35 | row = next(reader) 36 | 37 | except StopIteration: 38 | if cols is None: 39 | return None 40 | else: 41 | return result 42 | 43 | 44 | # noinspection PyMethodMayBeStatic 45 | class ReportOutputWriter(object): 46 | 47 | def __init__(self, _): 48 | self._data_ = None 49 | 50 | def write(self, data, _): 51 | self._data_ = data 52 | 53 | @property 54 | def data(self): 55 | return self._data_ 56 | 57 | def reset(self): 58 | self._data_ = None 59 | -------------------------------------------------------------------------------- /source/code/testing/s3.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import boto3 14 | 15 | 16 | class S3(object): 17 | 18 | def __init__(self): 19 | self._s3_resources = None 20 | 21 | @property 22 | def s3_resources(self): 23 | if self._s3_resources is None: 24 | self._s3_resources = boto3.resource("s3") 25 | return self._s3_resources 26 | 27 | def empty_bucket(self, bucket, exception_if_not_exists=True): 28 | try: 29 | self.s3_resources.Bucket(bucket).objects.all().delete() 30 | except Exception as ex: 31 | if type(ex).__name__ == "NoSuchBucket" and not exception_if_not_exists: 32 | pass 33 | else: 34 | raise ex 35 | 36 | def get_object(self, bucket, key): 37 | # noinspection PyBroadException 38 | try: 39 | body = self.s3_resources.Object(bucket, key).get()["Body"] 40 | return body.read().decode('utf-8').split("\n") 41 | except Exception: 42 | return None 43 | 44 | def put_object(self, bucket, filename, key=None): 45 | if key is None: 46 | key = filename 47 | with open(filename) as f: 48 | self.s3_resources.Bucket(bucket).put_object(Key=key, Body=f) 49 | 50 | def get_bucket_tags(self, bucket): 51 | bucket_tagging = self.s3_resources.BucketTagging(bucket) 52 | return {t["Key"]: t["Value"] for t in bucket_tagging.tag_set} 53 | 54 | def delete_object(self, bucket, key, exception_if_bucket_not_exists=True): 55 | try: 56 | self.s3_resources.Object(bucket, key).delete() 57 | 58 | except Exception as ex: 59 | if type(ex).__name__ == "NoSuchBucket" and not exception_if_bucket_not_exists: 60 | pass 61 | else: 62 | raise ex 63 | 64 | def delete_bucket(self, bucket, exception_if_bucket_not_exists=True, delete_objects=True): 65 | try: 66 | if delete_objects: 67 | self.empty_bucket(bucket, exception_if_bucket_not_exists) 68 | self.s3_resources.Bucket(bucket).delete() 69 | 70 | except Exception as ex: 71 | if type(ex).__name__ == "NoSuchBucket" and not exception_if_bucket_not_exists: 72 | pass 73 | else: 74 | raise ex 75 | -------------------------------------------------------------------------------- /source/code/testing/sts.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import boto3 14 | 15 | 16 | class Sts(object): 17 | 18 | def __init__(self, session=None): 19 | self._sts_client = None 20 | self._session = session 21 | 22 | @property 23 | def sts_client(self): 24 | if self._sts_client is None: 25 | if self._session is None: 26 | self._session = boto3.Session() 27 | self._sts_client = self._session.client("sts") 28 | return self._sts_client 29 | 30 | @property 31 | def user_id(self): 32 | return self.sts_client.get_caller_identity().get("UserId") 33 | 34 | @property 35 | def account(self): 36 | return self.sts_client.get_caller_identity().get("Account") 37 | 38 | @property 39 | def identity_arn(self): 40 | return self.sts_client.get_caller_identity().get("Arn") 41 | -------------------------------------------------------------------------------- /source/code/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/aws-ops-automator/215ff25181b5d9193f92fa0eb84a6036d8212163/source/code/tests/__init__.py -------------------------------------------------------------------------------- /source/code/tests/action_tests/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import os 14 | import os.path 15 | 16 | import boto3 17 | 18 | import actions 19 | from actions.ec2_copy_snapshot_action import Ec2CopySnapshotAction 20 | from helpers import safe_json, snake_to_pascal_case 21 | from testing import action_stack_name, ENV_TEST_STACK_PREFIX 22 | from testing.ec2 import Ec2 23 | from testing.stack import Stack 24 | from testing.task_test_runner import TaskTestRunner 25 | 26 | TESTED_REGION = boto3.Session().region_name 27 | TESTED_REMOTE_REGION = "eu-central-1" 28 | assert (TESTED_REGION != TESTED_REMOTE_REGION) 29 | 30 | TASKLIST_TAGNAME = "{}TestTaskList" 31 | RESOURCE_STACK_NAME_TEMPLATE = "{}{}-TestResources" 32 | 33 | 34 | def template_path(tested_module, template_name): 35 | return os.path.join(os.path.dirname(tested_module), template_name) 36 | 37 | 38 | def tasklist_tagname(tested_action): 39 | return snake_to_pascal_case(TASKLIST_TAGNAME.format(tested_action)) 40 | 41 | 42 | def region(): 43 | return TESTED_REGION 44 | 45 | 46 | def remote_region(): 47 | return TESTED_REMOTE_REGION 48 | 49 | 50 | def resources_stack_name(tested_action): 51 | prefix = os.getenv(ENV_TEST_STACK_PREFIX, "") 52 | return snake_to_pascal_case(RESOURCE_STACK_NAME_TEMPLATE.format(prefix, tested_action)) 53 | 54 | 55 | def get_resource_stack(tested_action, create_resource_stack_func, use_existing=False, region_name=None): 56 | stack_region = region_name if region_name is not None else region() 57 | resource_stack_name = resources_stack_name(tested_action) 58 | resource_stack = Stack(resource_stack_name, owned=False, region=stack_region) 59 | if not use_existing or not resource_stack.is_stack_present(): 60 | resource_stack = create_resource_stack_func(resource_stack_name) 61 | assert (resource_stack is not None) 62 | resource_stack.owned = True 63 | return resource_stack 64 | 65 | 66 | # noinspection PyUnusedLocal 67 | def get_task_runner(tested_action, use_existing_action_stack=False, interval=None): 68 | stack_name = action_stack_name(tested_action) 69 | if not use_existing_action_stack: 70 | action_stack = Stack(stack_name=stack_name, owned=True, region=region()) 71 | if action_stack.is_stack_present(): 72 | action_stack.delete_stack(1800) 73 | task_runner = TaskTestRunner(action_name=tested_action, 74 | action_stack_name=stack_name, 75 | task_list_tag_name=tasklist_tagname(tested_action), 76 | tested_region=region()) 77 | task_runner.create_stack() 78 | return task_runner 79 | 80 | 81 | def set_snapshot_sources_tags(snapshot_id, source_volume_id=None, source_snapshot_id=None, region_name=None): 82 | tags = {} 83 | if source_volume_id is not None: 84 | tags[actions.marker_snapshot_tag_source_source_volume_id()] = source_volume_id 85 | if source_snapshot_id is not None: 86 | tags[Ec2CopySnapshotAction.marker_tag_source_snapshot_id()] = source_snapshot_id 87 | if len(tags) > 0: 88 | ec2 = Ec2(region_name) 89 | ec2.create_tags([snapshot_id], tags) 90 | 91 | 92 | def set_snapshot_copied_tag(snapshot_id, task_name, destination_region, copy_snapshot_id, region_name=None): 93 | copied_tag_name = Ec2CopySnapshotAction.marker_tag_copied_to(task_name) 94 | tags = { 95 | copied_tag_name: safe_json( 96 | {"region": destination_region, 97 | "snapshot-id": copy_snapshot_id}) 98 | } 99 | ec2 = Ec2(region_name) 100 | ec2.create_tags(snapshot_id, tags) 101 | -------------------------------------------------------------------------------- /source/code/tests/action_tests/dynamodb_set_capacity/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### -------------------------------------------------------------------------------- /source/code/tests/action_tests/dynamodb_set_capacity/test_resources.template: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Test stack for Set DynamoDB Capacity", 4 | "Parameters": { 5 | "TaskListTagName": { 6 | "Description": "Name of the task list tag.", 7 | "Type": "String" 8 | }, 9 | "TaskListTagValue": { 10 | "Description": "Value of the task list tag for the instance.", 11 | "Type": "String" 12 | }, 13 | "IndexName": { 14 | "Description": "Name of the Global Secondary Index.", 15 | "Type": "String" 16 | } 17 | }, 18 | "Outputs": { 19 | "TableName": { 20 | "Description": "Table name", 21 | "Value": { 22 | "Ref": "CreateDynamDBBackupTestTable" 23 | } 24 | } 25 | }, 26 | "Resources": { 27 | "CreateDynamDBBackupTestTable": { 28 | "Type": "AWS::DynamoDB::Table", 29 | "Metadata": { 30 | "cfn_nag": { 31 | "rules_to_suppress": [ 32 | { 33 | "id": "W73", 34 | "reason": "This template is only for test cases" 35 | }, 36 | { 37 | "id": "W74", 38 | "reason": "This template is only for test cases" 39 | } 40 | ] 41 | } 42 | }, 43 | "Properties": { 44 | "AttributeDefinitions": [ 45 | { 46 | "AttributeName": "Id", 47 | "AttributeType": "S" 48 | } 49 | ], 50 | "KeySchema": [ 51 | { 52 | "AttributeName": "Id", 53 | "KeyType": "HASH" 54 | } 55 | ], 56 | "ProvisionedThroughput": { 57 | "ReadCapacityUnits": 1, 58 | "WriteCapacityUnits": 1 59 | }, 60 | "GlobalSecondaryIndexes": [ 61 | { 62 | "IndexName": { 63 | "Ref": "IndexName" 64 | }, 65 | "KeySchema": [ 66 | { 67 | "AttributeName": "Id", 68 | "KeyType": "HASH" 69 | } 70 | ], 71 | "Projection": { 72 | "ProjectionType": "ALL" 73 | }, 74 | "ProvisionedThroughput": { 75 | "ReadCapacityUnits": 1, 76 | "WriteCapacityUnits": 1 77 | } 78 | } 79 | ], 80 | "Tags": [ 81 | { 82 | "Key": { 83 | "Ref": "TaskListTagName" 84 | }, 85 | "Value": { 86 | "Ref": "TaskListTagValue" 87 | } 88 | } 89 | ] 90 | } 91 | } 92 | } 93 | } 94 | 95 | -------------------------------------------------------------------------------- /source/code/tests/action_tests/ec2_copy_snapshot/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### -------------------------------------------------------------------------------- /source/code/tests/action_tests/ec2_copy_snapshot/test_resources_destination_region.template: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Ec2CopySnapshotTestResourceStack", 4 | "Outputs": { 5 | "EncryptionKeyArn": { 6 | "Description": "Arn of custom key used for encrypting EBS snapshots", 7 | "Value": { 8 | "Fn::GetAtt": [ 9 | "EncryptionKey0", 10 | "Arn" 11 | ] 12 | } 13 | } 14 | }, 15 | "Parameters": { 16 | "TaskRole": { 17 | "Description": "Role executing the action that needs permission to KMS key", 18 | "Type": "String" 19 | } 20 | }, 21 | "Resources": { 22 | "EncryptionKey0": { 23 | "Metadata": { 24 | "cfn_nag": { 25 | "rules_to_suppress": [ 26 | { 27 | "id": "F19", 28 | "reason": "Template for unit test only" 29 | } 30 | ] 31 | } 32 | }, 33 | "Type": "AWS::KMS::Key", 34 | "Properties": { 35 | "Enabled": "True", 36 | "EnableKeyRotation": "False", 37 | "Description": "Ec2CopySnapshotTestResourceStack EBS custom encryption key ", 38 | "KeyPolicy": { 39 | "Version": "2012-10-17", 40 | "Id": "key-default-1", 41 | "Statement": [ 42 | { 43 | "Sid": "Allow administration of the key", 44 | "Effect": "Allow", 45 | "Principal": { 46 | "AWS": { 47 | "Fn::Sub": "arn:aws:iam::${AWS::AccountId}:root" 48 | } 49 | }, 50 | "Action": [ 51 | "kms:Create*", 52 | "kms:Decrypt", 53 | "kms:Describe*", 54 | "kms:Enable*", 55 | "kms:Encrypt", 56 | "kms:GenerateDataKey*", 57 | "kms:ReEncrypt*", 58 | "kms:List*", 59 | "kms:Put*", 60 | "kms:Update*", 61 | "kms:Revoke*", 62 | "kms:Disable*", 63 | "kms:Get*", 64 | "kms:Delete*", 65 | "kms:ScheduleKeyDeletion", 66 | "kms:CancelKeyDeletion" 67 | ], 68 | "Resource": "*" 69 | }, 70 | { 71 | "Sid": "Allow use of the key", 72 | "Effect": "Allow", 73 | "Principal": { 74 | "AWS": { 75 | "Fn::Sub": "arn:aws:iam::${AWS::AccountId}:role/${TaskRole}" 76 | } 77 | }, 78 | "Action": [ 79 | "kms:CreateGrant", 80 | "kms:Decrypt", 81 | "kms:Get*", 82 | "kms:Describe*", 83 | "kms:List*", 84 | "kms:ReEncrypt*", 85 | "kms:GenerateDataKey*", 86 | "kms:DescribeKey" 87 | ], 88 | "Resource": "*" 89 | } 90 | ] 91 | } 92 | } 93 | } 94 | } 95 | } 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /source/code/tests/action_tests/ec2_create_snapshot/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### -------------------------------------------------------------------------------- /source/code/tests/action_tests/ec2_create_snapshot/test_resources.template: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Ec2CreateSnapshotTestResourceStack", 4 | "Outputs": { 5 | "InstanceId0": { 6 | "Description": "Instance Id", 7 | "Value": { 8 | "Ref": "Instance0" 9 | } 10 | }, 11 | "VolumeId0": { 12 | "Description": "Volume Id 0", 13 | "Value": { 14 | "Ref": "Volume0" 15 | } 16 | }, 17 | "VolumeId1": { 18 | "Description": "Volume Id 1", 19 | "Value": { 20 | "Ref": "Volume1" 21 | } 22 | } 23 | }, 24 | "Parameters": { 25 | "InstanceAmi": { 26 | "Description": "AMI ID for instances.", 27 | "Type": "String" 28 | }, 29 | "InstanceType": { 30 | "Description": "AMI Instance Type (t2.micro, m1.large, etc.)", 31 | "Type": "String" 32 | }, 33 | "TaskListTagName": { 34 | "Description": "Name of the task list tag.", 35 | "Type": "String" 36 | }, 37 | "TaskListTagValue": { 38 | "Description": "Value of the task list tag.", 39 | "Type": "String" 40 | } 41 | }, 42 | "Resources": { 43 | "Volume0": { 44 | "Metadata": { 45 | "cfn_nag": { 46 | "rules_to_suppress": [ 47 | { 48 | "id": "W37", 49 | "reason": "Unit test template - not deployed with solution" 50 | }, 51 | { 52 | "id": "F1", 53 | "reason": "Unit test template - not deployed with solution" 54 | } 55 | ] 56 | } 57 | }, 58 | "Type": "AWS::EC2::Volume", 59 | "Properties": { 60 | "Size": 10, 61 | "AvailabilityZone": { 62 | "Fn::Select": [ 63 | 0, 64 | { 65 | "Fn::GetAZs": "" 66 | } 67 | ] 68 | }, 69 | "Tags": [ 70 | { 71 | "Key": "Name", 72 | "Value": "Ec2CreateSnapshotTestDataVolume0" 73 | }, 74 | { 75 | "Key": "VolumeTag", 76 | "Value": "Volume0" 77 | } 78 | ] 79 | } 80 | }, 81 | "Volume1": { 82 | "Metadata": { 83 | "cfn_nag": { 84 | "rules_to_suppress": [ 85 | { 86 | "id": "W37", 87 | "reason": "Unit test template - not deployed with solution" 88 | }, 89 | { 90 | "id": "F1", 91 | "reason": "Unit test template - not deployed with solution" 92 | } 93 | ] 94 | } 95 | }, 96 | "Type": "AWS::EC2::Volume", 97 | "Properties": { 98 | "Size": 10, 99 | "AvailabilityZone": { 100 | "Fn::Select": [ 101 | 0, 102 | { 103 | "Fn::GetAZs": "" 104 | } 105 | ] 106 | }, 107 | "Tags": [ 108 | { 109 | "Key": "Name", 110 | "Value": "Ec2CreateSnapshotTestDataVolume1" 111 | }, 112 | { 113 | "Key": "Backup", 114 | "Value": "True" 115 | }, 116 | { 117 | "Key": "VolumeTag", 118 | "Value": "Volume1" 119 | } 120 | ] 121 | } 122 | }, 123 | "Instance0": { 124 | "Type": "AWS::EC2::Instance", 125 | "Properties": { 126 | "ImageId": { 127 | "Ref": "InstanceAmi" 128 | }, 129 | "InstanceType": { 130 | "Ref": "InstanceType" 131 | }, 132 | "Tags": [ 133 | { 134 | "Key": "Name", 135 | "Value": "Ec2CreateSnapshotTestInstance0" 136 | }, 137 | { 138 | "Key": { 139 | "Ref": "TaskListTagName" 140 | }, 141 | "Value": { 142 | "Ref": "TaskListTagValue" 143 | } 144 | }, 145 | { 146 | "Key": "InstanceTag", 147 | "Value": "Instance0" 148 | } 149 | ], 150 | "AvailabilityZone": { 151 | "Fn::Select": [ 152 | 0, 153 | { 154 | "Fn::GetAZs": "" 155 | } 156 | ] 157 | }, 158 | "Volumes": [ 159 | { 160 | "VolumeId": { 161 | "Ref": "Volume0" 162 | }, 163 | "Device": "/dev/sdg" 164 | }, 165 | { 166 | "VolumeId": { 167 | "Ref": "Volume1" 168 | }, 169 | "Device": "/dev/sdh" 170 | } 171 | ] 172 | } 173 | } 174 | } 175 | } -------------------------------------------------------------------------------- /source/code/tests/action_tests/ec2_delete_snapshot/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### -------------------------------------------------------------------------------- /source/code/tests/action_tests/ec2_delete_snapshot/test_resources.template: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Ec2CDeleteSnapshotTestResourceStack", 4 | "Outputs": { 5 | "VolumeId0": { 6 | "Description": "Volume Id 0", 7 | "Value": { 8 | "Ref": "Volume0" 9 | } 10 | } 11 | }, 12 | "Resources": { 13 | "Volume0": { 14 | "Metadata": { 15 | "cfn_nag": { 16 | "rules_to_suppress": [ 17 | { 18 | "id": "W37", 19 | "reason": "Unit test template - not deployed with solution" 20 | }, 21 | { 22 | "id": "F1", 23 | "reason": "Unit test template - not deployed with solution" 24 | } 25 | ] 26 | } 27 | }, 28 | "Type": "AWS::EC2::Volume", 29 | "Properties": { 30 | "Size": 10, 31 | "AvailabilityZone": { 32 | "Fn::Select": [ 33 | 0, 34 | { 35 | "Fn::GetAZs": "" 36 | } 37 | ] 38 | }, 39 | "Tags": [ 40 | { 41 | "Key": "Name", 42 | "Value": "Ec2DeleteSnapshotTestDataVolume0" 43 | }, 44 | { 45 | "Key": "VolumeTag", 46 | "Value": "Volume0" 47 | } 48 | ] 49 | } 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /source/code/tests/action_tests/ec2_replace_instance/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### -------------------------------------------------------------------------------- /source/code/tests/action_tests/ec2_replace_instance/test_resources.template: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Ec2ReplaceInstanceTestResourceStack", 4 | "Outputs": { 5 | "V1LoadBalancerName": { 6 | "Description": "ELB load balancer", 7 | "Value": { 8 | "Ref": "LoadBalancer" 9 | } 10 | }, 11 | "V2TargetGroupArn": { 12 | "Description": "TargetGroup", 13 | "Value": { 14 | "Ref": "TargetGroup" 15 | } 16 | }, 17 | "InstanceProfile": { 18 | "Description": "InstanceProfile", 19 | "Value": { 20 | "Ref": "InstanceProfile" 21 | } 22 | } 23 | }, 24 | "Parameters": { 25 | "VpcId": { 26 | "Description": "Vpc for load balancer", 27 | "Type": "String" 28 | } 29 | }, 30 | "Resources": { 31 | "InstanceRole": { 32 | "Metadata": { 33 | "cfn_nag": { 34 | "rules_to_suppress": [ 35 | { 36 | "id": "W11", 37 | "reason": "Unit test template - not deployed with solution" 38 | } 39 | ] 40 | } 41 | }, 42 | "Type": "AWS::IAM::Role", 43 | "Properties": { 44 | "AssumeRolePolicyDocument": { 45 | "Version": "2012-10-17", 46 | "Statement": [ 47 | { 48 | "Effect": "Allow", 49 | "Principal": { 50 | "Service": "ec2.amazonaws.com" 51 | }, 52 | "Action": "sts:AssumeRole" 53 | } 54 | ] 55 | }, 56 | "Policies": [ 57 | { 58 | "PolicyName": "Policy0", 59 | "PolicyDocument": { 60 | "Version": "2012-10-17", 61 | "Statement": [ 62 | { 63 | "Effect": "Allow", 64 | "Action": [ 65 | "ec2:DescribeInstances" 66 | ], 67 | "Resource": [ 68 | "*" 69 | ] 70 | } 71 | ] 72 | } 73 | } 74 | ], 75 | "Path": "/" 76 | } 77 | }, 78 | "InstanceProfile": { 79 | "Type": "AWS::IAM::InstanceProfile", 80 | "Properties": { 81 | "Roles": [ 82 | { 83 | "Ref": "InstanceRole" 84 | } 85 | ], 86 | "InstanceProfileName": { 87 | "Fn::Join": [ 88 | "-", 89 | [ 90 | "InstanceProfile", 91 | { 92 | "Ref": "AWS::StackName" 93 | } 94 | ] 95 | ] 96 | } 97 | } 98 | }, 99 | "LoadBalancer": { 100 | "Metadata": { 101 | "cfn_nag": { 102 | "rules_to_suppress": [ 103 | { 104 | "id": "W26", 105 | "reason": "Unit test template - not deployed with solution" 106 | } 107 | ] 108 | } 109 | }, 110 | "Type": "AWS::ElasticLoadBalancing::LoadBalancer", 111 | "Properties": { 112 | "AvailabilityZones": { 113 | "Fn::GetAZs": { 114 | "Ref": "AWS::Region" 115 | } 116 | }, 117 | "ConnectionDrainingPolicy": { 118 | "Enabled": "True", 119 | "Timeout": 20 120 | }, 121 | "Listeners": [ 122 | { 123 | "InstancePort": "80", 124 | "InstanceProtocol": "HTTP", 125 | "LoadBalancerPort": "80", 126 | "Protocol": "HTTP" 127 | } 128 | ] 129 | } 130 | }, 131 | "TargetGroup": { 132 | "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", 133 | "Properties": { 134 | "TargetGroupAttributes": [ 135 | { 136 | "Key": "deregistration_delay.timeout_seconds", 137 | "Value": "20" 138 | } 139 | ], 140 | "Port": "80", 141 | "Protocol": "HTTP", 142 | "VpcId": { 143 | "Ref": "VpcId" 144 | } 145 | } 146 | } 147 | } 148 | } 149 | 150 | 151 | 152 | 153 | 154 | 155 | -------------------------------------------------------------------------------- /source/code/tests/action_tests/ec2_resize_instance/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### -------------------------------------------------------------------------------- /source/code/tests/action_tests/ec2_resize_instance/test_resources.template: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Ec2ResizeInstanceTestResourceStack", 4 | "Outputs": { 5 | "InstanceId": { 6 | "Description": "Instance Id", 7 | "Value": { 8 | "Ref": "Instance0" 9 | } 10 | } 11 | }, 12 | "Parameters": { 13 | "InstanceAmi": { 14 | "Description": "AMI ID for instances.", 15 | "Type": "String" 16 | }, 17 | "InstanceType": { 18 | "Description": "AMI Instance Type (t2.micro, m1.large, etc.)", 19 | "Type": "String" 20 | } 21 | }, 22 | "Resources": { 23 | "Instance0": { 24 | "Type": "AWS::EC2::Instance", 25 | "Properties": { 26 | "ImageId": { 27 | "Ref": "InstanceAmi" 28 | }, 29 | "InstanceType": { 30 | "Ref": "InstanceType" 31 | }, 32 | "Tags": [ 33 | { 34 | "Key": "Name", 35 | "Value": "Ec2ResizeInstanceTestInstance0" 36 | } 37 | ], 38 | "AvailabilityZone": { 39 | "Fn::Select": [ 40 | 0, 41 | { 42 | "Fn::GetAZs": "" 43 | } 44 | ] 45 | } 46 | } 47 | } 48 | } 49 | } 50 | 51 | 52 | -------------------------------------------------------------------------------- /source/code/tests/action_tests/ec2_tag_cpu_instance/__init__.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### -------------------------------------------------------------------------------- /source/code/tests/action_tests/ec2_tag_cpu_instance/test_resources.template: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Ec2TagLoadInstancesTestResourceStack", 4 | "Outputs": { 5 | "InstanceNoCPULoad": { 6 | "Description": "Instance Id", 7 | "Value": { 8 | "Ref": "TestInstanceNoCPULoad" 9 | } 10 | }, 11 | "InstanceCPULoad": { 12 | "Description": "Instance Id with CPU load", 13 | "Value": { 14 | "Ref": "TestInstanceCPULoad" 15 | } 16 | } 17 | }, 18 | "Parameters": { 19 | "InstanceAmi": { 20 | "Description": "AMI ID for instances.", 21 | "Type": "String" 22 | }, 23 | "InstanceType": { 24 | "Description": "AMI Instance Type (t2.micro, m1.large, etc.)", 25 | "Type": "String" 26 | }, 27 | "TaskListTagName": { 28 | "Description": "Name of the task list tag.", 29 | "Type": "String" 30 | }, 31 | "TaskListTagValueNoCPULoad": { 32 | "Description": "Value of the task list tag.", 33 | "Type": "String" 34 | }, 35 | "TaskListTagValueCPULoad": { 36 | "Description": "Value of the task list tag.", 37 | "Type": "String" 38 | } 39 | }, 40 | "Resources": { 41 | "TestInstanceNoCPULoad": { 42 | "Type": "AWS::EC2::Instance", 43 | "Properties": { 44 | "ImageId": { 45 | "Ref": "InstanceAmi" 46 | }, 47 | "InstanceType": { 48 | "Ref": "InstanceType" 49 | }, 50 | "Tags": [ 51 | { 52 | "Key": "Name", 53 | "Value": "Ec2TagLoadInstanceTestInstanceNoLoad" 54 | }, 55 | { 56 | "Key": { 57 | "Ref": "TaskListTagName" 58 | }, 59 | "Value": { 60 | "Ref": "TaskListTagValueNoCPULoad" 61 | } 62 | } 63 | ], 64 | "AvailabilityZone": { 65 | "Fn::Select": [ 66 | 0, 67 | { 68 | "Fn::GetAZs": "" 69 | } 70 | ] 71 | } 72 | } 73 | }, 74 | "TestInstanceCPULoad": { 75 | "Type": "AWS::EC2::Instance", 76 | "Properties": { 77 | "ImageId": { 78 | "Ref": "InstanceAmi" 79 | }, 80 | "InstanceType": { 81 | "Ref": "InstanceType" 82 | }, 83 | "UserData": { 84 | "Fn::Base64": { 85 | "Fn::Sub": "#!/bin/bash\nyes > /dev/null &\nyes > /dev/null &\nyes > /dev/null &\nyes > /dev/null &\n" 86 | } 87 | }, 88 | "Tags": [ 89 | { 90 | "Key": "Name", 91 | "Value": "Ec2TagLoadInstanceTestInstanceLoad" 92 | }, 93 | { 94 | "Key": { 95 | "Ref": "TaskListTagName" 96 | }, 97 | "Value": { 98 | "Ref": "TaskListTagValueCPULoad" 99 | } 100 | } 101 | ], 102 | "AvailabilityZone": { 103 | "Fn::Select": [ 104 | 0, 105 | { 106 | "Fn::GetAZs": "" 107 | } 108 | ] 109 | } 110 | } 111 | } 112 | } 113 | } -------------------------------------------------------------------------------- /source/code/tests/configuration_tests/test_hour_setbuilder.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import unittest 14 | 15 | from scheduling.hour_setbuilder import HourSetBuilder 16 | 17 | 18 | class TestHourSetBuilder(unittest.TestCase): 19 | def test_name(self): 20 | for i in range(0, 24): 21 | self.assertEqual(HourSetBuilder().build(str(i)), {i}) 22 | 23 | for i in range(1, 11): 24 | self.assertEqual(HourSetBuilder().build(str(i) + "am"), {i}) 25 | self.assertEqual(HourSetBuilder().build(str(i) + "AM"), {i}) 26 | 27 | for i in range(1, 11): 28 | self.assertEqual(HourSetBuilder().build(str(i) + "pm"), {i + 12}) 29 | self.assertEqual(HourSetBuilder().build(str(i) + "PM"), {i + 12}) 30 | 31 | self.assertEqual(HourSetBuilder().build("12am"), {0}) 32 | self.assertEqual(HourSetBuilder().build("12pm"), {12}) 33 | 34 | def test_exceptions(self): 35 | for h in range(13, 25): 36 | self.assertRaises(ValueError, HourSetBuilder().build, (str(h) + "PM")) 37 | self.assertRaises(ValueError, HourSetBuilder().build, "PM") 38 | 39 | self.assertRaises(ValueError, HourSetBuilder().build, "24") 40 | self.assertRaises(ValueError, HourSetBuilder().build, "-1") 41 | -------------------------------------------------------------------------------- /source/code/tests/configuration_tests/test_minute_setbuilder.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import unittest 14 | 15 | from scheduling.minute_setbuilder import MinuteSetBuilder 16 | 17 | 18 | class TestMinuteSetBuilder(unittest.TestCase): 19 | def test_name(self): 20 | for i in range(0, 59): 21 | self.assertEqual(MinuteSetBuilder().build(str(i)), {i}) 22 | 23 | def test_exceptions(self): 24 | self.assertRaises(ValueError, MinuteSetBuilder().build, "60") 25 | -------------------------------------------------------------------------------- /source/code/tests/configuration_tests/test_month_setbuilder.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import calendar 14 | import unittest 15 | 16 | from scheduling.month_setbuilder import MonthSetBuilder 17 | 18 | 19 | class TestMonthSetBuilder(unittest.TestCase): 20 | def test_name(self): 21 | # abbreviations 22 | for i, name in enumerate(calendar.month_abbr[1:]): 23 | self.assertEqual(MonthSetBuilder().build(name), {i + 1}) 24 | self.assertEqual(MonthSetBuilder().build(name.lower()), {i + 1}) 25 | self.assertEqual(MonthSetBuilder().build(name.upper()), {i + 1}) 26 | 27 | # full names 28 | for i, name in enumerate(calendar.month_name[1:]): 29 | self.assertEqual(MonthSetBuilder().build(name), {i + 1}) 30 | self.assertEqual(MonthSetBuilder().build(name.lower()), {i + 1}) 31 | self.assertEqual(MonthSetBuilder().build(name.upper()), {i + 1}) 32 | 33 | def test_value(self): 34 | for i in range(1, 12): 35 | self.assertEqual(MonthSetBuilder().build(str(i)), {i}) 36 | -------------------------------------------------------------------------------- /source/code/tests/configuration_tests/test_monthday_setbuilder.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import calendar 14 | import unittest 15 | 16 | from scheduling.monthday_setbuilder import MonthdaySetBuilder 17 | 18 | 19 | class TestMonthdaySetBuilder(unittest.TestCase): 20 | def test_name(self): 21 | years = [2016, 2017] # leap and normal year 22 | 23 | for year in years: 24 | for month in range(1, 13): 25 | _, days = calendar.monthrange(year, month) 26 | 27 | for day in range(1, days): 28 | self.assertEqual(MonthdaySetBuilder(year, month).build(str(day)), {day}) 29 | 30 | def test_L_wildcard(self): 31 | years = [2016, 2017] # leap and normal year 32 | 33 | for year in years: 34 | for month in range(1, 13): 35 | _, days = calendar.monthrange(year, month) 36 | self.assertEqual(MonthdaySetBuilder(year, month).build("L"), {days}) 37 | 38 | def test_W_wildcard(self): 39 | years = [2016, 2017] # leap and normal year 40 | 41 | for year in years: 42 | for month in range(1, 13): 43 | _, days = calendar.monthrange(year, month) 44 | 45 | for day in range(1, days): 46 | weekday = calendar.weekday(year, month, day) 47 | result = day 48 | if weekday == 5: 49 | result = day - 1 if day > 1 else day + 2 50 | elif weekday == 6: 51 | result = day + 1 if day < days else day - 2 52 | 53 | self.assertEqual(MonthdaySetBuilder(year, month).build(str(day) + "W"), {result}) 54 | 55 | def test_exceptions(self): 56 | for h in range(13, 25): 57 | self.assertRaises(ValueError, MonthdaySetBuilder(2016, 1).build, "W") 58 | self.assertRaises(ValueError, MonthdaySetBuilder(2016, 1).build, "32W") 59 | -------------------------------------------------------------------------------- /source/code/tests/configuration_tests/test_tag_expression.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import copy 14 | import unittest 15 | 16 | from tagging.tag_filter_expression import TagFilterExpression 17 | 18 | 19 | class TestTagFilterExpression(unittest.TestCase): 20 | 21 | def test_is_match(self): 22 | tags1 = {"A": "B"} 23 | tags2 = {"CD": "EFG"} 24 | tags3 = copy.copy(tags1) 25 | tags3.update(copy.copy(tags2)) 26 | 27 | self.assertTrue(TagFilterExpression("A=B").is_match(tags1)) 28 | self.assertTrue(TagFilterExpression("A=B").is_match(tags3)) 29 | self.assertFalse(TagFilterExpression("A=B").is_match(tags2)) 30 | self.assertTrue(TagFilterExpression("A=B&CD=EFG").is_match(tags3)) 31 | self.assertTrue(TagFilterExpression("A=!B|CD=EFG").is_match(tags3)) 32 | self.assertFalse(TagFilterExpression("A=!B|CD=EFG").is_match(tags1)) 33 | 34 | self.assertFalse(TagFilterExpression("A=!B|CD=!EFG").is_match(tags3)) 35 | self.assertFalse(TagFilterExpression("A=!B&CD=!EFG").is_match(tags3)) 36 | self.assertFalse(TagFilterExpression("!A&CD=EFG").is_match(tags3)) 37 | self.assertTrue(TagFilterExpression("!A=B|CD=EFG").is_match(tags3)) 38 | self.assertFalse(TagFilterExpression("A=B&!CD=EFG").is_match(tags3)) 39 | self.assertFalse(TagFilterExpression("!A=B|!CD=E*").is_match(tags3)) 40 | self.assertFalse(TagFilterExpression("!A=B|!CD=E*").is_match(tags1)) 41 | self.assertTrue(TagFilterExpression("(A=X|A=B").is_match(tags1)) 42 | self.assertFalse(TagFilterExpression("(A=X|A=B").is_match(tags2)) 43 | self.assertTrue(TagFilterExpression("(A=X|A=Y|CD=*").is_match(tags3)) 44 | self.assertTrue(TagFilterExpression("(A=B&CD=!XYZ").is_match(tags3)) 45 | self.assertTrue(TagFilterExpression("(A=B&CD=XYZ)|(A=Z|CD=EFG)").is_match(tags3)) 46 | 47 | self.assertFalse(TagFilterExpression("A=B&!CD=E*").is_match(tags3)) 48 | self.assertFalse(TagFilterExpression("A=B&!CD=!E*").is_match(tags3)) 49 | self.assertTrue(TagFilterExpression("A=B|!CD=!E*").is_match(tags3)) 50 | self.assertTrue(TagFilterExpression("A=1,2,3").is_match({"A": "1,2,3"})) 51 | self.assertTrue(TagFilterExpression("A=*").is_match({"A": "1,2,3"})) 52 | 53 | def test_helper_functions(self): 54 | self.assertEqual(TagFilterExpression("A=B&CD=!XYZ").get_filters(), ["A=B", "CD=!XYZ"]) 55 | self.assertEqual(TagFilterExpression("(A=B&CD=!XYZ)|(A=Z|CD=EFG)").get_filters(), ["A=B", "CD=!XYZ", "A=Z", "CD=EFG"]) 56 | self.assertEqual(TagFilterExpression("A&*=!XYZ").get_filters(), ["A", "*=!XYZ"]) 57 | 58 | self.assertEqual(TagFilterExpression("A=B&CD=!XYZ").get_filter_keys(), {"A", "CD"}) 59 | self.assertEqual(TagFilterExpression("(A=B&CD=!XYZ)|(A=Z|CD=EFG)").get_filter_keys(), {"A", "CD"}) 60 | self.assertEqual(TagFilterExpression("A&*=!XYZ").get_filter_keys(), {"A", "*"}) 61 | -------------------------------------------------------------------------------- /source/code/update-build-number.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | import sys 14 | 15 | version_file = sys.argv[1] 16 | 17 | with open(version_file, "rt") as f: 18 | version = f.readline() 19 | 20 | build = int(version.split(".")[3]) + 1 21 | 22 | with open(version_file, "wt") as f: 23 | f.write("{}.{}".format(version[::-1].partition('.')[2][::-1], build)) 24 | -------------------------------------------------------------------------------- /source/ecs/Dockerfile: -------------------------------------------------------------------------------- 1 | # dockerfile for Ops Automator Image, version %version% 2 | FROM amazonlinux:latest 3 | ENV HOME /homes/ec2-user 4 | WORKDIR /homes/ec2-user 5 | ADD ops-automator-ecs-runner.py ./ 6 | ADD requirements.txt ./ 7 | RUN yum update -y && \ 8 | yum install -y python3 9 | RUN echo "alias python=python3" >> /homes/ec2-user/.bashrc 10 | RUN alias python=python3 && \ 11 | alias pip=pip3 && \ 12 | pip3 install boto3 && \ 13 | pip3 install -r requirements.txt -------------------------------------------------------------------------------- /source/ecs/build-and-deploy-image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ###################################################################################################################### 3 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 4 | # # 5 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 6 | # with the License. A copy of the License is located at # 7 | # # 8 | # http://www.apache.org/licenses/ # 9 | # # 10 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 11 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 12 | # and limitations under the License. # 13 | ###################################################################################################################### 14 | # If running Docker requires sudo to run on this system then also run this script with sudo 15 | function usage { 16 | echo "usage: $0 [--stack | -s] stackname [--region | -r] awsregion" 17 | } 18 | 19 | function do_cmd { 20 | cmd=$* 21 | $cmd 22 | if [ $? -gt 0 ]; 23 | then 24 | echo Command failed: ${cmd} 25 | exit 1 26 | fi 27 | } 28 | 29 | do_replace() { 30 | replace="s|$2|$3|g" 31 | file=$1 32 | sed -i -e $replace $file 33 | } 34 | 35 | #------------------------------------------------------------------------------------------------ 36 | 37 | echo "Build and deploy docker image for Ops Automator, version %version%" 38 | # Before running this script: 39 | # - you have created an ECS Docker cluster 40 | # - you have updated the OA stack with the name of the cluster 41 | # - Cloudformation has created the ECR repo 42 | # - you have the name of the stack 43 | 44 | while [[ $# -gt 1 ]] 45 | do 46 | 47 | key="$1" 48 | 49 | case ${key} in 50 | -r|--region) 51 | region="$2" 52 | shift # past argument 53 | ;; 54 | -s|--stack) 55 | stack="$2" 56 | shift # past argument 57 | ;; 58 | *) 59 | usage 60 | exit 1 61 | ;; 62 | esac 63 | shift # past argument or value 64 | done 65 | 66 | if [ "${region}" == "" ] 67 | then 68 | echo "Error: No region specified, use -r or --region to specify the region." 69 | usage 70 | exit 1 71 | fi 72 | 73 | if [ "${stack}" == "" ] 74 | then 75 | echo "Error: No stack name specified, use -s or --stack to specify the name of the Ops Automator stack." 76 | usage 77 | exit 1 78 | fi 79 | 80 | # Get repository from the stack parameters 81 | repository=`aws cloudformation describe-stacks --region ${region} --stack-name ${stack} --query "Stacks[0].Outputs[?OutputKey=='Repository']|[0].OutputValue" --output text` 82 | if [ "${repository}" == "" ] 83 | then 84 | echo "No repository in output of stack $(stack)" 85 | exit 1 86 | fi 87 | 88 | # Get account id 89 | accountid=`aws sts get-caller-identity --region ${region} --output text | sed 's/\t.*//'` 90 | 91 | image=ops-automator 92 | 93 | echo 94 | echo "Image is : " ${image} 95 | echo "repository is : " ${repository} 96 | echo "Region is : " ${region} 97 | echo 98 | 99 | echo "=== Creating Dockerfile ==" 100 | cp Dockerfile.orig Dockerfile 101 | do_replace Dockerfile '' ${repository} 102 | echo 103 | 104 | # Pulling latest AWS Linux image. Note that this repo/region must match FROM value in Docker file 105 | echo "=== Pulling latest AWS Linux image from DockerHub ===" 106 | do_cmd docker pull amazonlinux 107 | 108 | echo 109 | echo "=== Building docker image ===" 110 | do_cmd docker build -t ${image} . 111 | 112 | echo 113 | echo "=== Tagging and pushing image $image to $repository ===" 114 | do_cmd docker tag ${image}:latest ${repository}:latest 115 | 116 | login=`aws ecr get-login --region ${region} --no-include-email` 117 | do_cmd $login 118 | do_cmd docker push ${repository}:latest 119 | -------------------------------------------------------------------------------- /source/ecs/ops-automator-ecs-runner.py: -------------------------------------------------------------------------------- 1 | ###################################################################################################################### 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 3 | # # 4 | # Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance # 5 | # with the License. A copy of the License is located at # 6 | # # 7 | # http://www.apache.org/licenses/ # 8 | # # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 10 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 11 | # and limitations under the License. # 12 | ###################################################################################################################### 13 | 14 | 15 | # import imp 16 | import importlib 17 | import types 18 | import json 19 | import os 20 | import sys 21 | import zipfile 22 | 23 | import boto3 24 | import requests 25 | 26 | 27 | def get_lambda_code(cmdline_args): 28 | """ 29 | Downloads and unzips the code of the Lambda 30 | :param cmdline_args: 31 | :return: 32 | """ 33 | stack_name = cmdline_args["stack"] 34 | stack_region = cmdline_args["stack_region"] 35 | 36 | lambda_client = boto3.client("lambda", region_name=stack_region) 37 | 38 | os.environ["AWS_DEFAULT_REGION"] = stack_region 39 | 40 | lambda_function_name = "{}-{}".format(stack_name, "OpsAutomator-Standard") 41 | lambda_function = lambda_client.get_function(FunctionName=lambda_function_name) 42 | lambda_environment = lambda_function["Configuration"]["Environment"]["Variables"] 43 | 44 | for ev in lambda_environment: 45 | os.environ[ev] = lambda_environment[ev] 46 | 47 | code_url = lambda_function["Code"]["Location"] 48 | code_stream = requests.get(code_url, stream=True) 49 | 50 | temp_code_directory = "./" 51 | lambda_code_zip_file = os.path.join(temp_code_directory, "code.zip") 52 | with open(lambda_code_zip_file, 'wb') as fd: 53 | for chunk in code_stream.iter_content(chunk_size=10240): 54 | fd.write(chunk) 55 | 56 | zip_ref = zipfile.ZipFile(lambda_code_zip_file, 'r') 57 | zip_ref.extractall(temp_code_directory) 58 | zip_ref.close() 59 | 60 | return temp_code_directory 61 | 62 | 63 | def run_ops_automator_step(cmdline_args): 64 | """ 65 | Runs ecs_handler 66 | :param cmdline_args: arguments used by ecs_handler to rebuild event for Ops Automator select or execute handler 67 | :return: result of the Ops Automator handler 68 | """ 69 | code_directory = get_lambda_code(cmdline_args) 70 | 71 | # load main module 72 | main_module_file = os.path.join(code_directory, "main.py") 73 | 74 | spec = importlib.util.find_spec("main") 75 | try: 76 | lambda_main_module = spec.loader.create_module(spec) 77 | except AttributeError: 78 | lambda_main_module = None 79 | if lambda_main_module is None: 80 | lambda_main_module = types.ModuleType(spec.name) 81 | # No clear way to set import-related attributes. 82 | spec.loader.exec_module(lambda_main_module) 83 | 84 | lambda_function_ecs_handler = lambda_main_module.ecs_handler 85 | 86 | # get and run ecs_handler method 87 | return lambda_function_ecs_handler(cmdline_args) 88 | 89 | 90 | if __name__ == "__main__": 91 | 92 | print("Running Ops Automator ECS Job, version %version%") 93 | 94 | if len(sys.argv) < 2: 95 | print("No task arguments passed as first parameter") 96 | exit(1) 97 | 98 | args = {} 99 | 100 | try: 101 | args = json.loads(sys.argv[1]) 102 | except Exception as ex: 103 | print(("\"{}\" is not valid JSON, {}", sys.argv[1], ex)) 104 | exit(1) 105 | 106 | try: 107 | print(("Task arguments to run the job are\n {}".format(json.dumps(args, indent=3)))) 108 | print(("Result is {}".format(run_ops_automator_step(args)))) 109 | exit(0) 110 | 111 | except Exception as e: 112 | print(e) 113 | exit(1) 114 | -------------------------------------------------------------------------------- /source/version.txt: -------------------------------------------------------------------------------- 1 | 2.2.0 -------------------------------------------------------------------------------- /version.txt: -------------------------------------------------------------------------------- 1 | 2.2.0 --------------------------------------------------------------------------------