├── source ├── es_tools │ ├── es_tools.egg-info │ │ ├── dependency_links.txt │ │ ├── top_level.txt │ │ ├── requires.txt │ │ ├── entry_points.txt │ │ ├── PKG-INFO │ │ └── SOURCES.txt │ ├── setup.cfg │ ├── dist │ │ └── es_tools-0.1.4.tar.gz │ ├── es_tools │ │ ├── __init__.py │ │ ├── es_config.py │ │ ├── es_export.py │ │ └── es_import.py │ ├── setup.py │ ├── README.md │ └── export.json └── helper │ └── helper.py ├── NOTICE.txt ├── CODE_OF_CONDUCT.md ├── deployment ├── run-unit-tests.sh ├── build-s3-dist.sh └── cost-optimization-monitor.template ├── README.md ├── CONTRIBUTING.md └── LICENSE.txt /source/es_tools/es_tools.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /source/es_tools/es_tools.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | es_tools 2 | -------------------------------------------------------------------------------- /source/es_tools/setup.cfg: -------------------------------------------------------------------------------- 1 | [egg_info] 2 | tag_build = 3 | tag_date = 0 4 | tag_svn_revision = 0 5 | 6 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Cost Optimization Monitor 2 | Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /source/es_tools/es_tools.egg-info/requires.txt: -------------------------------------------------------------------------------- 1 | boto3>=1.9.120 2 | click>=7.0 3 | elasticsearch<7.0.0,>=6.0.0 4 | requests_aws4auth>=0.9 5 | -------------------------------------------------------------------------------- /source/es_tools/dist/es_tools-0.1.4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/cost-optimization-monitor/HEAD/source/es_tools/dist/es_tools-0.1.4.tar.gz -------------------------------------------------------------------------------- /source/es_tools/es_tools.egg-info/entry_points.txt: -------------------------------------------------------------------------------- 1 | [console_scripts] 2 | es-export = es_tools.es_export:cli_export 3 | es-import = es_tools.es_import:cli_import 4 | 5 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /source/es_tools/es_tools.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 1.0 2 | Name: es-tools 3 | Version: 0.1.4 4 | Summary: Elasticsearch backup and restore for kibana dashboards 5 | Home-page: UNKNOWN 6 | Author: AWS Solutions Builders 7 | Author-email: UNKNOWN 8 | License: Apache 2.0 9 | Description: UNKNOWN 10 | Platform: UNKNOWN 11 | -------------------------------------------------------------------------------- /deployment/run-unit-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script should be run from the repo's deployment directory 4 | # cd deployment 5 | # ./run-unit-tests.sh 6 | 7 | # Run unit tests 8 | echo "Running unit tests" 9 | echo "cd ../source" 10 | cd ../source 11 | echo "No unit tests to run, so sad ..." 12 | echo "Completed unit tests" 13 | -------------------------------------------------------------------------------- /source/es_tools/es_tools.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | README.md 2 | setup.cfg 3 | setup.py 4 | es_tools/__init__.py 5 | es_tools/es_config.py 6 | es_tools/es_export.py 7 | es_tools/es_import.py 8 | es_tools.egg-info/PKG-INFO 9 | es_tools.egg-info/SOURCES.txt 10 | es_tools.egg-info/dependency_links.txt 11 | es_tools.egg-info/entry_points.txt 12 | es_tools.egg-info/requires.txt 13 | es_tools.egg-info/top_level.txt -------------------------------------------------------------------------------- /source/es_tools/es_tools/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | ##################################################################################################################### 4 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 7 | # with the License. A copy of the License is located at # 8 | # # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 12 | # OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing # 13 | # permissions and limitations under the License. # 14 | ###################################################################################################################### 15 | 16 | __version__ = '0.1.4' 17 | -------------------------------------------------------------------------------- /source/es_tools/es_tools/es_config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | ##################################################################################################################### 4 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 7 | # with the License. A copy of the License is located at # 8 | # # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 12 | # OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing # 13 | # permissions and limitations under the License. # 14 | ###################################################################################################################### 15 | 16 | INDEX_NAME = '.kibana-4' 17 | FILE_NAME = 'export.json' 18 | TIMEOUT = 30 19 | ES_PORT = 80 20 | -------------------------------------------------------------------------------- /source/es_tools/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | ##################################################################################################################### 4 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 7 | # with the License. A copy of the License is located at # 8 | # # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 12 | # OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing # 13 | # permissions and limitations under the License. # 14 | ###################################################################################################################### 15 | 16 | import io 17 | import os 18 | import re 19 | from setuptools import setup 20 | 21 | def read(*names, **kwargs): 22 | with io.open( 23 | os.path.join(os.path.dirname(__file__), *names), 24 | encoding=kwargs.get("encoding", "utf8") 25 | ) as fp: 26 | return fp.read() 27 | 28 | 29 | def find_version(*file_paths): 30 | version_file = read(*file_paths) 31 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 32 | version_file, re.M) 33 | if version_match: 34 | return version_match.group(1) 35 | raise RuntimeError("Unable to find version string.") 36 | 37 | setup( 38 | name='es_tools', 39 | version=find_version("es_tools", "__init__.py"), 40 | packages=['es_tools'], 41 | url='', 42 | license='Apache 2.0', 43 | author='AWS Solutions Builders', 44 | description='Elasticsearch backup and restore for kibana dashboards', 45 | entry_points={ 46 | 'console_scripts': [ 47 | 'es-export = es_tools.es_export:cli_export', 48 | 'es-import = es_tools.es_import:cli_import' 49 | ] 50 | 51 | }, 52 | install_requires=[ 53 | 'boto3>=1.9.120', 54 | 'click>=7.0', 55 | 'elasticsearch>=6.0.0,<7.0.0', 56 | 'requests_aws4auth>=0.9' 57 | ] 58 | ) 59 | -------------------------------------------------------------------------------- /source/es_tools/README.md: -------------------------------------------------------------------------------- 1 | Elastic Search tools 2 | ==================== 3 | 4 | This program is aimed to export and import kibana dashboards from/to Elastic Search clusters. 5 | The default parameters are aimed to import/export the kibana dashboard that runs on AWS Elasticsearch Service with version 1.5 (The default kibana dashboard is .kibana-4) 6 | 7 | You can run the tool to export/import virtually any index on Elasticsearch just pointing the index name. But the code has no performance optimization to extract/upload index with thousands or millions of documents. 8 | In this case it's advisable to use a different tool to export/import data. 9 | 10 | Currently the tool support: 11 | 12 | - Index to import/export 13 | - AWS IAM authentication with static credentials or dynamic credentials with STS, Roles or environment variables 14 | 15 | Running 16 | ------- 17 | 18 | The package install 2 cli applicantions: 19 | 20 | - ``es-export`` (export the index to file) 21 | parameters 22 | 23 | Usage: es-export [OPTIONS] 24 | 25 | Elasticsearch Export program :return: 26 | 27 | Options: 28 | -i, --index Index name to export. Default: .kibana-4 29 | -f, --file JSON filename to export. Default: export.json 30 | -e, --es-host Elasticsearch host name or IP address. [required] 31 | -p, --es-port Elasticsearch port number. Default is 80 32 | -t, --timeout Elasticsearch timeout connection. Default is 30 33 | -r, --region Change the region to run the program. Default is 34 | us-east-1 35 | -vv, --verbose Verbose output. 36 | -v, --version Display version number and exit. 37 | --help Show this message and exit. 38 | 39 | - ``es-import`` (import the index from file to elasticsearch) 40 | parameters 41 | 42 | Usage: es-import [OPTIONS] 43 | 44 | Elasticsearch Import program :return: 45 | 46 | Options: 47 | -i, --index Index name to export. Default: .kibana-4 48 | -f, --file JSON filename to import in the Elasticsearch. 49 | Default: export.json [required] 50 | -e, --es-host Elasticsearch host name or IP address. [required] 51 | -p, --es-port Elasticsearch port number. Default is 80 52 | -t, --timeout Elasticsearch timeout connection. Default is 30 53 | -di, --delete-index Delete current index before import. Default: false 54 | -r, --region Change the region to run the program. Default is 55 | us-east-1 56 | -v, --version Display version number and exit. 57 | --help Show this message and exit. 58 | 59 | Changes 60 | ------- 61 | 62 | - Version 0.1.4 63 | 64 | Bug fixes 65 | 66 | - Version 0.1.3 67 | 68 | es_export wasn't getting the region parameter 69 | 70 | - Version 0.1.2 71 | 72 | Corrected a problem where the kibana mapping was uploaded as document instead of mapping 73 | 74 | - Version 0.1.0 75 | 76 | Initial version 77 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cost Optimization Monitor 2 | Source code for the AWS solution "Cost Optimization: Monitor". 3 | 4 | For the full solution overview visit [Cost Optimization: Monitor](https://aws.amazon.com/answers/account-management/cost-optimization-monitor/). 5 | 6 | ## File Structure 7 | ``` 8 | |-deployment/ [folder containing templates and build scripts] 9 | |-source/ 10 | |-es-tools/ [this program is aimed to export and import kibana dashboards from/to Elastic Search clusters] 11 | |-helper/ [custom helper for CloudFormation deployment auxiliary functions] 12 | ``` 13 | 14 | ## Getting Started 15 | 16 | #### 01. Prerequisites 17 | The following procedures assumes that all of the OS-level configuration has been completed. They are: 18 | 19 | * [AWS Command Line Interface](https://aws.amazon.com/cli/) 20 | * Python 3.x 21 | 22 | The latest version has been tested with Python v3.7. 23 | 24 | #### 02. Clone Cost Optimization Monitor repository 25 | Clone the cost-optimization-monitor GitHub repository: 26 | 27 | ``` 28 | git clone https://github.com/awslabs/cost-optimization-monitor.git 29 | ``` 30 | 31 | #### 03. Declare enviroment variables: 32 | ``` 33 | export AWS_REGION= 34 | export VERSION_CODE= 35 | export DEPLOY_BUCKET= 36 | ``` 37 | - **aws-region-code**: AWS region code. Ex: ```us-east-1```. 38 | - **version-code**: version of the package. EX: ```v1.1.0```. 39 | - **source-bucket-base-name**: Name for the S3 bucket location where the template will source the Lambda code from. The template will append ```-[aws-region-code]``` to this bucket name. For example: ```./build-s3-dist.sh solutions v1.1.0```, the template will then expect the source code to be located in the ```solutions-[aws-region-code]``` bucket. 40 | 41 | #### 04. Build the Cost Optimization Monitor solution for deployment: 42 | ``` 43 | cd ./cost-optimization-monitor/deployment 44 | chmod +x build-s3-dist.sh 45 | ./build-s3-dist.sh $DEPLOY_BUCKET $VERSION_CODE 46 | ``` 47 | #### 05. Upload deployment assets to your Amazon S3 bucket: 48 | ``` 49 | aws s3 cp ./dist s3://$DEPLOY_BUCKET-$AWS_REGION/cost-optimization-monitor/latest --recursive --acl bucket-owner-full-control 50 | aws s3 cp ./dist s3://$DEPLOY_BUCKET-$AWS_REGION/cost-optimization-monitor/$VERSION_CODE --recursive --acl bucket-owner-full-control 51 | ``` 52 | 53 | #### 06. Deploy the Cost Optimization Monitor solution: 54 | * From your designated Amazon S3 bucket where you uploaded the deployment assets, copy the link location for the cost-optimization-monitor.template. 55 | * Using AWS CloudFormation, launch the Cost Optimization Monitor solution stack using the copied Amazon S3 link for the cost-optimization-monitor.template. 56 | 57 | *** 58 | 59 | Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 60 | 61 | Licensed under the Amazon Software License (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 62 | 63 | http://aws.amazon.com/asl/ 64 | 65 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and limitations under the License. 66 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check [existing open](https://github.com/awslabs/cost-optimization-monitor/issues), or [recently closed](https://github.com/awslabs/cost-optimization-monitor/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/cost-optimization-monitor/labels/help%20wanted) issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](https://github.com/awslabs/cost-optimization-monitor/blob/master/LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /deployment/build-s3-dist.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # This assumes all of the OS-level configuration has been completed and git repo has already been cloned 4 | # 5 | # This script should be run from the repo's deployment directory 6 | # cd deployment 7 | # ./build-s3-dist.sh source-bucket-base-name version-code 8 | # 9 | # Parameters: 10 | # - source-bucket-base-name: Name for the S3 bucket location where the template will source the Lambda 11 | # code from. The template will append '-[region_name]' to this bucket name. 12 | # For example: ./build-s3-dist.sh solutions v1.1.0 13 | # The template will then expect the source code to be located in the solutions-[region_name] bucket 14 | # 15 | # - version-code: version of the package 16 | 17 | # Check to see if input has been provided: 18 | if [[ -z "$1" ]] || [[ -z "$2" ]]; then 19 | echo "Please provide the base source bucket name and version where the lambda code will eventually reside." 20 | echo "For example: ./build-s3-dist.sh solutions v1.1.0" 21 | exit 1 22 | fi 23 | 24 | # Get reference for all important folders 25 | template_dir="$PWD" 26 | dist_dir="$template_dir/dist" 27 | source_dir="$template_dir/../source" 28 | 29 | echo "------------------------------------------------------------------------------" 30 | echo "[Init] Clean old dist and es_tools.egg-info folders" 31 | echo "------------------------------------------------------------------------------" 32 | echo "rm -rf $dist_dir" 33 | rm -rf "$dist_dir" 34 | echo "find $source_dir -iname \"es_tools.egg-info\" -type d -exec rm -r \"{}\" \; 2> /dev/null" 35 | find "$source_dir" -iname "es_tools.egg-info" -type d -exec rm -r "{}" \; 2> /dev/null 36 | echo "find $source_dir -iname \"dist\" -type d -exec rm -r \"{}\" \; 2> /dev/null" 37 | find "$source_dir" -iname "dist" -type d -exec rm -r "{}" \; 2> /dev/null 38 | echo "find ../ -type f -name '.DS_Store' -delete" 39 | find "$source_dir" -type f -name '.DS_Store' -delete 40 | echo "mkdir -p $dist_dir" 41 | mkdir -p "$dist_dir" 42 | 43 | echo "------------------------------------------------------------------------------" 44 | echo "[Packing] Templates" 45 | echo "------------------------------------------------------------------------------" 46 | echo "cp -f $template_dir/cost-optimization-monitor.template dist" 47 | cp -f "$template_dir/cost-optimization-monitor.template" "$dist_dir" 48 | 49 | echo "Updating code source bucket in template with $1" 50 | replace="s/%%TEMPLATE_BUCKET_NAME%%/$1/g" 51 | echo "sed -i '' -e $replace $dist_dir/cost-optimization-monitor.template" 52 | sed -i '' -e "$replace" "$dist_dir"/cost-optimization-monitor.template 53 | 54 | echo "Updating code source version in template with $2" 55 | replace="s/%%VERSION%%/$2/g" 56 | echo "sed -i '' -e $replace $dist_dir/cost-optimization-monitor.template" 57 | sed -i '' -e "$replace" "$dist_dir"/cost-optimization-monitor.template 58 | 59 | echo "Updating dist bucket in template with $3" 60 | replace="s/%%DIST_BUCKET_NAME%%/$3/g" 61 | echo "sed -i '' -e $replace $dist_dir/cost-optimization-monitor.template" 62 | sed -i '' -e "$replace" "$dist_dir"/cost-optimization-monitor.template 63 | 64 | echo "------------------------------------------------------------------------------" 65 | echo "[Packing] ES Tools" 66 | echo "------------------------------------------------------------------------------" 67 | cd "$source_dir"/es_tools || exit 1 68 | python setup.py sdist 69 | cp "$source_dir"/es_tools/dist/*.gz "$dist_dir"/ 70 | cp "$source_dir"/es_tools/export.json "$dist_dir"/ 71 | 72 | echo "Updating code source version in dashboard description with $2" 73 | replace="s/%%VERSION%%/$2/g" 74 | echo "sed -i '' -e $replace $dist_dir/export.json" 75 | sed -i '' -e "$replace" "$dist_dir"/export.json 76 | 77 | echo "------------------------------------------------------------------------------" 78 | echo "[Packing] Helper" 79 | echo "------------------------------------------------------------------------------" 80 | cd "$source_dir"/helper || exit 1 81 | zip -q -r9 "$dist_dir"/helper.zip ./* 82 | 83 | -------------------------------------------------------------------------------- /source/es_tools/es_tools/es_export.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | ##################################################################################################################### 4 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 7 | # with the License. A copy of the License is located at # 8 | # # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 12 | # OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing # 13 | # permissions and limitations under the License. # 14 | ###################################################################################################################### 15 | 16 | import json 17 | 18 | import boto3 19 | import click 20 | from elasticsearch import Elasticsearch, RequestsHttpConnection 21 | from es_config import INDEX_NAME, FILE_NAME, TIMEOUT, ES_PORT 22 | from pkg_resources import require 23 | from requests_aws4auth import AWS4Auth 24 | 25 | 26 | @click.command() 27 | @click.option('-i', '--index', default=INDEX_NAME, metavar='', 28 | help='Index name to export. Default: {name}'.format(name=INDEX_NAME)) 29 | @click.option('-f', '--file', default=FILE_NAME, metavar='', 30 | help='JSON filename to export. Default: {name}'.format(name=FILE_NAME)) 31 | @click.option('-e', '--es-host', required=True, metavar='', help='Elasticsearch host name or IP address.') 32 | @click.option('-p', '--es-port', type=int, default=ES_PORT, metavar='', 33 | help='Elasticsearch port number. Default is {port}'.format(port=ES_PORT)) 34 | @click.option('-t', '--timeout', type=int, default=TIMEOUT, metavar='', 35 | help='Elasticsearch timeout connection. Default is {timeout}'.format(timeout=TIMEOUT)) 36 | @click.option( 37 | '-r', '--region', metavar='', default='us-east-1', 38 | help='Change the region to run the program. Default is us-east-1') 39 | @click.option('-vv', '--verbose', is_flag=True, default=False, help='Verbose output.') 40 | @click.option('-v', '--version', is_flag=True, default=False, help='Display version number and exit.') 41 | def cli_export(*args, **kwargs): 42 | """ 43 | Elasticsearch Export program 44 | :return: 45 | """ 46 | version = kwargs.pop('version') 47 | verbose = kwargs.pop('verbose') 48 | host = kwargs.pop('es_host') 49 | port = kwargs.pop('es_port') 50 | timeout = kwargs.pop('timeout') 51 | index = kwargs.pop('index') 52 | filename = kwargs.pop('file') 53 | 54 | # Import boto3 credentials 55 | session = boto3.Session() 56 | credentials = session.get_credentials() 57 | awsauth = None 58 | if credentials: 59 | region = kwargs.pop('region') 60 | awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, 'es', 61 | session_token=credentials.token) 62 | 63 | click.echo('Elasticsearch export utility version {0}'.format(require("es_tools")[0].version)) 64 | if version: 65 | return 66 | 67 | es = Elasticsearch([{'host': host, 'port': port}], timeout=timeout, http_auth=awsauth, 68 | connection_class=RequestsHttpConnection) 69 | output = open(filename, 'wb') 70 | mapping = es.indices.get_mapping(index) 71 | click.echo('[+] Extracting {name} mapping'.format(name=INDEX_NAME)) 72 | if verbose: 73 | click.echo(json.dumps(mapping[index])) 74 | output.write(json.dumps(mapping[index])) 75 | output.write('\n') 76 | 77 | results = es.search(index=index, body={"query": {"match_all": {}}}, scroll='10m', size=10000) 78 | click.echo('[+] Extracting {name} Documents'.format(name=INDEX_NAME)) 79 | documents = results['hits']['hits'] 80 | for doc in documents: 81 | doc.pop('_index', None) 82 | doc.pop('_score', None) 83 | if verbose: 84 | click.echo(json.dumps(doc)) 85 | output.write(json.dumps(doc)) 86 | output.write('\n') 87 | output.close() 88 | click.echo('[=] Finished processing') 89 | 90 | 91 | if __name__ == "__main__": 92 | cli_export() 93 | -------------------------------------------------------------------------------- /source/es_tools/es_tools/es_import.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | ##################################################################################################################### 4 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # 7 | # with the License. A copy of the License is located at # 8 | # # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 12 | # OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing # 13 | # permissions and limitations under the License. # 14 | ###################################################################################################################### 15 | 16 | import json 17 | 18 | import boto3 19 | import click 20 | from elasticsearch import Elasticsearch, RequestsHttpConnection 21 | from es_config import INDEX_NAME, FILE_NAME 22 | from pkg_resources import require 23 | from requests_aws4auth import AWS4Auth 24 | 25 | 26 | @click.command() 27 | @click.option('-i', '--index', default=INDEX_NAME, metavar='', 28 | help='Index name to export. Default: {name}'.format(name=INDEX_NAME)) 29 | @click.option('-f', '--file', required=True, metavar='', default='export.json', type=click.File('rb'), 30 | help='JSON filename to import in the Elasticsearch. Default: {name}'.format(name=FILE_NAME)) 31 | @click.option('-e', '--es-host', required=True, metavar='', help='Elasticsearch host name or IP address.') 32 | @click.option('-p', '--es-port', type=int, default=80, metavar='', 33 | help='Elasticsearch port number. Default is 80') 34 | @click.option('-t', '--timeout', type=int, default=30, metavar='', 35 | help='Elasticsearch timeout connection. Default is 30') 36 | @click.option('-di', '--delete-index', is_flag=True, default=False, 37 | help='Delete current index before import. Default: false') 38 | @click.option( 39 | '-r', '--region', metavar='', default='us-east-1', 40 | help='Change the region to run the program. Default is us-east-1') 41 | @click.option('-v', '--version', is_flag=True, default=False, help='Display version number and exit.') 42 | def cli_import(*args, **kwargs): 43 | """ 44 | Elasticsearch Import program 45 | :return: 46 | """ 47 | version = kwargs.pop('version') 48 | host = kwargs.pop('es_host') 49 | port = kwargs.pop('es_port') 50 | timeout = kwargs.pop('timeout') 51 | index = kwargs.pop('index') 52 | delete_index = kwargs.pop('delete_index') 53 | filename = kwargs.pop('file') 54 | # Import boto3 credentials 55 | session = boto3.Session() 56 | credentials = session.get_credentials() 57 | awsauth = None 58 | if credentials: 59 | region = kwargs.pop('region') 60 | awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, 'es', 61 | session_token=credentials.token) 62 | 63 | click.echo('Elasticsearch import utility version {0}'.format(require("es_tools")[0].version)) 64 | if version: 65 | return 66 | 67 | click.echo('[+] Connecting to ES: {host}:{port}'.format(host=host, port=port)) 68 | es = Elasticsearch([{'host': host, 'port': port}], timeout=timeout, http_auth=awsauth, 69 | connection_class=RequestsHttpConnection) 70 | 71 | documents = list(json.loads(doc) for doc in filename.readlines()) 72 | 73 | if delete_index: 74 | click.echo('[-] Deleting current index: {}'.format(index)) 75 | es.indices.delete(index, ignore=404) 76 | click.echo('[+] Creating {name} mapping'.format(name=INDEX_NAME)) 77 | es.indices.create(index, body={'mappings': documents[0]['mappings']}, ignore=400) 78 | # Remove mappings from list 79 | documents.pop(0) 80 | 81 | click.echo('[+] Creating {name} Documents'.format(name=INDEX_NAME)) 82 | for doc in documents: 83 | try: 84 | response = es.index(index=INDEX_NAME, doc_type=doc.pop('_type'), body=json.dumps(doc.pop('_source')), 85 | id=doc.pop('_id')) 86 | except Exception as e: 87 | click.echo(e.error) 88 | 89 | click.echo('[=] Finished processing') 90 | 91 | 92 | if __name__ == "__main__": 93 | cli_import() 94 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Amazon Software License 1.0 2 | 3 | This Amazon Software License ("License") governs your use, reproduction, and 4 | distribution of the accompanying software as specified below. 5 | 6 | 1. Definitions 7 | 8 | "Licensor" means any person or entity that distributes its Work. 9 | 10 | "Software" means the original work of authorship made available under this 11 | License. 12 | 13 | "Work" means the Software and any additions to or derivative works of the 14 | Software that are made available under this License. 15 | 16 | The terms "reproduce," "reproduction," "derivative works," and 17 | "distribution" have the meaning as provided under U.S. copyright law; 18 | provided, however, that for the purposes of this License, derivative works 19 | shall not include works that remain separable from, or merely link (or bind 20 | by name) to the interfaces of, the Work. 21 | 22 | Works, including the Software, are "made available" under this License by 23 | including in or with the Work either (a) a copyright notice referencing the 24 | applicability of this License to the Work, or (b) a copy of this License. 25 | 26 | 2. License Grants 27 | 28 | 2.1 Copyright Grant. Subject to the terms and conditions of this License, 29 | each Licensor grants to you a perpetual, worldwide, non-exclusive, 30 | royalty-free, copyright license to reproduce, prepare derivative works of, 31 | publicly display, publicly perform, sublicense and distribute its Work and 32 | any resulting derivative works in any form. 33 | 34 | 2.2 Patent Grant. Subject to the terms and conditions of this License, each 35 | Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free 36 | patent license to make, have made, use, sell, offer for sale, import, and 37 | otherwise transfer its Work, in whole or in part. The foregoing license 38 | applies only to the patent claims licensable by Licensor that would be 39 | infringed by Licensor's Work (or portion thereof) individually and 40 | excluding any combinations with any other materials or technology. 41 | 42 | 3. Limitations 43 | 44 | 3.1 Redistribution. You may reproduce or distribute the Work only if 45 | (a) you do so under this License, (b) you include a complete copy of this 46 | License with your distribution, and (c) you retain without modification 47 | any copyright, patent, trademark, or attribution notices that are present 48 | in the Work. 49 | 50 | 3.2 Derivative Works. You may specify that additional or different terms 51 | apply to the use, reproduction, and distribution of your derivative works 52 | of the Work ("Your Terms") only if (a) Your Terms provide that the use 53 | limitation in Section 3.3 applies to your derivative works, and (b) you 54 | identify the specific derivative works that are subject to Your Terms. 55 | Notwithstanding Your Terms, this License (including the redistribution 56 | requirements in Section 3.1) will continue to apply to the Work itself. 57 | 58 | 3.3 Use Limitation. The Work and any derivative works thereof only may be 59 | used or intended for use with the web services, computing platforms or 60 | applications provided by Amazon.com, Inc. or its affiliates, including 61 | Amazon Web Services, Inc. 62 | 63 | 3.4 Patent Claims. If you bring or threaten to bring a patent claim against 64 | any Licensor (including any claim, cross-claim or counterclaim in a 65 | lawsuit) to enforce any patents that you allege are infringed by any Work, 66 | then your rights under this License from such Licensor (including the 67 | grants in Sections 2.1 and 2.2) will terminate immediately. 68 | 69 | 3.5 Trademarks. This License does not grant any rights to use any 70 | Licensor's or its affiliates' names, logos, or trademarks, except as 71 | necessary to reproduce the notices described in this License. 72 | 73 | 3.6 Termination. If you violate any term of this License, then your rights 74 | under this License (including the grants in Sections 2.1 and 2.2) will 75 | terminate immediately. 76 | 77 | 4. Disclaimer of Warranty. 78 | 79 | THE WORK IS PROVIDED "AS IS" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 80 | EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF 81 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR 82 | NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER 83 | THIS LICENSE. SOME STATES' CONSUMER LAWS DO NOT ALLOW EXCLUSION OF AN 84 | IMPLIED WARRANTY, SO THIS DISCLAIMER MAY NOT APPLY TO YOU. 85 | 86 | 5. Limitation of Liability. 87 | 88 | EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL 89 | THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE 90 | SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, 91 | INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR 92 | RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK (INCLUDING 93 | BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, LOST PROFITS 94 | OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER COMM ERCIAL DAMAGES 95 | OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF 96 | SUCH DAMAGES. 97 | -------------------------------------------------------------------------------- /source/helper/helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | ##################################################################################################################### 4 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # 5 | # # 6 | # Licensed under the Amazon Software License (the "License"). You may not use this file except in compliance # 7 | # with the License. A copy of the License is located at # 8 | # # 9 | # http://aws.amazon.com/asl/ # 10 | # # 11 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # 12 | # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # 13 | # and limitations under the License. # 14 | ###################################################################################################################### 15 | 16 | import json 17 | import logging 18 | import uuid 19 | from botocore.vendored import requests 20 | from os import environ 21 | 22 | logging.getLogger().debug('Loading function') 23 | 24 | #====================================================================================================================== 25 | # Auxiliary Functions 26 | #====================================================================================================================== 27 | def send_response(event, context, responseStatus, responseData, resourceId, reason=None): 28 | logging.getLogger().debug("[send_response] Start") 29 | 30 | responseUrl = event['ResponseURL'] 31 | cw_logs_url = "https://console.aws.amazon.com/cloudwatch/home?region=%s#logEventViewer:group=%s;stream=%s"%(context.invoked_function_arn.split(':')[3], context.log_group_name, context.log_stream_name) 32 | 33 | logging.getLogger().info(responseUrl) 34 | responseBody = {} 35 | responseBody['Status'] = responseStatus 36 | responseBody['Reason'] = reason or ('See the details in CloudWatch Logs: ' + cw_logs_url) 37 | responseBody['PhysicalResourceId'] = resourceId 38 | responseBody['StackId'] = event['StackId'] 39 | responseBody['RequestId'] = event['RequestId'] 40 | responseBody['LogicalResourceId'] = event['LogicalResourceId'] 41 | responseBody['NoEcho'] = False 42 | responseBody['Data'] = responseData 43 | 44 | json_responseBody = json.dumps(responseBody) 45 | logging.getLogger().debug("Response body:\n" + json_responseBody) 46 | 47 | headers = { 48 | 'content-type' : '', 49 | 'content-length' : str(len(json_responseBody)) 50 | } 51 | 52 | try: 53 | response = requests.put(responseUrl, 54 | data=json_responseBody, 55 | headers=headers) 56 | logging.getLogger().debug("Status code: " + response.reason) 57 | 58 | except Exception as error: 59 | logging.getLogger().error("[send_response] Failed executing requests.put(..)") 60 | logging.getLogger().error(str(error)) 61 | 62 | logging.getLogger().debug("[send_response] End") 63 | 64 | #====================================================================================================================== 65 | # Lambda Entry Point 66 | #====================================================================================================================== 67 | def lambda_handler(event, context): 68 | responseStatus = 'SUCCESS' 69 | reason = None 70 | responseData = {} 71 | resourceId = event['PhysicalResourceId'] if 'PhysicalResourceId' in event else event['LogicalResourceId'] 72 | result = { 73 | 'StatusCode': '200', 74 | 'Body': {'message': 'success'} 75 | } 76 | 77 | try: 78 | #------------------------------------------------------------------ 79 | # Set Log Level 80 | #------------------------------------------------------------------ 81 | global log_level 82 | log_level = str(environ['LOG_LEVEL'].upper()) 83 | if log_level not in ['DEBUG', 'INFO','WARNING', 'ERROR','CRITICAL']: 84 | log_level = 'ERROR' 85 | logging.getLogger().setLevel(log_level) 86 | 87 | #---------------------------------------------------------- 88 | # Read inputs parameters 89 | #---------------------------------------------------------- 90 | logging.getLogger().info(event) 91 | request_type = event['RequestType'].upper() if ('RequestType' in event) else "" 92 | logging.getLogger().info(request_type) 93 | 94 | #---------------------------------------------------------- 95 | # Process event 96 | #---------------------------------------------------------- 97 | if event['ResourceType'] == "Custom::CreateUUID": 98 | if 'CREATE' in request_type: 99 | responseData['UUID'] = str(uuid.uuid4()) 100 | logging.getLogger().debug("UUID: %s"%responseData['UUID']) 101 | 102 | # UPDATE: do nothing 103 | # DELETE: do nothing 104 | 105 | except Exception as error: 106 | logging.getLogger().error(error) 107 | responseStatus = 'FAILED' 108 | reason = str(error) 109 | result = { 110 | 'statusCode': '400', 111 | 'body': {'message': reason} 112 | } 113 | 114 | finally: 115 | #------------------------------------------------------------------ 116 | # Send Result 117 | #------------------------------------------------------------------ 118 | if 'ResponseURL' in event: 119 | send_response(event, context, responseStatus, responseData, resourceId, reason) 120 | 121 | return json.dumps(result) 122 | -------------------------------------------------------------------------------- /source/es_tools/export.json: -------------------------------------------------------------------------------- 1 | {"mappings": {"config": {"properties": {"buildNum": {"type": "long"}, "defaultIndex": {"type": "string"}}}, "search": {"properties": {"sort": {"type": "string"}, "hits": {"type": "integer"}, "description": {"type": "string"}, "title": {"type": "string"}, "version": {"type": "integer"}, "kibanaSavedObjectMeta": {"properties": {"searchSourceJSON": {"type": "string"}}}, "columns": {"type": "string"}}}, "index-pattern": {"properties": {"timeFieldName": {"type": "string"}, "title": {"type": "string"}, "fields": {"type": "string"}, "fieldFormatMap": {"type": "string"}, "customFormats": {"type": "string"}, "intervalName": {"type": "string"}}}, "dashboard": {"properties": {"hits": {"type": "integer"}, "timeRestore": {"type": "boolean"}, "description": {"type": "string"}, "title": {"type": "string"}, "panelsJSON": {"type": "string"}, "version": {"type": "integer"}, "kibanaSavedObjectMeta": {"properties": {"searchSourceJSON": {"type": "string"}}}}}, "visualization": {"properties": {"visState": {"type": "string"}, "description": {"type": "string"}, "title": {"type": "string"}, "version": {"type": "integer"}, "savedSearchId": {"type": "string"}, "kibanaSavedObjectMeta": {"properties": {"searchSourceJSON": {"type": "string"}}}}}}} 2 | {"_type": "search", "_id": "Elasticity", "_source": {"sort": ["UsageStartDate", "desc"], "hits": 0, "description": "", "title": "Elasticity", "version": 1, "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\":\"billing-*\",\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[],\"query\":{\"query_string\":{\"query\":\"_type: elasticity\",\"analyze_wildcard\":true}}}"}, "columns": ["_source"]}} 3 | {"_type": "dashboard", "_id": "Consolidated-Account-Dashboard", "_source": {"hits": 0, "timeRestore": false, "description": "", "title": "Consolidated Account Dashboard", "panelsJSON": "[{\"col\":1,\"id\":\"Instructions\",\"row\":6,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"EC2-Instances-Running-per-Hour\",\"row\":1,\"size_x\":9,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"EC2-Hours-per-Dollar-Invested-Unblended-Cost\",\"row\":3,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Cost-by-TAG-Name-Unblended-Cost\",\"row\":3,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Cost-by-EC2-Instance-Types\",\"row\":3,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Total-Unblended-Cost\",\"row\":1,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":10,\"id\":\"EC2-Elasticity\",\"row\":3,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"}]", "version": 1, "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"NOT ProductName : \\\"AWS Premium Support (PS Charge)\\\" AND NOT ProductName : \\\"AWS Support (Business)\\\"\"}}}]}"}}} 4 | {"_type": "visualization", "_id": "EC2-Hours-per-Dollar-Invested", "_source": {"visState": "{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":false,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"EPU_Cost\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"UsageStartDate\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", "description": "", "title": "EC2 Hours per Dollar Invested", "version": 1, "savedSearchId": "EC2-per-USD", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"filter\":[]}"}}} 5 | {"_type": "visualization", "_id": "Cost-by-EC2-Instance-Types", "_source": {"visState": "{\n \"type\": \"pie\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"type\": \"sum\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"UnBlendedCost\"\n }\n },\n {\n \"id\": \"2\",\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"InstanceType\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}", "description": "", "title": "Cost by EC2 Instance Type (Unblended)", "version": 1, "savedSearchId": "EC2-Running-Instances", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\n \"filter\": []\n}"}}} 6 | {"_type": "index-pattern", "_id": "billing-*", "_source": {"fields": "[{\"name\":\"UsageStartDate\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"EPU_Cost\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"EPU_UnBlended\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"RateId\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"ProductName\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"aws.cloudformation:logical-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"Operation\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"ItemDescription\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"UsageType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"Cost\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"UnBlendedRate\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"SubscriptionId\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"RecordType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"PayerAccountId\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"user.Name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"UnBlendedCost\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"RecordId\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"InstanceType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"aws.cloudformation:stack-name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"aws.cloudformation:stack-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"BlendedCost\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"UsageItem\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"Rate\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"user.ENV\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"AvailabilityZone\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"UsageQuantity\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"InvoiceID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"LinkedAccountId\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"BlendedRate\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"PricingPlanId\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ReservedInstance\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ResourceId\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"UsageEndDate\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"Elasticity\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"SpotCoverage\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ReservedCoverage\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true}]", "fieldFormatMap": "{\"Cost\":{\"id\":\"number\",\"params\":{\"pattern\":\"$ 0,0.[00]\"}},\"UnBlendedCost\":{\"id\":\"number\",\"params\":{\"pattern\":\"$ 0,0.[00]\"}},\"BlendedCost\":{\"id\":\"number\",\"params\":{\"pattern\":\"$ 0,0.[00]\"}},\"Elasticity\":{\"id\":\"percent\",\"params\":{\"pattern\":\"0,0.[00]%\"}},\"SpotCoverage\":{\"id\":\"percent\",\"params\":{\"pattern\":\"0,0.[00]%\"}},\"ReservedCoverage\":{\"id\":\"percent\",\"params\":{\"pattern\":\"0,0.[00]%\"}}}", "timeFieldName": "UsageStartDate", "title": "billing-*"}} 7 | {"_type": "visualization", "_id": "Cost-by-Product", "_source": {"visState": "{\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"Cost\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"ProductName\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", "description": "", "title": "Cost by Product", "version": 1, "savedSearchId": "Billing", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"filter\":[]}"}}} 8 | {"_type": "visualization", "_id": "EC2-Instances-Running-per-Hour", "_source": {"visState": "{\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"UsageStartDate\",\"interval\":\"h\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"UsageItem\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", "description": "", "title": "EC2 Instances Running per Hour", "version": 1, "savedSearchId": "EC2-Running-Instances", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"filter\":[]}"}}} 9 | {"_type": "visualization", "_id": "EC2-Hours-per-Dollar-Invested-Unblended-Cost", "_source": {"visState": "{\n \"type\": \"line\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"showCircles\": false,\n \"smoothLines\": true,\n \"interpolate\": \"linear\",\n \"scale\": \"linear\",\n \"drawLinesBetweenPoints\": true,\n \"radiusRatio\": 9,\n \"times\": [],\n \"addTimeMarker\": false,\n \"defaultYExtents\": false,\n \"setYExtents\": false,\n \"yAxis\": {}\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"type\": \"avg\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"EPU_UnBlended\"\n }\n },\n {\n \"id\": \"2\",\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"UsageStartDate\",\n \"interval\": \"d\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n }\n ],\n \"listeners\": {}\n}", "description": "", "title": "EC2 Hours per Dollar Invested (Unblended)", "version": 1, "savedSearchId": "EC2-per-USD", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\n \"filter\": []\n}"}}} 10 | {"_type": "visualization", "_id": "Total-Unblended-Cost", "_source": {"visState": "{\n \"type\": \"metric\",\n \"params\": {\n \"fontSize\": \"32\"\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"type\": \"sum\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"UnBlendedCost\"\n }\n }\n ],\n \"listeners\": {}\n}", "description": "", "title": "Total Cost (Unblended)", "version": 1, "savedSearchId": "Billing", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\n \"filter\": []\n}"}}} 11 | {"_type": "config", "_id": "4.1.2-es-2.0", "_source": {"buildNum": 7562, "defaultIndex": "billing-*"}} 12 | {"_type": "search", "_id": "EC2-Running-Instances", "_source": {"sort": ["UsageStartDate", "desc"], "hits": 0, "description": "", "title": "EC2 Running Instances", "version": 1, "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\":\"billing-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"_type: billing\"}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[{\"meta\":{\"disabled\":false,\"index\":\"billing-*\",\"key\":\"ProductName\",\"negate\":false,\"value\":\"Amazon Elastic Compute Cloud\"},\"query\":{\"match\":{\"ProductName\":{\"query\":\"Amazon Elastic Compute Cloud\",\"type\":\"phrase\"}}}},{\"meta\":{\"disabled\":false,\"index\":\"billing-*\",\"key\":\"UsageItem\",\"negate\":true,\"value\":\"\"},\"query\":{\"match\":{\"UsageItem\":{\"query\":\"\",\"type\":\"phrase\"}}}}]}"}, "columns": ["ItemDescription", "Operation"]}} 13 | {"_type": "dashboard", "_id": "Single-Account-Dashboard", "_source": {"hits": 0, "timeRestore": false, "description": "", "title": "Single Account Dashboard", "panelsJSON": "[{\"col\":10,\"id\":\"Total-Cost\",\"row\":1,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Instructions\",\"row\":6,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"EC2-Instances-Running-per-Hour\",\"row\":1,\"size_x\":9,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Cost-by-EC2-Instance-Types-\",\"row\":3,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"EC2-Hours-per-Dollar-Invested\",\"row\":3,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Cost-by-TAG-Name\",\"row\":3,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"id\":\"EC2-Elasticity-\",\"type\":\"visualization\",\"size_x\":3,\"size_y\":3,\"col\":10,\"row\":3}]", "version": 1, "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"NOT ProductName : \\\"AWS Premium Support (PS Charge)\\\" AND NOT ProductName : \\\"AWS Support (Business)\\\"\"}}}]}"}}} 14 | {"_type": "visualization", "_id": "Cost-by-TAG-Name", "_source": {"visState": "{\n \"type\": \"pie\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"type\": \"sum\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"Cost\"\n }\n },\n {\n \"id\": \"2\",\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"user.Name\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\n \"index\": \"billing-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}"}, "version": 1, "description": "", "title": "Cost by Tag Key: Name"}} 15 | {"_type": "visualization", "_id": "Cost-by-EC2-Instance-Types-", "_source": {"visState": "{\n \"type\": \"pie\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"type\": \"sum\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"Cost\"\n }\n },\n {\n \"id\": \"2\",\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"InstanceType\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}", "description": "", "title": "Cost by EC2 Instance Type", "version": 1, "savedSearchId": "EC2-Running-Instances", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\n \"filter\": []\n}"}}} 16 | {"_type": "visualization", "_id": "Instructions", "_source": {"visState": "{\n \"aggs\": [],\n \"listeners\": {},\n \"params\": {\n \"markdown\": \"![AWS logo](https://a0.awsstatic.com/main/images/logos/aws_logo_179x109.gif) (SO0010) - Cost Optimization Monitor %%VERSION%%\\n----\\n\\n#### This tool will help you monitor the estimated costs of your AWS account and allow you to filter by products, tags, and services.\\n\\nSome information is precalculated and won't change when you filter the information by EC2 instance type, tag, etc.:\\n\\n* **Elasticity** = How much your instances grow and shrink. A higher elasticity percentage can indicate that you are using the benefits of the AWS Cloud to optimize costs as compared to an on-premises environment.\\n\\n* **RI Coverage** = How many Reserved Instances you are using from all EC2 instances executed. \\n> Reserved Instances can provide significant cost savings\\n\\n* **Spot Coverage** = How many Spot instances you are using in proportion to all executed instanced in the period.\\n> More Spot instances can significantly lower operational costs\\n\\n\\n#### Do you have any questions or comments?\\n[Send your Feedback](https://aws.amazon.com/forms/aws-doc-feedback?hidden_service_name=AWS%20Solutions&hidden_guide_name=Answers&hidden_api_version=&hidden_file_name=costoptimizatiomonitor%20kibana%20dashboard)\"\n },\n \"type\": \"markdown\"\n}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\n \"query\": {\n \"query_string\": {\n \"analyze_wildcard\": true,\n \"query\": \"*\"\n }\n },\n \"filter\": []\n}"}, "version": 1, "description": "", "title": "Instructions"}} 17 | {"_type": "config", "_id": "4.0.3", "_source": {"buildNum": 6103, "defaultIndex": "billing-*"}} 18 | {"_type": "search", "_id": "EC2-per-USD", "_source": {"sort": ["UsageStartDate", "desc"], "hits": 0, "description": "", "title": "EC2 per USD", "version": 1, "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\":\"billing-*\",\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[],\"query\":{\"query_string\":{\"query\":\"_type: ec2_per_usd\",\"analyze_wildcard\":true}}}"}, "columns": ["_source"]}} 19 | {"_type": "search", "_id": "Billing", "_source": {"sort": ["UsageStartDate", "desc"], "hits": 0, "description": "", "title": "Billing", "version": 1, "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"index\":\"billing-*\",\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[],\"query\":{\"query_string\":{\"query\":\"_type: billing\",\"analyze_wildcard\":true}}}"}, "columns": ["_source"]}} 20 | {"_type": "visualization", "_id": "Total-Cost", "_source": {"visState": "{\"type\":\"metric\",\"params\":{\"fontSize\":\"32\"},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"Cost\"}}],\"listeners\":{}}", "description": "", "title": "Total Cost", "version": 1, "savedSearchId": "Billing", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"filter\":[]}"}}} 21 | {"_type": "visualization", "_id": "Cost-by-TAG-Name-Unblended-Cost", "_source": {"visState": "{\n \"type\": \"pie\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"isDonut\": true\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"type\": \"sum\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"UnBlendedCost\"\n }\n },\n {\n \"id\": \"2\",\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"user.Name\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\n \"index\": \"billing-*\",\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n },\n \"filter\": []\n}"}, "version": 1, "description": "", "title": "Cost by Tag Key: Name (Unblended)"}} 22 | {"_type": "visualization", "_id": "EC2-Elasticity", "_source": {"visState": "{\"type\":\"metric\",\"params\":{\"fontSize\":\"24\"},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"Elasticity\"}},{\"id\":\"2\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"ReservedCoverage\"}},{\"id\":\"3\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"SpotCoverage\"}}],\"listeners\":{}}", "description": "", "title": "EC2 Elasticity", "version": 1, "savedSearchId": "Elasticity", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\"filter\":[]}"}}} 23 | {"_type": "visualization", "_id": "EC2-Elasticity-", "_source": {"visState": "{\n \"type\": \"metric\",\n \"params\": {\n \"fontSize\": \"24\"\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"type\": \"avg\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"Elasticity\"\n }\n },\n {\n \"id\": \"2\",\n \"type\": \"avg\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"ReservedCoverage\"\n }\n },\n {\n \"id\": \"3\",\n \"type\": \"avg\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"SpotCoverage\"\n }\n }\n ],\n \"listeners\": {}\n}", "description": "", "title": "EC2 Elasticity -", "version": 1, "savedSearchId": "Elasticity", "kibanaSavedObjectMeta": {"searchSourceJSON": "{\n \"filter\": []\n}"}}} 24 | -------------------------------------------------------------------------------- /deployment/cost-optimization-monitor.template: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "(SO0010) - Cost Optimization Monitor - AWS CloudFormation Template for AWS Solutions Builder Cost Optimization Tool - Monitor - **WARNING** This template creates AWS resources. You will be billed for the AWS resources used if you create a stack from this template.", 4 | "Parameters": { 5 | "KeyName": { 6 | "Description": "Existing Amazon EC2 key pair for SSH access to the instances to the Nginx proxy server", 7 | "Type": "AWS::EC2::KeyPair::KeyName", 8 | "ConstraintDescription": "must be the name of an existing EC2 KeyPair." 9 | }, 10 | "SSHLocation": { 11 | "Description": "IP address range that can access the Nginx proxy server", 12 | "Type": "String", 13 | "MinLength": "9", 14 | "MaxLength": "18", 15 | "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", 16 | "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x." 17 | }, 18 | "ElasticsearchDomainName": { 19 | "Type": "String", 20 | "Default": "comdomain", 21 | "Description": "Name for the Amazon ES domain that this template will create. Note: Domain names must start with a lowercase letter and must be between 3 and 28 characters. Valid characters are a-z (lowercase only), 0-9, and – (hyphen)", 22 | "AllowedPattern": "[a-z][a-z0-9\\-]+" 23 | }, 24 | "ClusterSize": { 25 | "Type": "String", 26 | "Default": "Small", 27 | "AllowedValues": [ 28 | "Small", 29 | "Medium", 30 | "Large" 31 | ], 32 | "Description": "Amazon ES cluster size: small, medium, large" 33 | }, 34 | "DBRBucketInput": { 35 | "Type": "String", 36 | "Description": "Select Yes if you want to use an existing S3 bucket to store detailed billing reports", 37 | "Default": "No", 38 | "AllowedValues": [ 39 | "Yes", 40 | "No" 41 | ] 42 | }, 43 | "DBRBucketName": { 44 | "Type": "String", 45 | "Description": "If you selected Yes for the previous parameter, type the name of existing S3 bucket name that you want to use to store billing reports. If you selected No, leave this field empty" 46 | }, 47 | "ProxyUsername": { 48 | "Type": "String", 49 | "Description": "User name for dashboard access via the proxy server" 50 | }, 51 | "ProxyPass": { 52 | "NoEcho": "true", 53 | "Description": "Password for dashboard access via the proxy server", 54 | "Type": "String", 55 | "MinLength": "6", 56 | "MaxLength": "41", 57 | "AllowedPattern": "^(?=.*[0-9])(?=.*[a-z])(?=.*[A-Z]).{6,}$", 58 | "ConstraintDescription": "Must contain at least 1 Upper/Lower alphanumeric characters and number (Mininum lenght is 6)" 59 | }, 60 | "VPCCidrparameter": { 61 | "Description": "CIDR block for VPC", 62 | "Type": "String", 63 | "MinLength": "9", 64 | "MaxLength": "18", 65 | "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", 66 | "Default": "10.250.0.0/16", 67 | "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x." 68 | }, 69 | "Subnet1Cidrparameter": { 70 | "Description": "IP address range for subnet created in AZ1", 71 | "Type": "String", 72 | "MinLength": "9", 73 | "MaxLength": "18", 74 | "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", 75 | "Default": "10.250.250.0/24", 76 | "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x." 77 | }, 78 | "Subnet2Cidrparameter": { 79 | "Description": "IP address range for subnet created in AZ2", 80 | "Type": "String", 81 | "MinLength": "9", 82 | "MaxLength": "18", 83 | "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", 84 | "Default": "10.250.251.0/24", 85 | "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x." 86 | }, 87 | "SendAnonymousData": { 88 | "Description": "Send anonymous data to AWS", 89 | "Type": "String", 90 | "Default": "Yes", 91 | "AllowedValues": [ 92 | "Yes", 93 | "No" 94 | ] 95 | } 96 | }, 97 | "Metadata": { 98 | "AWS::CloudFormation::Interface": { 99 | "ParameterGroups": [{ 100 | "Label": { 101 | "default": "Proxy Configuration" 102 | }, 103 | "Parameters": ["ProxyUsername", "ProxyPass", "SSHLocation", "KeyName"] 104 | }, 105 | { 106 | "Label": { 107 | "default": "Amazon ES Domain Configuration" 108 | }, 109 | "Parameters": [ 110 | "ElasticsearchDomainName", 111 | "ClusterSize" 112 | ] 113 | }, 114 | { 115 | "Label": { 116 | "default": "Amazon S3 Bucket Configuration" 117 | }, 118 | "Parameters": [ 119 | "DBRBucketInput", 120 | "DBRBucketName" 121 | ] 122 | }, 123 | { 124 | "Label": { 125 | "default": "Network Settings" 126 | }, 127 | "Parameters": ["VPCCidrparameter", "Subnet1Cidrparameter", "Subnet2Cidrparameter"] 128 | }, 129 | { 130 | "Label": { 131 | "default": "Send anonymous data to AWS" 132 | }, 133 | "Parameters": [ 134 | "SendAnonymousData" 135 | ] 136 | } 137 | 138 | ], 139 | "ParameterLabels": { 140 | "SendAnonymousData": { 141 | "default": "Send Anonymous Usage Data" 142 | }, 143 | "VPCCidrparameter": { 144 | "default": "VPC CIDR block" 145 | }, 146 | "Subnet1Cidrparameter": { 147 | "default": "1st Subnet Network" 148 | }, 149 | "Subnet2Cidrparameter": { 150 | "default": "2nd Subnet Network" 151 | }, 152 | "KeyName": { 153 | "default": "SSH Key" 154 | }, 155 | "SSHLocation": { 156 | "default": "Access CIDR Block" 157 | }, 158 | "DBRBucketInput": { 159 | "default": "Use existing bucket?" 160 | }, 161 | "DBRBucketName": { 162 | "default": "Existing S3 bucket name" 163 | }, 164 | "ProxyUsername": { 165 | "default": "User Name" 166 | }, 167 | "ProxyPass": { 168 | "default": "Password" 169 | }, 170 | "ElasticsearchDomainName": { 171 | "default": "Domain Name" 172 | }, 173 | "ClusterSize": { 174 | "default": "Cluster Size" 175 | } 176 | } 177 | 178 | } 179 | }, 180 | "Conditions": { 181 | "SendData": { 182 | "Fn::Equals": [{ 183 | "Ref": "SendAnonymousData" 184 | }, 185 | "Yes" 186 | ] 187 | }, 188 | "UseExistingBucket": { 189 | "Fn::Equals": [{ 190 | "Ref": "DBRBucketInput" 191 | }, 192 | "Yes" 193 | ] 194 | }, 195 | "UseExistingBucketNot": { 196 | "Fn::Equals": [{ 197 | "Ref": "DBRBucketInput" 198 | }, 199 | "No" 200 | ] 201 | } 202 | 203 | }, 204 | "Mappings": { 205 | "AmiMap": { 206 | "us-west-1": { 207 | "HVM64": "ami-0019ef04ac50be30f" 208 | }, 209 | "eu-central-1": { 210 | "HVM64": "ami-09def150731bdbcc2" 211 | }, 212 | "cn-north-1": { 213 | "HVM64": "ami-0cad3dea07a7c36f9" 214 | }, 215 | "us-east-1": { 216 | "HVM64": "ami-0de53d8956e8dcf80" 217 | }, 218 | "ap-northeast-2": { 219 | "HVM64": "ami-047f7b46bd6dd5d84" 220 | }, 221 | "us-gov-west-1": { 222 | "HVM64": "ami-6b157f0a" 223 | }, 224 | "sa-east-1": { 225 | "HVM64": "ami-0669a96e355eac82f" 226 | }, 227 | "ap-northeast-3": { 228 | "HVM64": "ami-088d713d672ed235e" 229 | }, 230 | "ap-northeast-1": { 231 | "HVM64": "ami-0f9ae750e8274075b" 232 | }, 233 | "ap-southeast-1": { 234 | "HVM64": "ami-0b419c3a4b01d1859" 235 | }, 236 | "us-east-2": { 237 | "HVM64": "ami-02bcbb802e03574ba" 238 | }, 239 | "ap-southeast-2": { 240 | "HVM64": "ami-04481c741a0311bbb" 241 | }, 242 | "cn-northwest-1": { 243 | "HVM64": "ami-094b7433620966eb5" 244 | }, 245 | "eu-west-1": { 246 | "HVM64": "ami-07683a44e80cd32c5" 247 | }, 248 | "eu-north-1": { 249 | "HVM64": "ami-d16fe6af" 250 | }, 251 | "us-gov-east-1": { 252 | "HVM64": "ami-1208ee63" 253 | }, 254 | "ap-south-1": { 255 | "HVM64": "ami-0889b8a448de4fc44" 256 | }, 257 | "eu-west-3": { 258 | "HVM64": "ami-0451ae4fd8dd178f7" 259 | }, 260 | "eu-west-2": { 261 | "HVM64": "ami-09ead922c1dad67e4" 262 | }, 263 | "ca-central-1": { 264 | "HVM64": "ami-03338e1f67dae0168" 265 | }, 266 | "us-west-2": { 267 | "HVM64": "ami-061392db613a6357b" 268 | } 269 | }, 270 | "ec2instanceSizing": { 271 | "elasticsearch": { 272 | "Small": "t2.large", 273 | "Medium": "m4.large", 274 | "Large": "m4.xlarge" 275 | } 276 | }, 277 | "esRegion2Type": { 278 | "us-west-1": { 279 | "type": "elasticsearchm4" 280 | }, 281 | "eu-central-1": { 282 | "type": "elasticsearchm4" 283 | }, 284 | "cn-north-1": { 285 | "type": "elasticsearchm4" 286 | }, 287 | "us-east-1": { 288 | "type": "elasticsearchm4" 289 | }, 290 | "ap-northeast-2": { 291 | "type": "elasticsearchm4" 292 | }, 293 | "us-gov-west-1": { 294 | "type": "elasticsearchm4" 295 | }, 296 | "sa-east-1": { 297 | "type": "elasticsearchm4" 298 | }, 299 | "ap-northeast-3": { 300 | "type": "elasticsearchm4" 301 | }, 302 | "ap-northeast-1": { 303 | "type": "elasticsearchm4" 304 | }, 305 | "ap-southeast-1": { 306 | "type": "elasticsearchm4" 307 | }, 308 | "us-east-2": { 309 | "type": "elasticsearchm4" 310 | }, 311 | "ap-southeast-2": { 312 | "type": "elasticsearchm4" 313 | }, 314 | "cn-northwest-1": { 315 | "type": "elasticsearchm4" 316 | }, 317 | "eu-west-1": { 318 | "type": "elasticsearchm4" 319 | }, 320 | "eu-north-1": { 321 | "type": "elasticsearchi3" 322 | }, 323 | "us-gov-east-1": { 324 | "type": "elasticsearchr4" 325 | }, 326 | "ap-south-1": { 327 | "type": "elasticsearchm4" 328 | }, 329 | "eu-west-3": { 330 | "type": "elasticsearchr4" 331 | }, 332 | "eu-west-2": { 333 | "type": "elasticsearchm4" 334 | }, 335 | "ca-central-1": { 336 | "type": "elasticsearchm4" 337 | }, 338 | "us-west-2": { 339 | "type": "elasticsearchm4" 340 | } 341 | }, 342 | "esinstanceSizing": { 343 | "elasticsearchm4": { 344 | "Small": "m4.large.elasticsearch", 345 | "Medium": "m4.large.elasticsearch", 346 | "Large": "m4.xlarge.elasticsearch" 347 | }, 348 | "elasticsearchr4": { 349 | "Small": "t2.small.elasticsearch", 350 | "Medium": "t2.small.elasticsearch", 351 | "Large": "r4.large.elasticsearch" 352 | }, 353 | "elasticsearchi3": { 354 | "Small": "i3.large.elasticsearch", 355 | "Medium": "i3.large.elasticsearch", 356 | "Large": "i3.large.elasticsearch" 357 | } 358 | }, 359 | "esmasterSizing": { 360 | "elasticsearchm4": { 361 | "Small": "t2.small.elasticsearch", 362 | "Medium": "t2.small.elasticsearch", 363 | "Large": "m4.large.elasticsearch" 364 | }, 365 | "elasticsearchr4": { 366 | "Small": "t2.small.elasticsearch", 367 | "Medium": "t2.small.elasticsearch", 368 | "Large": "r4.large.elasticsearch" 369 | }, 370 | "elasticsearchi3": { 371 | "Small": "i3.large.elasticsearch", 372 | "Medium": "i3.large.elasticsearch", 373 | "Large": "i3.large.elasticsearch" 374 | } 375 | }, 376 | "instanceCount": { 377 | "elasticsearch": { 378 | "Small": "2", 379 | "Medium": "4", 380 | "Large": "8" 381 | } 382 | }, 383 | "ELBLoggingAccountIDMap": { 384 | "us-east-1": { 385 | "AccoundID": "127311923021" 386 | }, 387 | "us-east-2": { 388 | "AccoundID": "033677994240" 389 | }, 390 | "us-west-1": { 391 | "AccoundID": "027434742980" 392 | }, 393 | "us-west-2": { 394 | "AccoundID": "797873946194" 395 | }, 396 | "ca-central-1": { 397 | "AccoundID": "985666609251" 398 | }, 399 | "eu-central-1": { 400 | "AccoundID": "054676820928" 401 | }, 402 | "eu-west-1": { 403 | "AccoundID": "156460612806" 404 | }, 405 | "eu-west-2": { 406 | "AccoundID": "652711504416" 407 | }, 408 | "eu-west-3": { 409 | "AccoundID": "009996457667" 410 | }, 411 | "eu-north-1": { 412 | "AccoundID": "897822967062" 413 | }, 414 | "ap-northeast-1": { 415 | "AccoundID": "582318560864" 416 | }, 417 | "ap-northeast-2": { 418 | "AccoundID": "600734575887" 419 | }, 420 | "ap-northeast-3": { 421 | "AccoundID": "383597477331" 422 | }, 423 | "ap-southeast-1": { 424 | "AccoundID": "114774131450" 425 | }, 426 | "ap-southeast-2": { 427 | "AccoundID": "783225319266" 428 | }, 429 | "ap-south-1": { 430 | "AccoundID": "718504428378" 431 | }, 432 | "sa-east-1": { 433 | "AccoundID": "507241528517" 434 | }, 435 | "us-gov-west-1": { 436 | "AccoundID": "048591011584" 437 | }, 438 | "us-gov-east-1": { 439 | "AccoundID": "190560391635" 440 | }, 441 | "cn-north-1": { 442 | "AccoundID": "638102146993" 443 | }, 444 | "cn-northwest-1": { 445 | "AccoundID": "037604701340" 446 | } 447 | } 448 | }, 449 | "Resources": { 450 | "VPC": { 451 | "Type": "AWS::EC2::VPC", 452 | "Properties": { 453 | "CidrBlock": { 454 | "Ref": "VPCCidrparameter" 455 | }, 456 | "Tags": [{ 457 | "Key": "Name", 458 | "Value": "Monitor VPC" 459 | }] 460 | } 461 | }, 462 | "VPCRouteTable": { 463 | "Type": "AWS::EC2::RouteTable", 464 | "Properties": { 465 | "VpcId": { 466 | "Ref": "VPC" 467 | }, 468 | "Tags": [{ 469 | "Key": "Network", 470 | "Value": "Public" 471 | }, 472 | { 473 | "Key": "Name", 474 | "Value": "Monitor VPC" 475 | } 476 | ] 477 | } 478 | }, 479 | "IGW": { 480 | "Type": "AWS::EC2::InternetGateway", 481 | "Properties": { 482 | "Tags": [{ 483 | "Key": "Name", 484 | "Value": "Monitor VPC IGW" 485 | }] 486 | } 487 | }, 488 | "IGWToInternet": { 489 | "Type": "AWS::EC2::VPCGatewayAttachment", 490 | "Properties": { 491 | "VpcId": { 492 | "Ref": "VPC" 493 | }, 494 | "InternetGatewayId": { 495 | "Ref": "IGW" 496 | } 497 | } 498 | }, 499 | "VPCPublicRoute": { 500 | "Type": "AWS::EC2::Route", 501 | "Properties": { 502 | "RouteTableId": { 503 | "Ref": "VPCRouteTable" 504 | }, 505 | "DestinationCidrBlock": "0.0.0.0/0", 506 | "GatewayId": { 507 | "Ref": "IGW" 508 | } 509 | } 510 | }, 511 | "VPCPubSub1": { 512 | "Type": "AWS::EC2::Subnet", 513 | "Properties": { 514 | "VpcId": { 515 | "Ref": "VPC" 516 | }, 517 | "CidrBlock": { 518 | "Ref": "Subnet1Cidrparameter" 519 | }, 520 | "AvailabilityZone": { 521 | "Fn::Select": [ 522 | "0", 523 | { 524 | "Fn::GetAZs": "" 525 | } 526 | ] 527 | }, 528 | "Tags": [{ 529 | "Key": "Network", 530 | "Value": "Public" 531 | }, 532 | { 533 | "Key": "Name", 534 | "Value": "Monitor VPC Subnet" 535 | } 536 | ] 537 | } 538 | }, 539 | "VPCPubSub2": { 540 | "Type": "AWS::EC2::Subnet", 541 | "Properties": { 542 | "VpcId": { 543 | "Ref": "VPC" 544 | }, 545 | "CidrBlock": { 546 | "Ref": "Subnet2Cidrparameter" 547 | }, 548 | "AvailabilityZone": { 549 | "Fn::Select": [ 550 | "1", 551 | { 552 | "Fn::GetAZs": "" 553 | } 554 | ] 555 | }, 556 | "Tags": [{ 557 | "Key": "Network", 558 | "Value": "Public" 559 | }, 560 | { 561 | "Key": "Name", 562 | "Value": "Monitor VPC Subnet" 563 | } 564 | ] 565 | } 566 | }, 567 | "VPCPubSubnet1RouteTableAssociation": { 568 | "Type": "AWS::EC2::SubnetRouteTableAssociation", 569 | "Properties": { 570 | "SubnetId": { 571 | "Ref": "VPCPubSub1" 572 | }, 573 | "RouteTableId": { 574 | "Ref": "VPCRouteTable" 575 | } 576 | } 577 | }, 578 | "VPCPubSubnet2RouteTableAssociation": { 579 | "Type": "AWS::EC2::SubnetRouteTableAssociation", 580 | "Properties": { 581 | "SubnetId": { 582 | "Ref": "VPCPubSub2" 583 | }, 584 | "RouteTableId": { 585 | "Ref": "VPCRouteTable" 586 | } 587 | } 588 | }, 589 | "S3DBRBucket": { 590 | "Type": "AWS::S3::Bucket", 591 | "Condition": "UseExistingBucketNot", 592 | "DeletionPolicy": "Retain", 593 | "Properties": { 594 | "AccessControl": "Private", 595 | "PublicAccessBlockConfiguration": { 596 | "BlockPublicAcls": true, 597 | "BlockPublicPolicy": true, 598 | "IgnorePublicAcls": true, 599 | "RestrictPublicBuckets": true 600 | } 601 | } 602 | }, 603 | "S3DBRBucketPolicy": { 604 | "Type": "AWS::S3::BucketPolicy", 605 | "Condition": "UseExistingBucketNot", 606 | "Properties": { 607 | "Bucket": { 608 | "Ref": "S3DBRBucket" 609 | }, 610 | "PolicyDocument": { 611 | "Version": "2012-10-17", 612 | "Statement": [{ 613 | "Effect": "Allow", 614 | "Action": [ 615 | "s3:GetBucketAcl", 616 | "s3:GetBucketPolicy" 617 | ], 618 | "Principal": { 619 | "AWS": "386209384616" 620 | }, 621 | "Resource": { 622 | "Fn::Sub": "arn:aws:s3:::${S3DBRBucket}" 623 | } 624 | }, 625 | { 626 | "Effect": "Allow", 627 | "Action": "s3:PutObject", 628 | "Principal": { 629 | "AWS": "386209384616" 630 | }, 631 | "Resource": { 632 | "Fn::Sub": "arn:aws:s3:::${S3DBRBucket}/*" 633 | } 634 | } 635 | ] 636 | } 637 | } 638 | }, 639 | "ELBSecurityGroup": { 640 | "Type": "AWS::EC2::SecurityGroup", 641 | "Properties": { 642 | "GroupDescription": "ELB - Port 80 access", 643 | "VpcId": { 644 | "Ref": "VPC" 645 | }, 646 | "SecurityGroupIngress": [{ 647 | "IpProtocol": "tcp", 648 | "FromPort": 80, 649 | "ToPort": 80, 650 | "CidrIp": { 651 | "Ref": "SSHLocation" 652 | } 653 | }], 654 | "SecurityGroupEgress": [{ 655 | "IpProtocol": "-1", 656 | "FromPort": -1, 657 | "ToPort": -1, 658 | "CidrIp": "0.0.0.0/0" 659 | }] 660 | } 661 | }, 662 | "S3LoggingBucket": { 663 | "Type": "AWS::S3::Bucket", 664 | "DeletionPolicy": "Retain", 665 | "Properties": { 666 | "AccessControl": "Private", 667 | "PublicAccessBlockConfiguration": { 668 | "BlockPublicAcls": true, 669 | "BlockPublicPolicy": true, 670 | "IgnorePublicAcls": true, 671 | "RestrictPublicBuckets": true 672 | } 673 | } 674 | }, 675 | "S3LoggingBucketPolicy": { 676 | "Type": "AWS::S3::BucketPolicy", 677 | "Properties": { 678 | "Bucket": { 679 | "Ref": "S3LoggingBucket" 680 | }, 681 | "PolicyDocument": { 682 | "Version": "2012-10-17", 683 | "Statement": [{ 684 | "Effect": "Allow", 685 | "Action": [ 686 | "s3:PutObject" 687 | ], 688 | "Resource": { 689 | "Fn::Sub": "arn:aws:s3:::${S3LoggingBucket}/Logs/AWSLogs/${AWS::AccountId}/*" 690 | }, 691 | "Principal": { 692 | "AWS": [{ 693 | "Fn::FindInMap": ["ELBLoggingAccountIDMap", { 694 | "Ref": "AWS::Region" 695 | }, "AccoundID"] 696 | }] 697 | } 698 | }] 699 | } 700 | } 701 | }, 702 | "ELB": { 703 | "Type": "AWS::ElasticLoadBalancing::LoadBalancer", 704 | "DependsOn": "S3LoggingBucketPolicy", 705 | "Properties": { 706 | "SecurityGroups": [{ 707 | "Fn::GetAtt": ["ELBSecurityGroup", "GroupId"] 708 | }], 709 | "Subnets": [{ 710 | "Ref": "VPCPubSub1" 711 | }, { 712 | "Ref": "VPCPubSub2" 713 | }], 714 | "CrossZone": true, 715 | "Instances": [{ 716 | "Ref": "ProxyServer1" 717 | }, { 718 | "Ref": "ProxyServer2" 719 | }], 720 | "LoadBalancerName": { 721 | "Ref": "AWS::StackName" 722 | }, 723 | "Listeners": [{ 724 | "LoadBalancerPort": "80", 725 | "InstancePort": "80", 726 | "Protocol": "HTTP" 727 | }], 728 | "HealthCheck": { 729 | "Target": "TCP:80", 730 | "HealthyThreshold": "3", 731 | "UnhealthyThreshold": "5", 732 | "Interval": "30", 733 | "Timeout": "5" 734 | }, 735 | "AccessLoggingPolicy": { 736 | "S3BucketName": { 737 | "Ref": "S3LoggingBucket" 738 | }, 739 | "S3BucketPrefix": "Logs", 740 | "Enabled": true, 741 | "EmitInterval": 60 742 | } 743 | } 744 | }, 745 | "ProxyServerRole": { 746 | "Type": "AWS::IAM::Role", 747 | "Properties": { 748 | "AssumeRolePolicyDocument": { 749 | "Version": "2012-10-17", 750 | "Statement": [{ 751 | "Effect": "Allow", 752 | "Principal": { 753 | "Service": [ 754 | "ec2.amazonaws.com" 755 | ] 756 | }, 757 | "Action": [ 758 | "sts:AssumeRole" 759 | ] 760 | }] 761 | }, 762 | "Path": "/", 763 | "Policies": [{ 764 | "PolicyName": "ProxyServerRole", 765 | "PolicyDocument": { 766 | "Version": "2012-10-17", 767 | "Statement": [{ 768 | "Effect": "Allow", 769 | "Action": [ 770 | "s3:GetObject" 771 | ], 772 | "Resource": { 773 | "Fn::Sub": [ 774 | "arn:aws:s3:::${BucketName}/*", 775 | { 776 | "BucketName": { 777 | "Fn::If": ["UseExistingBucket", { 778 | "Ref": "DBRBucketName" 779 | }, { 780 | "Ref": "S3DBRBucket" 781 | }] 782 | } 783 | } 784 | ] 785 | } 786 | }] 787 | } 788 | }] 789 | } 790 | }, 791 | "ProxyServerInstanceProfile": { 792 | "Type": "AWS::IAM::InstanceProfile", 793 | "Properties": { 794 | "Path": "/", 795 | "Roles": [{ 796 | "Ref": "ProxyServerRole" 797 | }] 798 | } 799 | }, 800 | "ProxyServerSecurityGroup": { 801 | "Type": "AWS::EC2::SecurityGroup", 802 | "Properties": { 803 | "GroupDescription": "Enable HTTP access via port 80 locked down to the load balancer + SSH access", 804 | "VpcId": { 805 | "Ref": "VPC" 806 | }, 807 | "SecurityGroupIngress": [{ 808 | "IpProtocol": "tcp", 809 | "FromPort": 80, 810 | "ToPort": 80, 811 | "CidrIp": { 812 | "Ref": "SSHLocation" 813 | } 814 | }, 815 | { 816 | "IpProtocol": "tcp", 817 | "FromPort": 22, 818 | "ToPort": 22, 819 | "CidrIp": { 820 | "Ref": "SSHLocation" 821 | } 822 | }, 823 | { 824 | "IpProtocol": "tcp", 825 | "FromPort": 80, 826 | "ToPort": 80, 827 | "SourceSecurityGroupId": { 828 | "Fn::GetAtt": ["ELBSecurityGroup", "GroupId"] 829 | } 830 | } 831 | ], 832 | "SecurityGroupEgress": [{ 833 | "IpProtocol": "-1", 834 | "FromPort": -1, 835 | "ToPort": -1, 836 | "CidrIp": "0.0.0.0/0" 837 | }] 838 | } 839 | }, 840 | "ProxyServer1EIP": { 841 | "DependsOn": "VPCPubSubnet1RouteTableAssociation", 842 | "Type": "AWS::EC2::EIP", 843 | "Properties": { 844 | "Domain": "vpc" 845 | } 846 | }, 847 | "ProxyServer1EIPAssoc": { 848 | "Type": "AWS::EC2::EIPAssociation", 849 | "Properties": { 850 | "AllocationId": { 851 | "Fn::GetAtt": ["ProxyServer1EIP", "AllocationId"] 852 | }, 853 | "InstanceId": { 854 | "Ref": "ProxyServer1" 855 | } 856 | } 857 | }, 858 | "ProxyServer1": { 859 | "Type": "AWS::EC2::Instance", 860 | "DependsOn": "VPCPubSubnet1RouteTableAssociation", 861 | "Metadata": { 862 | "AWS::CloudFormation::Init": { 863 | "configSets": { 864 | "Monitor_install": [ 865 | "install_cfn", 866 | "install_nginx", 867 | "configure_nginx", 868 | "install_awsdbrparser", 869 | "install_estool", 870 | "populate_es" 871 | ] 872 | }, 873 | "install_cfn": { 874 | "files": { 875 | "/etc/cfn/cfn-hup.conf": { 876 | "content": { 877 | "Fn::Join": [ 878 | "", 879 | [ 880 | "[main]\n", 881 | "stack=", 882 | { 883 | "Ref": "AWS::StackId" 884 | }, 885 | "\n", 886 | "region=", 887 | { 888 | "Ref": "AWS::Region" 889 | }, 890 | "\n" 891 | ] 892 | ] 893 | }, 894 | "mode": "000400", 895 | "owner": "root", 896 | "group": "root" 897 | }, 898 | "/etc/cfn/hooks.d/cfn-auto-reloader.conf": { 899 | "content": { 900 | "Fn::Join": [ 901 | "", 902 | [ 903 | "[cfn-auto-reloader-hook]\n", 904 | "triggers=post.update\n", 905 | "path=Resources.ProxyServer1.Metadata.AWS::CloudFormation::Init\n", 906 | "action=/opt/aws/bin/cfn-init -v ", 907 | " --stack ", 908 | { 909 | "Ref": "AWS::StackName" 910 | }, 911 | " --resource ProxyServer1 ", 912 | " --configsets Monitor_install ", 913 | " --region ", 914 | { 915 | "Ref": "AWS::Region" 916 | }, 917 | "\n" 918 | ] 919 | ] 920 | }, 921 | "mode": "000400", 922 | "owner": "root", 923 | "group": "root" 924 | } 925 | }, 926 | "services": { 927 | "sysvinit": { 928 | "cfn-hup": { 929 | "enabled": "true", 930 | "ensureRunning": "true", 931 | "files": [ 932 | "/etc/cfn/cfn-hup.conf", 933 | "/etc/cfn/hooks.d/cfn-auto-reloader.conf" 934 | ] 935 | } 936 | } 937 | } 938 | }, 939 | "install_nginx": { 940 | "packages": { 941 | "yum": { 942 | "git": [] 943 | } 944 | }, 945 | "commands": { 946 | "0-setup": { 947 | "command": "amazon-linux-extras install nginx1.12", 948 | "cwd": "/home/ec2-user" 949 | } 950 | }, 951 | "files": { 952 | "/tmp/crontab": { 953 | "content": { 954 | "Fn::Join": [ 955 | "", 956 | [ 957 | "#Content to include in the user crontab.\n", 958 | "#You can just create the file first and then copy to the crontab folder\n", 959 | "\n", 960 | "#Without this path the user wont find the dbrparser program\n", 961 | "PATH=/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/aws/bin:/home/dbrparser/.local/bin:/home/dbrparser/bin\n", 962 | "\n", 963 | "# m h dom mon dow usercommand\n", 964 | "00 23 * * * /home/dbrparser/dbr-job/job.sh >>/home/dbrparser/dbr-job/job.log\n", 965 | "30 23 1 * * /home/dbrparser/dbr-job/job.sh -pm >>/home/dbrparser/dbr-job/job.log\n" 966 | ] 967 | ] 968 | }, 969 | "mode": "000400", 970 | "owner": "root", 971 | "group": "root" 972 | }, 973 | "/tmp/setup.dbr": { 974 | "content": { 975 | "Fn::Join": [ 976 | "", 977 | [ 978 | "adduser dbrparser\n", 979 | "mkdir /home/dbrparser/dbr-job\n", 980 | "cp /tmp/crontab /home/dbrparser/crontab\n", 981 | "cp /home/ec2-user/aws-detailed-billing-parser-0.6.0/job.sh /home/dbrparser/dbr-job\n", 982 | "sed -i 's/bucket-123456/", 983 | { 984 | "Fn::If": [ 985 | "UseExistingBucket", 986 | { 987 | "Ref": "DBRBucketName" 988 | }, 989 | { 990 | "Fn::GetAtt": [ 991 | "S3DBRBucket", 992 | "DomainName" 993 | ] 994 | } 995 | ] 996 | }, 997 | "/g' /home/dbrparser/dbr-job/job.sh\n", 998 | "sed -i 's:.s3.amazonaws.com::g' /home/dbrparser/dbr-job/job.sh\n", 999 | "sed -i 's/123456789012/", 1000 | { 1001 | "Ref": "AWS::AccountId" 1002 | }, 1003 | "/g' /home/dbrparser/dbr-job/job.sh\n", 1004 | "sed -i 's#/mnt/jobs#/home/dbrparser/dbr-job#g' /home/dbrparser/dbr-job/job.sh\n", 1005 | "sed -i 's/elastic-search-host.endopoint.name/", 1006 | { 1007 | "Fn::GetAtt": [ 1008 | "MyElasticsearchDomain", 1009 | "DomainEndpoint" 1010 | ] 1011 | }, 1012 | "/g' /home/dbrparser/dbr-job/job.sh\n", 1013 | "chown -R dbrparser:dbrparser /home/dbrparser/dbr-job\n", 1014 | "chmod +x /home/dbrparser/dbr-job/job.sh\n", 1015 | "#try the initial import with awsauth option\n", 1016 | "# the initial import still doesn't work but leaving this code in for future reference\n", 1017 | "cp /home/dbrparser/dbr-job/job.sh /home/dbrparser/dbr-job/jobawsauth.sh\n", 1018 | "sed -i 's#--delete-index –bi#--delete-index –bi --awsauth#g' /home/dbrparser/dbr-job/jobawsauth.sh\n", 1019 | "crontab -u dbrparser /home/dbrparser/crontab\n", 1020 | "\n" 1021 | ] 1022 | ] 1023 | }, 1024 | "mode": "000700", 1025 | "owner": "root", 1026 | "group": "root" 1027 | }, 1028 | "/tmp/setup.nginx": { 1029 | "content": { 1030 | "Fn::Join": [ 1031 | "", 1032 | [ 1033 | "#!/bin/bash\n", 1034 | "yum update -y\n", 1035 | "sed -i 's/elastic-search-host.endopoint.name/", 1036 | { 1037 | "Fn::GetAtt": [ 1038 | "MyElasticsearchDomain", 1039 | "DomainEndpoint" 1040 | ] 1041 | }, 1042 | "/g' /etc/nginx/default.d/default.conf\n", 1043 | "nameserver=$(cat /etc/resolv.conf | grep -m1 nameserver | cut -d ' ' -f 2)\n", 1044 | "sed -i 's/resolver-ip/'$nameserver'/g' /etc/nginx/default.d/default.conf\n" 1045 | ] 1046 | ] 1047 | }, 1048 | "mode": "000700", 1049 | "owner": "root", 1050 | "group": "root" 1051 | }, 1052 | "/tmp/es.sh": { 1053 | "content": { 1054 | "Fn::Join": [ 1055 | "", 1056 | [ 1057 | "#!/bin/bash -v\n", 1058 | "\n", 1059 | "PATH=/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/aws/bin:/home/ec2-user/.local/bin:/home/ec2-user/bin\n", 1060 | "es-import -r ", 1061 | { 1062 | "Ref": "AWS::Region" 1063 | }, 1064 | " -e ", 1065 | { 1066 | "Fn::GetAtt": [ 1067 | "MyElasticsearchDomain", 1068 | "DomainEndpoint" 1069 | ] 1070 | }, 1071 | " -p 80 -f /tmp/export.json" 1072 | ] 1073 | ] 1074 | }, 1075 | "mode": "000700", 1076 | "owner": "root", 1077 | "group": "root" 1078 | }, 1079 | "/etc/nginx/default.d/default.conf": { 1080 | "content": { 1081 | "Fn::Join": [ 1082 | "", 1083 | [ 1084 | "location / {\n", 1085 | " resolver resolver-ip;\n", 1086 | " set $es ", 1087 | { 1088 | "Fn::GetAtt": [ 1089 | "MyElasticsearchDomain", 1090 | "DomainEndpoint" 1091 | ] 1092 | }, 1093 | ";\n", 1094 | " auth_basic 'Restricted';\n", 1095 | " auth_basic_user_file /etc/nginx/conf.d/kibana.htpasswd;\n", 1096 | " proxy_pass_request_headers off;\n", 1097 | " proxy_set_header Host $host;\n", 1098 | " proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n", 1099 | " proxy_pass http://$es;", 1100 | "}\n" 1101 | ] 1102 | ] 1103 | }, 1104 | "mode": "000644", 1105 | "owner": "root", 1106 | "group": "root" 1107 | } 1108 | }, 1109 | "services": { 1110 | "sysvinit": { 1111 | "nginx": { 1112 | "enabled": "true" 1113 | } 1114 | } 1115 | } 1116 | }, 1117 | "configure_nginx": { 1118 | "commands": { 1119 | "0-setup": { 1120 | "command": "/tmp/setup.nginx", 1121 | "cwd": "/home/ec2-user" 1122 | } 1123 | } 1124 | }, 1125 | "install_awsdbrparser": { 1126 | "commands": { 1127 | "0-pip": { 1128 | "command": "yum install python-pip -y", 1129 | "cwd": "/home/ec2-user" 1130 | }, 1131 | "1-wget_awsdbrparser": { 1132 | "command": "wget https://github.com/awslabs/aws-detailed-billing-parser/archive/0.6.0.tar.gz", 1133 | "cwd": "/home/ec2-user" 1134 | }, 1135 | "2-unzip_awsdbrparser": { 1136 | "command": "tar xvzf 0.6.0.tar.gz", 1137 | "cwd": "/home/ec2-user" 1138 | }, 1139 | "3-upgrade_urllib3_botocore": { 1140 | "command": "pip install --upgrade urllib3 botocore", 1141 | "cwd": "/home/ec2-user/aws-detailed-billing-parser-0.6.0" 1142 | }, 1143 | "4-refresh_awscli": { 1144 | "command": "pip install awscli", 1145 | "cwd": "/home/ec2-user/aws-detailed-billing-parser-0.6.0" 1146 | }, 1147 | "5-python_setup": { 1148 | "command": "python setup.py install", 1149 | "cwd": "/home/ec2-user/aws-detailed-billing-parser-0.6.0" 1150 | }, 1151 | "6-dbr_setup": { 1152 | "command": "/tmp/setup.dbr", 1153 | "cwd": "/home/ec2-user" 1154 | } 1155 | } 1156 | }, 1157 | "install_estool": { 1158 | "commands": { 1159 | "0-git_clone": { 1160 | "command": { 1161 | "Fn::Sub": "wget https://s3.amazonaws.com/%%TEMPLATE_BUCKET_NAME%%/cost-optimization-monitor/%%VERSION%%/es_tools-0.1.4.tar.gz" 1162 | }, 1163 | "cwd": "/home/ec2-user" 1164 | }, 1165 | "1-python_setup": { 1166 | "command": "pip install es_tools-0.1.4.tar.gz", 1167 | "cwd": "/home/ec2-user" 1168 | }, 1169 | "2-kibana_index": { 1170 | "command": { 1171 | "Fn::Sub": "wget https://s3.amazonaws.com/%%TEMPLATE_BUCKET_NAME%%/cost-optimization-monitor/%%VERSION%%/export.json" 1172 | }, 1173 | "cwd": "/tmp/" 1174 | }, 1175 | "3-import_dashboard": { 1176 | "command": "sh /tmp/es.sh" 1177 | } 1178 | } 1179 | }, 1180 | "populate_es": { 1181 | "commands": { 1182 | "0-run_dbrparser": { 1183 | "command": "su - dbrparser -c '/home/dbrparser/dbr-job/jobawsauth.sh >>/home/dbrparser/dbr-job/jobawsauth.log'", 1184 | "cwd": "/home/dbrparser/dbr-job" 1185 | } 1186 | } 1187 | } 1188 | } 1189 | }, 1190 | "Properties": { 1191 | "ImageId": { 1192 | "Fn::FindInMap": ["AmiMap", { 1193 | "Ref": "AWS::Region" 1194 | }, "HVM64"] 1195 | }, 1196 | "InstanceType": { 1197 | "Fn::FindInMap": [ 1198 | "ec2instanceSizing", 1199 | "elasticsearch", 1200 | { 1201 | "Ref": "ClusterSize" 1202 | } 1203 | ] 1204 | }, 1205 | "IamInstanceProfile": { 1206 | "Ref": "ProxyServerInstanceProfile" 1207 | }, 1208 | "NetworkInterfaces": [{ 1209 | "AssociatePublicIpAddress": true, 1210 | "DeviceIndex": "0", 1211 | "GroupSet": [{ 1212 | "Ref": "ProxyServerSecurityGroup" 1213 | }], 1214 | "SubnetId": { 1215 | "Ref": "VPCPubSub1" 1216 | } 1217 | }], 1218 | "KeyName": { 1219 | "Ref": "KeyName" 1220 | }, 1221 | "UserData": { 1222 | "Fn::Base64": { 1223 | "Fn::Join": [ 1224 | "", 1225 | [ 1226 | "#!/bin/bash -xe\n", 1227 | "yum update -y aws-cfn-bootstrap\n", 1228 | "/opt/aws/bin/cfn-init -v ", 1229 | " --stack ", 1230 | { 1231 | "Ref": "AWS::StackName" 1232 | }, 1233 | " --resource ProxyServer1 ", 1234 | " --configsets Monitor_install ", 1235 | " --region ", 1236 | { 1237 | "Ref": "AWS::Region" 1238 | }, 1239 | "\n", 1240 | "/opt/aws/bin/cfn-signal -e $? ", 1241 | " --stack ", 1242 | { 1243 | "Ref": "AWS::StackName" 1244 | }, 1245 | " --resource ProxyServer1 ", 1246 | " --region ", 1247 | { 1248 | "Ref": "AWS::Region" 1249 | }, 1250 | "\n", 1251 | "# Create a new username/password for nginx\n", 1252 | "printf ", 1253 | { 1254 | "Ref": "ProxyUsername" 1255 | }, 1256 | ":`openssl passwd -apr1 ", 1257 | { 1258 | "Ref": "ProxyPass" 1259 | }, 1260 | "` >> /etc/nginx/conf.d/kibana.htpasswd\n", 1261 | "# Remove the default location from nginx config\n", 1262 | "sed -ri '/location \\//,/.*\\}/d' /etc/nginx/nginx.conf\n", 1263 | "service nginx restart" 1264 | ] 1265 | ] 1266 | } 1267 | } 1268 | }, 1269 | "CreationPolicy": { 1270 | "ResourceSignal": { 1271 | "Timeout": "PT15M" 1272 | } 1273 | } 1274 | }, 1275 | "ProxyServer1Alarm": { 1276 | "Type": "AWS::CloudWatch::Alarm", 1277 | "Properties": { 1278 | "AlarmDescription": "Trigger a recovery when instance status check fails for 15 consecutive minutes.", 1279 | "Namespace": "AWS/EC2", 1280 | "MetricName": "StatusCheckFailed_System", 1281 | "Statistic": "Minimum", 1282 | "Period": 60, 1283 | "EvaluationPeriods": 15, 1284 | "ComparisonOperator": "GreaterThanThreshold", 1285 | "Threshold": 0, 1286 | "AlarmActions": [{ 1287 | "Fn::Join": ["", ["arn:aws:automate:", { 1288 | "Ref": "AWS::Region" 1289 | }, ":ec2:recover"]] 1290 | }], 1291 | "Dimensions": [{ 1292 | "Name": "InstanceId", 1293 | "Value": { 1294 | "Ref": "ProxyServer1" 1295 | } 1296 | }] 1297 | } 1298 | }, 1299 | "ProxyServer2EIP": { 1300 | "DependsOn": "VPCPubSubnet2RouteTableAssociation", 1301 | "Type": "AWS::EC2::EIP", 1302 | "Properties": { 1303 | "Domain": "vpc" 1304 | } 1305 | }, 1306 | "ProxyServer2EIPAssoc": { 1307 | "Type": "AWS::EC2::EIPAssociation", 1308 | "Properties": { 1309 | "AllocationId": { 1310 | "Fn::GetAtt": ["ProxyServer2EIP", "AllocationId"] 1311 | }, 1312 | "InstanceId": { 1313 | "Ref": "ProxyServer2" 1314 | } 1315 | } 1316 | }, 1317 | "ProxyServer2": { 1318 | "Type": "AWS::EC2::Instance", 1319 | "DependsOn": "VPCPubSubnet2RouteTableAssociation", 1320 | "Metadata": { 1321 | "AWS::CloudFormation::Init": { 1322 | "configSets": { 1323 | "Monitor_install": [ 1324 | "install_cfn", 1325 | "install_nginx", 1326 | "configure_nginx" 1327 | ] 1328 | }, 1329 | "install_cfn": { 1330 | "files": { 1331 | "/etc/cfn/cfn-hup.conf": { 1332 | "content": { 1333 | "Fn::Join": [ 1334 | "", 1335 | [ 1336 | "[main]\n", 1337 | "stack=", 1338 | { 1339 | "Ref": "AWS::StackId" 1340 | }, 1341 | "\n", 1342 | "region=", 1343 | { 1344 | "Ref": "AWS::Region" 1345 | }, 1346 | "\n" 1347 | ] 1348 | ] 1349 | }, 1350 | "mode": "000400", 1351 | "owner": "root", 1352 | "group": "root" 1353 | }, 1354 | "/etc/cfn/hooks.d/cfn-auto-reloader.conf": { 1355 | "content": { 1356 | "Fn::Join": [ 1357 | "", 1358 | [ 1359 | "[cfn-auto-reloader-hook]\n", 1360 | "triggers=post.update\n", 1361 | "path=Resources.ProxyServer2.Metadata.AWS::CloudFormation::Init\n", 1362 | "action=/opt/aws/bin/cfn-init -v ", 1363 | " --stack ", 1364 | { 1365 | "Ref": "AWS::StackName" 1366 | }, 1367 | " --resource ProxyServer2 ", 1368 | " --configsets Monitor_install ", 1369 | " --region ", 1370 | { 1371 | "Ref": "AWS::Region" 1372 | }, 1373 | "\n" 1374 | ] 1375 | ] 1376 | }, 1377 | "mode": "000400", 1378 | "owner": "root", 1379 | "group": "root" 1380 | } 1381 | }, 1382 | "services": { 1383 | "sysvinit": { 1384 | "cfn-hup": { 1385 | "enabled": "true", 1386 | "ensureRunning": "true", 1387 | "files": [ 1388 | "/etc/cfn/cfn-hup.conf", 1389 | "/etc/cfn/hooks.d/cfn-auto-reloader.conf" 1390 | ] 1391 | } 1392 | } 1393 | } 1394 | }, 1395 | "install_nginx": { 1396 | "packages": { 1397 | "yum": { 1398 | "git": [] 1399 | } 1400 | }, 1401 | "commands": { 1402 | "0-setup": { 1403 | "command": "amazon-linux-extras install nginx1.12", 1404 | "cwd": "/home/ec2-user" 1405 | } 1406 | }, 1407 | "files": { 1408 | "/tmp/setup.nginx": { 1409 | "content": { 1410 | "Fn::Join": [ 1411 | "", 1412 | [ 1413 | "sed -i 's/elastic-search-host.endopoint.name/", 1414 | { 1415 | "Fn::GetAtt": [ 1416 | "MyElasticsearchDomain", 1417 | "DomainEndpoint" 1418 | ] 1419 | }, 1420 | "/g' /etc/nginx/default.d/default.conf\n", 1421 | "nameserver=$(cat /etc/resolv.conf | grep -m1 nameserver | cut -d ' ' -f 2)\n", 1422 | "sed -i 's/resolver-ip/'$nameserver'/g' /etc/nginx/default.d/default.conf\n" 1423 | ] 1424 | ] 1425 | }, 1426 | "mode": "000700", 1427 | "owner": "root", 1428 | "group": "root" 1429 | }, 1430 | "/etc/nginx/default.d/default.conf": { 1431 | "content": { 1432 | "Fn::Join": [ 1433 | "", 1434 | [ 1435 | "location / {\n", 1436 | " resolver resolver-ip;\n", 1437 | " set $es ", 1438 | { 1439 | "Fn::GetAtt": [ 1440 | "MyElasticsearchDomain", 1441 | "DomainEndpoint" 1442 | ] 1443 | }, 1444 | ";\n", 1445 | " auth_basic 'Restricted';\n", 1446 | " auth_basic_user_file /etc/nginx/conf.d/kibana.htpasswd;\n", 1447 | " proxy_http_version 1.1;\n", 1448 | " proxy_set_header Authorization \"\";\n", 1449 | " proxy_set_header Upgrade $http_upgrade;\n", 1450 | " proxy_set_header Connection 'upgrade';\n", 1451 | " proxy_set_header Host $host;\n", 1452 | " proxy_cache_bypass $http_upgrade;\n", 1453 | " proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n", 1454 | " proxy_pass http://$es;", 1455 | "}\n" 1456 | ] 1457 | ] 1458 | }, 1459 | "mode": "000644", 1460 | "owner": "root", 1461 | "group": "root" 1462 | } 1463 | }, 1464 | "services": { 1465 | "sysvinit": { 1466 | "nginx": { 1467 | "enabled": "true" 1468 | } 1469 | } 1470 | } 1471 | }, 1472 | "configure_nginx": { 1473 | "commands": { 1474 | "0-setup": { 1475 | "command": "/tmp/setup.nginx", 1476 | "cwd": "/home/ec2-user" 1477 | } 1478 | } 1479 | } 1480 | } 1481 | }, 1482 | "Properties": { 1483 | "ImageId": { 1484 | "Fn::FindInMap": ["AmiMap", { 1485 | "Ref": "AWS::Region" 1486 | }, "HVM64"] 1487 | }, 1488 | "InstanceType": { 1489 | "Fn::FindInMap": [ 1490 | "ec2instanceSizing", 1491 | "elasticsearch", 1492 | { 1493 | "Ref": "ClusterSize" 1494 | } 1495 | ] 1496 | }, 1497 | "IamInstanceProfile": { 1498 | "Ref": "ProxyServerInstanceProfile" 1499 | }, 1500 | "NetworkInterfaces": [{ 1501 | "AssociatePublicIpAddress": true, 1502 | "DeviceIndex": "0", 1503 | "GroupSet": [{ 1504 | "Ref": "ProxyServerSecurityGroup" 1505 | }], 1506 | "SubnetId": { 1507 | "Ref": "VPCPubSub2" 1508 | } 1509 | }], 1510 | "KeyName": { 1511 | "Ref": "KeyName" 1512 | }, 1513 | "UserData": { 1514 | "Fn::Base64": { 1515 | "Fn::Join": [ 1516 | "", 1517 | [ 1518 | "#!/bin/bash -xe\n", 1519 | "yum update -y aws-cfn-bootstrap\n", 1520 | "/opt/aws/bin/cfn-init -v ", 1521 | " --stack ", 1522 | { 1523 | "Ref": "AWS::StackName" 1524 | }, 1525 | " --resource ProxyServer2 ", 1526 | " --configsets Monitor_install ", 1527 | " --region ", 1528 | { 1529 | "Ref": "AWS::Region" 1530 | }, 1531 | "\n", 1532 | "/opt/aws/bin/cfn-signal -e $? ", 1533 | " --stack ", 1534 | { 1535 | "Ref": "AWS::StackName" 1536 | }, 1537 | " --resource ProxyServer2 ", 1538 | " --region ", 1539 | { 1540 | "Ref": "AWS::Region" 1541 | }, 1542 | "\n", 1543 | "# Create a new username/password for nginx\n", 1544 | "printf ", 1545 | { 1546 | "Ref": "ProxyUsername" 1547 | }, 1548 | ":`openssl passwd -apr1 ", 1549 | { 1550 | "Ref": "ProxyPass" 1551 | }, 1552 | "` >> /etc/nginx/conf.d/kibana.htpasswd\n", 1553 | "# Remove the default location from nginx config\n", 1554 | "sed -ri '/location \\//,/.*\\}/d' /etc/nginx/nginx.conf\n", 1555 | "service nginx restart" 1556 | ] 1557 | ] 1558 | } 1559 | } 1560 | }, 1561 | "CreationPolicy": { 1562 | "ResourceSignal": { 1563 | "Timeout": "PT15M" 1564 | } 1565 | } 1566 | }, 1567 | "ProxyServer2Alarm": { 1568 | "Type": "AWS::CloudWatch::Alarm", 1569 | "Properties": { 1570 | "AlarmDescription": "Trigger a recovery when instance status check fails for 15 consecutive minutes.", 1571 | "Namespace": "AWS/EC2", 1572 | "MetricName": "StatusCheckFailed_System", 1573 | "Statistic": "Minimum", 1574 | "Period": 60, 1575 | "EvaluationPeriods": 15, 1576 | "ComparisonOperator": "GreaterThanThreshold", 1577 | "Threshold": 0, 1578 | "AlarmActions": [{ 1579 | "Fn::Join": ["", ["arn:aws:automate:", { 1580 | "Ref": "AWS::Region" 1581 | }, ":ec2:recover"]] 1582 | }], 1583 | "Dimensions": [{ 1584 | "Name": "InstanceId", 1585 | "Value": { 1586 | "Ref": "ProxyServer2" 1587 | } 1588 | }] 1589 | } 1590 | }, 1591 | "MyElasticsearchDomain": { 1592 | "Type": "AWS::Elasticsearch::Domain", 1593 | "Properties": { 1594 | "DomainName": { 1595 | "Ref": "ElasticsearchDomainName" 1596 | }, 1597 | "ElasticsearchVersion": "2.3", 1598 | "EBSOptions": { 1599 | "EBSEnabled": true, 1600 | "VolumeType": "gp2", 1601 | "VolumeSize": 20 1602 | }, 1603 | "AccessPolicies": { 1604 | "Version": "2012-10-17", 1605 | "Statement": [{ 1606 | "Sid": "IP restriction", 1607 | "Action": "es:*", 1608 | "Principal": { 1609 | "AWS": "*" 1610 | }, 1611 | "Effect": "Allow", 1612 | "Resource": { 1613 | "Fn::Join": [ 1614 | "", 1615 | [ 1616 | "arn:aws:es:", 1617 | { 1618 | "Ref": "AWS::Region" 1619 | }, 1620 | ":", 1621 | { 1622 | "Ref": "AWS::AccountId" 1623 | }, 1624 | ":domain/", 1625 | { 1626 | "Ref": "ElasticsearchDomainName" 1627 | }, 1628 | "/*" 1629 | ] 1630 | ] 1631 | }, 1632 | "Condition": { 1633 | "IpAddress": { 1634 | "aws:SourceIp": [{ 1635 | "Ref": "ProxyServer1EIP" 1636 | }, { 1637 | "Ref": "ProxyServer2EIP" 1638 | }] 1639 | } 1640 | } 1641 | }, 1642 | { 1643 | "Sid": "EC2-Role restriction", 1644 | "Action": "es:*", 1645 | "Principal": { 1646 | "AWS": { 1647 | "Fn::Join": [ 1648 | "", 1649 | [{ 1650 | "Fn::GetAtt": [ 1651 | "ProxyServerRole", 1652 | "Arn" 1653 | ] 1654 | }] 1655 | ] 1656 | } 1657 | }, 1658 | "Effect": "Allow", 1659 | "Resource": { 1660 | "Fn::Join": [ 1661 | "", 1662 | [ 1663 | "arn:aws:es:", 1664 | { 1665 | "Ref": "AWS::Region" 1666 | }, 1667 | ":", 1668 | { 1669 | "Ref": "AWS::AccountId" 1670 | }, 1671 | ":domain/", 1672 | { 1673 | "Ref": "ElasticsearchDomainName" 1674 | }, 1675 | "/*" 1676 | ] 1677 | ] 1678 | } 1679 | } 1680 | ] 1681 | }, 1682 | "AdvancedOptions": { 1683 | "rest.action.multi.allow_explicit_index": "true" 1684 | }, 1685 | "ElasticsearchClusterConfig": { 1686 | "ZoneAwarenessEnabled": true, 1687 | "DedicatedMasterCount": 3, 1688 | "DedicatedMasterEnabled": true, 1689 | "DedicatedMasterType": { 1690 | "Fn::FindInMap": [ 1691 | "esmasterSizing", 1692 | { 1693 | "Fn::FindInMap": [ 1694 | "esRegion2Type", 1695 | { 1696 | "Ref": "AWS::Region" 1697 | }, 1698 | "type" 1699 | ] 1700 | }, 1701 | { 1702 | "Ref": "ClusterSize" 1703 | } 1704 | ] 1705 | }, 1706 | "InstanceCount": { 1707 | "Fn::FindInMap": [ 1708 | "instanceCount", 1709 | "elasticsearch", 1710 | { 1711 | "Ref": "ClusterSize" 1712 | } 1713 | ] 1714 | }, 1715 | "InstanceType": { 1716 | "Fn::FindInMap": [ 1717 | "esinstanceSizing", 1718 | { 1719 | "Fn::FindInMap": [ 1720 | "esRegion2Type", 1721 | { 1722 | "Ref": "AWS::Region" 1723 | }, 1724 | "type" 1725 | ] 1726 | }, 1727 | { 1728 | "Ref": "ClusterSize" 1729 | } 1730 | ] 1731 | } 1732 | } 1733 | } 1734 | }, 1735 | "SolutionHelperRole": { 1736 | "Type": "AWS::IAM::Role", 1737 | "Properties": { 1738 | "AssumeRolePolicyDocument": { 1739 | "Version": "2012-10-17", 1740 | "Statement": [{ 1741 | "Effect": "Allow", 1742 | "Principal": { 1743 | "Service": "lambda.amazonaws.com" 1744 | }, 1745 | "Action": "sts:AssumeRole" 1746 | }] 1747 | }, 1748 | "Path": "/", 1749 | "Policies": [{ 1750 | "PolicyName": "Solution_Helper_Permissions", 1751 | "PolicyDocument": { 1752 | "Version": "2012-10-17", 1753 | "Statement": [{ 1754 | "Effect": "Allow", 1755 | "Action": [ 1756 | "logs:CreateLogGroup", 1757 | "logs:CreateLogStream", 1758 | "logs:PutLogEvents" 1759 | ], 1760 | "Resource": { 1761 | "Fn::Sub": "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*" 1762 | } 1763 | }] 1764 | } 1765 | }] 1766 | } 1767 | }, 1768 | "SolutionHelper": { 1769 | "Type": "AWS::Lambda::Function", 1770 | "Properties": { 1771 | "Handler": "helper.lambda_handler", 1772 | "Role": { 1773 | "Fn::GetAtt": ["SolutionHelperRole", "Arn"] 1774 | }, 1775 | "Description": "This function creates a CloudFormation custom lambda resource that creates custom lambda functions by finding and replacing specific values from existing lambda function code.", 1776 | "Code": { 1777 | "S3Bucket": { 1778 | "Fn::Join": [ 1779 | "", 1780 | [ 1781 | "%%DIST_BUCKET_NAME%%-", 1782 | { 1783 | "Ref": "AWS::Region" 1784 | } 1785 | ] 1786 | ] 1787 | }, 1788 | "S3Key": "cost-optimization-monitor/%%VERSION%%/helper.zip" 1789 | }, 1790 | "Environment": { 1791 | "Variables": { 1792 | "LOG_LEVEL": "INFO" 1793 | } 1794 | }, 1795 | "Runtime": "python3.7", 1796 | "Timeout": 300 1797 | } 1798 | }, 1799 | "CreateUniqueID": { 1800 | "Type": "Custom::CreateUUID", 1801 | "Properties": { 1802 | "ServiceToken": { 1803 | "Fn::GetAtt": ["SolutionHelper", "Arn"] 1804 | } 1805 | } 1806 | }, 1807 | "SendingAnonymousData": { 1808 | "Type": "Custom::LoadLambda", 1809 | "Condition": "SendData", 1810 | "Properties": { 1811 | "ServiceToken": { 1812 | "Fn::GetAtt": ["SolutionHelper", "Arn"] 1813 | }, 1814 | "SendAnonymousData": { 1815 | "Fn::Join": [ 1816 | "", 1817 | [ 1818 | "{ 'Solution' : '", 1819 | "SO00010", 1820 | "', ", 1821 | "'UUID' : '", 1822 | { 1823 | "Fn::GetAtt": [ 1824 | "CreateUniqueID", 1825 | "UUID" 1826 | ] 1827 | }, 1828 | "', ", 1829 | "'Data': {", 1830 | "'ClusterSize': '", 1831 | { 1832 | "Ref": "ClusterSize" 1833 | }, 1834 | "'", 1835 | "}", 1836 | "}" 1837 | ] 1838 | ] 1839 | } 1840 | } 1841 | } 1842 | }, 1843 | "Outputs": { 1844 | "SingleDashboardURL": { 1845 | "Value": { 1846 | "Fn::Join": [ 1847 | "", 1848 | [ 1849 | "http://", { 1850 | "Fn::GetAtt": ["ELB", "DNSName"] 1851 | }, 1852 | "/_plugin/kibana/#/dashboard/Single-Account-Dashboard?_g=%28refreshInterval:%28display:Off,pause:!f,section:0,value:0%29,time:%28from:now%2FM,mode:quick,to:now%2FM%29%29" 1853 | ] 1854 | ] 1855 | }, 1856 | "Description": "Single Account Dashboard" 1857 | }, 1858 | "ConsolidatedDashboardURL": { 1859 | "Value": { 1860 | "Fn::Join": [ 1861 | "", 1862 | [ 1863 | "http://", { 1864 | "Fn::GetAtt": ["ELB", "DNSName"] 1865 | }, 1866 | "/_plugin/kibana/#/dashboard/Consolidated-Account-Dashboard?_g=%28refreshInterval:%28display:Off,pause:!f,section:0,value:0%29,time:%28from:now%2FM,mode:quick,to:now%2FM%29%29" 1867 | ] 1868 | ] 1869 | }, 1870 | "Description": "Consolidated Account Dashboard" 1871 | }, 1872 | "BucketName": { 1873 | "Description": "Bucket for storing Detailed Billing Records", 1874 | "Value": { 1875 | "Fn::If": [ 1876 | "UseExistingBucket", 1877 | { 1878 | "Ref": "DBRBucketName" 1879 | }, 1880 | { 1881 | "Ref": "S3DBRBucket" 1882 | } 1883 | ] 1884 | } 1885 | 1886 | }, 1887 | "UUID": { 1888 | "Description": "Newly created random anonymous UUID.", 1889 | "Value": { 1890 | "Fn::GetAtt": [ 1891 | "CreateUniqueID", 1892 | "UUID" 1893 | ] 1894 | } 1895 | } 1896 | } 1897 | } 1898 | --------------------------------------------------------------------------------