├── meta └── main.yml ├── README.md └── library └── ec2_elasticsearch.py /meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | author: Jose Armesto 4 | description: Ansible modules for working with Amazon ElasticSearch Clusters 5 | license: MIT 6 | min_ansible_version: 2.0 7 | platforms: 8 | - name: Amazon 9 | versions: 10 | - all 11 | categories: 12 | - cloud 13 | - cloud:ec2 14 | - development 15 | - packaging 16 | - web 17 | dependencies: [] 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ansible AWS ElasticSearch module 2 | 3 | For configuring/managing aws managed elasticsearch clusters 4 | 5 | --- 6 | 7 | - hosts: localhost 8 | tasks: 9 | - name: "Create ElasticSearch cluster" 10 | ec2_elasticsearch: 11 | name: "my-cluster" 12 | elasticsearch_version: "2.3" 13 | region: "us-west-1" 14 | instance_type: "m3.medium.elasticsearch" 15 | instance_count: 2 16 | dedicated_master: True 17 | zone_awareness: True 18 | dedicated_master_instance_type: "t2.micro.elasticsearch" 19 | dedicated_master_instance_count: 2 20 | ebs: True 21 | volume_type: "standard" 22 | volume_size: 10 23 | vpc_subnets: "subnet-e537d64a" 24 | vpc_security_groups: "sg-dd2f13cb" 25 | snapshot_hour: 13 26 | access_policies: "{{ lookup('file', 'cluster_policies.json') | from_json }}" 27 | profile: "myawsaccount" 28 | register: response 29 | 30 | ## VPC Configuration 31 | 32 | ### Endpoints 33 | 34 | Non VPC clusters give endpoints at `DomainStatus.Endpoint`, however VPC clusters return it at `DomainStatus.Endpoints.vpc` 35 | 36 | ### Service Roles 37 | 38 | AWS currently provides no way to create the correct service role for a vpc limited elasticsearch cluster. To 39 | get the correct role configured in your account, create a test cluster in your vpc using the aws console. You can 40 | delete it afterwards. If you don't have this, the module will fail with this error: 41 | 42 | > Before you can proceed, you must enable a service-linked role to give Amazon ES permissions to access your VPC. 43 | 44 | ## Pitfalls 45 | 46 | Access Policies may trigger continual updates if the format in `cluster_policies.json` differs from the AWS 47 | returned value. To debug this, you can use the cli to get the policy back so you can compare or replace the contents 48 | of your `cluster_policies.json` file. 49 | 50 | `aws es describe-elasticsearch-domain --domain-name my-cluster --query DomainStatus.AccessPolicies --output text` 51 | -------------------------------------------------------------------------------- /library/ec2_elasticsearch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # encoding: utf-8 3 | 4 | # (c) 2015, Jose Armesto 5 | # 6 | # This file is part of Ansible 7 | # 8 | # This module is free software: you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by 10 | # the Free Software Foundation, either version 3 of the License, or 11 | # (at your option) any later version. 12 | # 13 | # This software is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License 19 | # along with this software. If not, see . 20 | 21 | DOCUMENTATION = """ 22 | --- 23 | module: ec2_elasticsearch 24 | short_description: Creates ElasticSearch cluster on Amazon 25 | description: 26 | - It depends on boto3 27 | 28 | version_added: "2.1" 29 | author: "Jose Armesto (@fiunchinho)" 30 | options: 31 | name: 32 | description: 33 | - Cluster name to be used. 34 | required: true 35 | elasticsearch_version: 36 | description: 37 | - Elasticsearch version to deploy. Default is '2.3'. 38 | required: false 39 | region: 40 | description: 41 | - The AWS region to use. 42 | required: true 43 | aliases: ['aws_region', 'ec2_region'] 44 | instance_type: 45 | description: 46 | - Type of the instances to use for the cluster. Valid types are: 'm3.medium.elasticsearch'|'m3.large.elasticsearch'|'m3.xlarge.elasticsearch'|'m3.2xlarge.elasticsearch'|'t2.micro.elasticsearch'|'t2.small.elasticsearch'|'t2.medium.elasticsearch'|'r3.large.elasticsearch'|'r3.xlarge.elasticsearch'|'r3.2xlarge.elasticsearch'|'r3.4xlarge.elasticsearch'|'r3.8xlarge.elasticsearch'|'i2.xlarge.elasticsearch'|'i2.2xlarge.elasticsearch' 47 | required: true 48 | instance_count: 49 | description: 50 | - Number of instances for the cluster. 51 | required: true 52 | dedicated_master: 53 | description: 54 | - A boolean value to indicate whether a dedicated master node is enabled. 55 | required: true 56 | zone_awareness: 57 | description: 58 | - A boolean value to indicate whether zone awareness is enabled. 59 | required: true 60 | ebs: 61 | description: 62 | - Specifies whether EBS-based storage is enabled. 63 | required: true 64 | dedicated_master_instance_type: 65 | description: 66 | - The instance type for a dedicated master node. 67 | required: false 68 | dedicated_master_instance_count: 69 | description: 70 | - Total number of dedicated master nodes, active and on standby, for the cluster. 71 | required: false 72 | volume_type: 73 | description: 74 | - Specifies the volume type for EBS-based storage. 75 | required: true 76 | volume_size: 77 | description: 78 | - Integer to specify the size of an EBS volume. 79 | required: true 80 | vpc_subnets: 81 | description: 82 | - Specifies the subnet ids for VPC endpoint. 83 | required: false 84 | vpc_security_groups: 85 | description: 86 | - Specifies the security group ids for VPC endpoint. 87 | required: false 88 | snapshot_hour: 89 | description: 90 | - Integer value from 0 to 23 specifying when the service takes a daily automated snapshot of the specified Elasticsearch domain. 91 | required: true 92 | access_policies: 93 | description: 94 | - IAM access policy as a JSON-formatted string. 95 | required: true 96 | profile: 97 | description: 98 | - What Boto profile use to connect to AWS. 99 | required: false 100 | encryption_at_rest_enabled: 101 | description: 102 | - Should data be encrypted while at rest. 103 | required: false 104 | encryption_at_rest_kms_key_id: 105 | description: 106 | - If encryption_at_rest_enabled is True, this identifies the encryption key to use 107 | required: false 108 | 109 | requirements: 110 | - "python >= 2.6" 111 | - boto3 112 | """ 113 | 114 | EXAMPLES = ''' 115 | 116 | - ec2_elasticsearch: 117 | name: "my-cluster" 118 | elasticsearch_version: "2.3" 119 | region: "eu-west-1" 120 | instance_type: "m3.medium.elasticsearch" 121 | instance_count: 2 122 | dedicated_master: True 123 | zone_awareness: True 124 | dedicated_master_instance_type: "t2.micro.elasticsearch" 125 | dedicated_master_instance_count: 2 126 | ebs: True 127 | volume_type: "standard" 128 | volume_size: 10 129 | vpc_subnets: "subnet-e537d64a" 130 | vpc_security_groups: "sg-dd2f13cb" 131 | snapshot_hour: 13 132 | access_policies: "{{ lookup('file', 'files/cluster_policies.json') | from_json }}" 133 | profile: "myawsaccount" 134 | ''' 135 | try: 136 | import botocore 137 | import boto3 138 | import json 139 | 140 | HAS_BOTO=True 141 | except ImportError: 142 | HAS_BOTO=False 143 | 144 | def main(): 145 | argument_spec = ec2_argument_spec() 146 | argument_spec.update(dict( 147 | name = dict(required=True), 148 | instance_type = dict(required=True), 149 | instance_count = dict(required=True, type='int'), 150 | dedicated_master = dict(required=True, type='bool'), 151 | zone_awareness = dict(required=True, type='bool'), 152 | dedicated_master_instance_type = dict(), 153 | dedicated_master_instance_count = dict(type='int'), 154 | ebs = dict(required=True, type='bool'), 155 | volume_type = dict(required=True), 156 | volume_size = dict(required=True, type='int'), 157 | access_policies = dict(required=True, type='dict'), 158 | vpc_subnets = dict(required=False), 159 | vpc_security_groups = dict(required=False), 160 | snapshot_hour = dict(required=True, type='int'), 161 | elasticsearch_version = dict(default='2.3'), 162 | encryption_at_rest_enabled = dict(default=False), 163 | encryption_at_rest_kms_key_id = dict(required=False), 164 | )) 165 | 166 | module = AnsibleModule( 167 | argument_spec=argument_spec, 168 | ) 169 | 170 | if not HAS_BOTO: 171 | module.fail_json(msg='boto3 required for this module, install via pip or your package manager') 172 | 173 | region, ec2_url, aws_connect_params = get_aws_connection_info(module, True) 174 | client = boto3_conn(module=module, conn_type='client', resource='es', region=region, **aws_connect_params) 175 | 176 | cluster_config = { 177 | 'InstanceType': module.params.get('instance_type'), 178 | 'InstanceCount': int(module.params.get('instance_count')), 179 | 'DedicatedMasterEnabled': module.params.get('dedicated_master'), 180 | 'ZoneAwarenessEnabled': module.params.get('zone_awareness') 181 | } 182 | 183 | ebs_options = { 184 | 'EBSEnabled': module.params.get('ebs') 185 | } 186 | 187 | encryption_at_rest_enabled = module.params.get('encryption_at_rest_enabled') == 'True' 188 | encryption_at_rest_options = { 189 | 'Enabled': encryption_at_rest_enabled 190 | } 191 | 192 | if encryption_at_rest_enabled: 193 | encryption_at_rest_options['KmsKeyId'] = module.params.get('encryption_at_rest_kms_key_id') 194 | 195 | vpc_options = {} 196 | 197 | if module.params.get('vpc_subnets'): 198 | vpc_options['SubnetIds'] = [x.strip() for x in module.params.get('vpc_subnets').split(',')] 199 | 200 | if module.params.get('vpc_security_groups'): 201 | vpc_options['SecurityGroupIds'] = [x.strip() for x in module.params.get('vpc_security_groups').split(',')] 202 | 203 | if cluster_config['DedicatedMasterEnabled']: 204 | cluster_config['DedicatedMasterType'] = module.params.get('dedicated_master_instance_type') 205 | cluster_config['DedicatedMasterCount'] = module.params.get('dedicated_master_instance_count') 206 | 207 | if ebs_options['EBSEnabled']: 208 | ebs_options['VolumeType'] = module.params.get('volume_type') 209 | ebs_options['VolumeSize'] = module.params.get('volume_size') 210 | 211 | snapshot_options = { 212 | 'AutomatedSnapshotStartHour': module.params.get('snapshot_hour') 213 | } 214 | 215 | changed = False 216 | 217 | try: 218 | pdoc = json.dumps(module.params.get('access_policies')) 219 | except Exception as e: 220 | module.fail_json(msg='Failed to convert the policy into valid JSON: %s' % str(e)) 221 | 222 | try: 223 | response = client.describe_elasticsearch_domain(DomainName=module.params.get('name')) 224 | status = response['DomainStatus'] 225 | 226 | # Modify the provided policy to provide reliable changed detection 227 | policy_dict = module.params.get('access_policies') 228 | for statement in policy_dict['Statement']: 229 | if 'Resource' not in statement: 230 | # The ES APIs will implicitly set this 231 | statement['Resource'] = '%s/*' % status['ARN'] 232 | pdoc = json.dumps(policy_dict) 233 | 234 | if status['ElasticsearchClusterConfig'] != cluster_config: 235 | changed = True 236 | 237 | if status['EBSOptions'] != ebs_options: 238 | changed = True 239 | 240 | if 'VPCOptions' in status: 241 | if status['VPCOptions']['SubnetIds'] != vpc_options['SubnetIds']: 242 | changed = True 243 | if status['VPCOptions']['SecurityGroupIds'] != vpc_options['SecurityGroupIds']: 244 | changed = True 245 | 246 | if status['SnapshotOptions'] != snapshot_options: 247 | changed = True 248 | 249 | current_policy_dict = json.loads(status['AccessPolicies']) 250 | if current_policy_dict != policy_dict: 251 | changed = True 252 | 253 | if changed: 254 | keyword_args = { 255 | 'DomainName': module.params.get('name'), 256 | 'ElasticsearchClusterConfig': cluster_config, 257 | 'EBSOptions': ebs_options, 258 | 'SnapshotOptions': snapshot_options, 259 | 'AccessPolicies': pdoc, 260 | } 261 | 262 | if vpc_options['SubnetIds'] or vpc_options['SecurityGroupIds']: 263 | keyword_args['VPCOptions'] = vpc_options 264 | 265 | response = client.update_elasticsearch_domain_config(**keyword_args) 266 | 267 | except botocore.exceptions.ClientError as e: 268 | changed = True 269 | 270 | if e.response['Error']['Code'] == 'ResourceNotFoundException': 271 | keyword_args = { 272 | 'DomainName': module.params.get('name'), 273 | 'ElasticsearchVersion': module.params.get('elasticsearch_version'), 274 | 'EncryptionAtRestOptions': encryption_at_rest_options, 275 | 'ElasticsearchClusterConfig': cluster_config, 276 | 'EBSOptions': ebs_options, 277 | 'SnapshotOptions': snapshot_options, 278 | 'AccessPolicies': pdoc, 279 | } 280 | 281 | if vpc_options['SubnetIds'] or vpc_options['SecurityGroupIds']: 282 | keyword_args['VPCOptions'] = vpc_options 283 | 284 | response = client.create_elasticsearch_domain(**keyword_args) 285 | 286 | else: 287 | module.fail_json(msg='Error: %s %s' % (str(e.response['Error']['Code']), str(e.response['Error']['Message'])),) 288 | 289 | # Retrieve response from describe, as create/update differ in their response format 290 | response = client.describe_elasticsearch_domain(DomainName=module.params.get('name')) 291 | module.exit_json(changed=changed, response=response) 292 | 293 | # import module snippets 294 | from ansible.module_utils.basic import * 295 | from ansible.module_utils.ec2 import * 296 | 297 | if __name__ == '__main__': 298 | main() 299 | --------------------------------------------------------------------------------