├── VERSION ├── lib ├── aws-tasks.rb └── aws-tasks │ ├── tasks │ ├── rds.rake │ └── redshift.rake │ ├── security_group.rb │ └── vpc_prefix_list.rb ├── aws-tasks.gemspec └── LICENSE.txt /VERSION: -------------------------------------------------------------------------------- 1 | 0.3.3 2 | -------------------------------------------------------------------------------- /lib/aws-tasks.rb: -------------------------------------------------------------------------------- 1 | # require 'aws-sdk-ec2' 2 | # require 'aws-sdk-rds' 3 | # require 'aws-sdk-redshift' 4 | -------------------------------------------------------------------------------- /aws-tasks.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Gem::Specification.new do |s| 4 | s.name = 'aws-tasks' 5 | s.version = File.read(File.expand_path('../VERSION', __FILE__)).strip 6 | s.summary = 'AWS helper tasks' 7 | s.description = 'AWS helper tasks for DB and such' 8 | s.homepage = 'http://github.com/attribution/aws-tasks' 9 | s.licenses = ['MIT'] 10 | s.authors = ['Sam Reh'] 11 | s.email = ['samuelreh@gmail.com'] 12 | 13 | s.require_paths = ['lib'] 14 | s.files = [ 15 | 'LICENSE.txt', 16 | 'VERSION', 17 | 'aws-tasks.gemspec', 18 | 'lib/aws-tasks.rb', 19 | 'lib/aws-tasks/security_group.rb', 20 | 'lib/aws-tasks/vpc_prefix_list.rb', 21 | 'lib/aws-tasks/tasks/rds.rake', 22 | 'lib/aws-tasks/tasks/redshift.rake' 23 | ] 24 | 25 | s.add_dependency 'aws-sdk-ec2', '~> 1' 26 | s.add_dependency 'aws-sdk-rds', '~> 1' 27 | s.add_dependency 'aws-sdk-redshift', '~> 1' 28 | end 29 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Sam Reh 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /lib/aws-tasks/tasks/rds.rake: -------------------------------------------------------------------------------- 1 | require 'aws-sdk-rds' 2 | 3 | namespace :aws do 4 | namespace :rds do 5 | desc "Launch an RDS instance" 6 | task :launch, [:id, :key, :snapshot, :new_db, :region] do |cmd, args| 7 | credentials = Aws::Credentials.new(args[:id], args[:key]) 8 | client = Aws::RDS::Client.new(region: args[:region], credentials: credentials) 9 | 10 | dump_instance = client.describe_db_instances.db_instances. 11 | select { |instance| instance.db_instance_identifier == args[:new_db]}. 12 | first 13 | 14 | if !dump_instance 15 | puts "Launching Database with identifier #{args[:new_db]}" 16 | latest_snapshot = client.describe_db_snapshots.db_snapshots. 17 | select { |snap| snap.db_instance_identifier == args[:snapshot] }. 18 | select { |snap| snap.status == 'available' }. 19 | sort { |s1,s2| s1.snapshot_create_time <=> s2.snapshot_create_time }. 20 | last 21 | 22 | dump_instance = client.restore_db_instance_from_db_snapshot( 23 | db_instance_identifier: args[:new_db], 24 | db_snapshot_identifier: latest_snapshot.db_snapshot_identifier 25 | ) 26 | else 27 | puts "Instance #{args[:new_db]} already exists" 28 | end 29 | end 30 | 31 | desc "Destroy an RDS instance" 32 | task :destroy, [:id, :key, :region, :identifier] do |cmd, args| 33 | credentials = Aws::Credentials.new(args[:id], args[:key]) 34 | client = Aws::RDS::Client.new(region: args[:region], credentials: credentials) 35 | client.delete_db_instance(db_instance_identifier: args[:identifier], skip_final_snapshot: true) 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/aws-tasks/tasks/redshift.rake: -------------------------------------------------------------------------------- 1 | require 'aws-sdk-redshift' 2 | 3 | namespace :aws do 4 | namespace :redshift do 5 | desc "Launch a Redshift instance" 6 | task :launch, [:id, :key, :snapshot, :new_db, :region, :security_group_id] do |cmd, args| 7 | credentials = Aws::Credentials.new(args[:id], args[:key]) 8 | client = Aws::Redshift::Client.new(region: args[:region], credentials: credentials) 9 | 10 | dump_instance = client.describe_clusters.clusters. 11 | select { |instance| instance.cluster_identifier == args[:new_db]}. 12 | first 13 | 14 | if !dump_instance 15 | puts "Launching Database with identifier #{args[:new_db]}" 16 | latest_snapshot = client.describe_cluster_snapshots.snapshots. 17 | select do |snap| 18 | # allow to pass cluster_identifier or snapshot_identifier for search or latest dump 19 | (snap.cluster_identifier == args[:snapshot]) || 20 | (snap.snapshot_identifier == args[:snapshot]) 21 | end. 22 | sort { |s1,s2| s1.snapshot_create_time <=> s2.snapshot_create_time }. 23 | last 24 | 25 | dump_instance = client.restore_from_cluster_snapshot( 26 | cluster_identifier: args[:new_db], 27 | snapshot_identifier: latest_snapshot.snapshot_identifier, 28 | vpc_security_group_ids: Array.wrap(args[:security_group_id]) 29 | ) 30 | else 31 | puts "Instance #{args[:new_db]} already exists" 32 | end 33 | end 34 | 35 | desc "Destroy a Redshift instance" 36 | task :destroy, [:id, :key, :region, :identifier] do |cmd, args| 37 | credentials = Aws::Credentials.new(args[:id], args[:key]) 38 | client = Aws::Redshift::Client.new(region: args[:region], credentials: credentials) 39 | client.delete_cluster(cluster_identifier: args[:identifier], skip_final_cluster_snapshot: true) 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/aws-tasks/security_group.rb: -------------------------------------------------------------------------------- 1 | # TODO remove/replace Rollbar with callback? 2 | module AwsTasks 3 | class SecurityGroup 4 | # Open APIs which allow to detect current IP 5 | # http://ipv4.whatismyip.akamai.com/ 6 | # https://ifconfig.me/ip 7 | # https://ipecho.net/plain 8 | # https://icanhazip.com/ 9 | # http://ident.me/ 10 | IP_LOOKUP_URL = 'http://ipv4.whatismyip.akamai.com/' 11 | 12 | attr_accessor :security_group_id, :port, :client 13 | 14 | # AWS CLI methods 15 | # aws ec2 describe-security-groups --group-ids sg-0dca03885a00d9ade 16 | # aws ec2 authorize-security-group-ingress \ 17 | # --group-id sg-0dca03885a00d9ade \ 18 | # --protocol tcp \ 19 | # --port 5432 \ 20 | # --cidr 3.219.217.67/32 21 | 22 | def self.get_my_ip 23 | # URI.open(IP_LOOKUP_URL).read # alternative 24 | Faraday.new(IP_LOOKUP_URL).get.body 25 | end 26 | 27 | def self.authorize_my_ip_for_ingress 28 | new.authorize_ingress_ip(get_my_ip) 29 | end 30 | 31 | def initialize(security_group_id: nil, port: nil, client: nil) 32 | if ENV['AWS_SECURITY_GROUP_COMBO'] 33 | access_key_id, secret_access_key, region, security_group_id, port = ENV['AWS_SECURITY_GROUP_COMBO'].split(':') 34 | end 35 | 36 | @security_group_id = security_group_id || ENV['AWS_SECURITY_GROUP_ID'] 37 | @port = port || ENV['AWS_SECURITY_GROUP_PORT'] 38 | @client = client || init_client( 39 | access_key_id || ENV['AWS_SECURITY_GROUP_ACCESS_KEY_ID'], 40 | secret_access_key || ENV['AWS_SECURITY_GROUP_SECRET_ACCESS_KEY'], 41 | region || ENV['AWS_SECURITY_GROUP_REGION'] 42 | ) 43 | end 44 | 45 | def init_client(access_key_id, secret_access_key, region) 46 | @client ||= Aws::EC2::Client.new( 47 | access_key_id: access_key_id, 48 | secret_access_key: secret_access_key, 49 | region: region 50 | ) 51 | end 52 | 53 | def authorize_ingress_ip(ip, retries: 0) 54 | project = ENV['PROJECT_PATH'] || 'unknown' 55 | dyno = ENV['DYNO'] || 'unknown' 56 | host = Socket.gethostname 57 | 58 | @client.authorize_security_group_ingress({ 59 | group_id: @security_group_id, 60 | ip_permissions: [{ 61 | from_port: @port, 62 | to_port: @port, 63 | ip_protocol: 'tcp', 64 | ip_ranges: [{ 65 | cidr_ip: "#{ip}/32", 66 | description: "#{Time.now.to_i} Heroku Instance p:#{project}, d:#{dyno}, h:#{host}", 67 | }] 68 | }] 69 | }) 70 | rescue Aws::EC2::Errors::InvalidPermissionDuplicate 71 | # nothing to do - IP already has access 72 | rescue Aws::EC2::Errors::RulesPerSecurityGroupLimitExceeded => error 73 | retries += 1 74 | 75 | if retries >= 3 76 | raise 77 | else 78 | Rollbar.info(error, { 79 | tries: retries, 80 | security_group_id: @security_group_id, 81 | port: @port, 82 | project: project 83 | }) 84 | 85 | revoke_oldest_security_group_rule 86 | retry 87 | end 88 | end 89 | 90 | # describe security group 91 | # result = cli.describe_security_groups(group_ids: ['sg-0dca03885a00d9ade']) 92 | # result.security_groups.first.ip_permissions.each {|sgr| p sgr} 93 | 94 | # list ingress security_group_rules sorted by description 95 | def list_security_group_rules 96 | @client. 97 | describe_security_group_rules(filters: [{ name: 'group-id', values: [@security_group_id] }]). 98 | security_group_rules. 99 | select{ _1.description.present? && _1.is_egress == false }. 100 | sort_by(&:description) 101 | end 102 | 103 | def revoke_security_group_rules(ids) 104 | @client. 105 | revoke_security_group_ingress( 106 | group_id: @security_group_id, 107 | security_group_rule_ids: Array.wrap(ids) 108 | ). 109 | tap { Rollbar.info('Removed Security Group Rule', { security_group_rules: ids }) } 110 | end 111 | 112 | def revoke_oldest_security_group_rule 113 | oldest_sgr = list_security_group_rules.first 114 | return unless oldest_sgr 115 | 116 | revoke_security_group_rules(oldest_sgr.security_group_rule_id) 117 | end 118 | end 119 | end 120 | -------------------------------------------------------------------------------- /lib/aws-tasks/vpc_prefix_list.rb: -------------------------------------------------------------------------------- 1 | require 'aws-sdk-ec2' 2 | 3 | module AwsTasks 4 | class VpcPrefixList 5 | # Open APIs which allow to detect current IP 6 | # http://ipv4.whatismyip.akamai.com/ 7 | # https://ifconfig.me/ip 8 | # https://ipecho.net/plain 9 | # https://icanhazip.com/ 10 | # http://ident.me/ 11 | IP_LOOKUP_URL = 'http://ipv4.whatismyip.akamai.com/' 12 | 13 | RETRYABLE_ERRORS = [ 14 | Aws::EC2::Errors::IncorrectState, 15 | Aws::EC2::Errors::InvalidPrefixListModification, 16 | Aws::EC2::Errors::PrefixListVersionMismatch, 17 | Aws::EC2::Errors::PrefixListMaxEntriesExceeded 18 | ] 19 | 20 | attr_accessor :prefix_list_id, :client 21 | 22 | # AWS CLI methods examples: 23 | # aws ec2 get-managed-prefix-list-entries --prefix-list-id pl-036a11494222c3d5c 24 | # aws ec2 modify-managed-prefix-list --prefix-list-id pl-036a11494222c3d5c \ 25 | # --current-version 2 \ 26 | # --add-entries Cidr=1.1.1.1/32,Description=test 27 | # aws ec2 modify-managed-prefix-list --prefix-list-id pl-036a11494222c3d5c \ 28 | # --current-version 3 \ 29 | # --remove-entries Cidr=1.1.1.1/32 \ 30 | # --add-entries Cidr=1.1.1.2/32,Description=test 31 | 32 | def self.get_my_ip 33 | require 'open-uri' 34 | URI.open(IP_LOOKUP_URL).read 35 | end 36 | 37 | # Usage: 38 | # add_my_ip { Faraday.get(Services::AwsVpcPrefixList::IP_LOOKUP_URL).body } 39 | # add_my_ip('1.1.1.1') 40 | # add_my_ip 41 | def self.add_my_ip(ip=nil, prefix_list_id: nil) 42 | ip ||= yield if block_given? 43 | ip ||= get_my_ip 44 | new(prefix_list_id: prefix_list_id).add_entry(ip) 45 | end 46 | 47 | def initialize(prefix_list_id: nil, client: nil) 48 | if ENV['AWS_VPC_COMBO'] 49 | access_key_id, secret_access_key, region, combo_prefix_list_id = ENV['AWS_VPC_COMBO'].split(':') 50 | end 51 | 52 | @prefix_list_id = prefix_list_id || combo_prefix_list_id || ENV['AWS_VPC_PREFIX_LIST_ID'] 53 | @client = client || init_client( 54 | access_key_id || ENV['AWS_VPC_ACCESS_KEY_ID'], 55 | secret_access_key || ENV['AWS_VPC_SECRET_ACCESS_KEY'], 56 | region || ENV['AWS_VPC_REGION'] 57 | ) 58 | end 59 | 60 | def init_client(access_key_id, secret_access_key, region) 61 | @client ||= Aws::EC2::Client.new( 62 | access_key_id: access_key_id, 63 | secret_access_key: secret_access_key, 64 | region: region 65 | ) 66 | end 67 | 68 | # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/EC2/Client.html#describe_managed_prefix_lists-instance_method 69 | def get_prefix_list 70 | @client. 71 | describe_managed_prefix_lists(prefix_list_ids: [@prefix_list_id]). 72 | prefix_lists. 73 | first 74 | end 75 | 76 | # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/EC2/Client.html#get_managed_prefix_list_entries-instance_method 77 | def get_prefix_list_entries 78 | @client. 79 | get_managed_prefix_list_entries(prefix_list_id: @prefix_list_id). 80 | data. 81 | entries. 82 | sort_by { _1.description } 83 | end 84 | 85 | # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/EC2/Client.html#modify_managed_prefix_list-instance_method 86 | def add_entry(ip, remove_entry_cidr: nil, version: nil, max_retries: 5) 87 | prefix_list = get_prefix_list 88 | project = ENV['PROJECT_PATH'] || 'unknown' 89 | dyno = ENV['DYNO'] || 'unknown' 90 | host = Socket.gethostname 91 | 92 | params = { 93 | prefix_list_id: @prefix_list_id, 94 | current_version: (version || prefix_list.version), 95 | add_entries: [{ 96 | cidr: "#{ip}/32", 97 | description: "#{Time.now.utc.iso8601} Heroku Instance p:#{project}, d:#{dyno}, h:#{host}" 98 | }] 99 | } 100 | 101 | if remove_entry_cidr 102 | params[:remove_entries] = [{ cidr: remove_entry_cidr }] 103 | end 104 | 105 | @client. 106 | modify_managed_prefix_list(params). 107 | tap { puts "AwsTasks::VpcPrefixList #{ip} added to #{@prefix_list_id}" + (remove_entry_cidr ? ", removed #{remove_entry_cidr}" : '') } 108 | rescue *RETRYABLE_ERRORS => error 109 | raise if max_retries < 1 110 | 111 | sleep rand(2.0...10.0) / max_retries 112 | max_retries -= 1 113 | 114 | # puts "Error #{error}, retrying..." 115 | if 116 | error.kind_of?(Aws::EC2::Errors::PrefixListMaxEntriesExceeded) || 117 | error.kind_of?(Aws::EC2::Errors::InvalidPrefixListModification) 118 | then 119 | remove_entry_cidr = get_prefix_list_entries.first.cidr 120 | end 121 | 122 | retry 123 | end 124 | end 125 | end 126 | --------------------------------------------------------------------------------