.
675 |
--------------------------------------------------------------------------------
/ansible-base.yml:
--------------------------------------------------------------------------------
1 | ---
2 | - hosts: all
3 | remote_user: root
4 | become: yes
5 | tasks:
6 | - name: install Ruby
7 | command: bash amazon-linux-extras install ruby2.4
8 | - name: install sinatra
9 | gem: name=sinatra state=latest user_install=no
10 | - name: install aws-sdk
11 | gem: name=aws-sdk state=latest user_install=no
12 | - name: disable ssh
13 | command: chkconfig sshd off
14 | - name: remove AWS remote access
15 | command: rpm -e ec2-instance-connect
16 | - name: remove ssh
17 | command: rpm -e openssh-server
18 | - name: set startup permissions
19 | command: chmod +x /etc/rc.d/rc.local
20 | - name: start demo web server
21 | lineinfile: dest=/etc/rc.d/rc.local line="nohup ruby /home/ec2-user/cloudsec_advanced_server.rb &"
22 | - name: download inspector
23 | command: wget https://d1wk0tztpsntt1.cloudfront.net/linux/latest/install
24 | - name: swap in log receiver address
25 | lineinfile:
26 | path: /etc/rsyslog.conf
27 | regexp: '#*.* @@remote-host:514'
28 | line: '*.* @@10.0.1.249:5140'
29 | - name: install inspector
30 | command: bash install
31 |
--------------------------------------------------------------------------------
/cloudsec_advanced_server.rb:
--------------------------------------------------------------------------------
1 | # This is an extremely lightweight web app to show the current status of a rolling auto scale update, updated every 1 second
2 |
3 | require 'sinatra'
4 | require 'aws-sdk'
5 | require 'net/http'
6 | require 'open-uri'
7 |
8 | # Hardcode region
9 | $region = "us-west-2"
10 | # Create EC2 and autoscaling clients. Wide variable scope due to Sinatra
11 |
12 | @@ec2 = Aws::EC2::Client.new(region: "#{$region}")
13 | @@autoscaling = Aws::AutoScaling::Client.new(region: "#{$region}")
14 |
15 | # Pull the current image and instance ID
16 | metadata_endpoint = 'http://169.254.169.254/latest/meta-data/'
17 | @@image_id = Net::HTTP.get( URI.parse( metadata_endpoint + 'ami-id' ) )
18 | @@instance_id = Net::HTTP.get( URI.parse( metadata_endpoint + 'instance-id' ) )
19 |
20 | # Identify the current auto scale group. Since this demo may change over time that's better than hard coding.
21 | @metadata = @@ec2.describe_instances(instance_ids: ["#{@@instance_id}"])
22 | tags = @metadata.reservations.first.instances.first
23 | # covert to hash to make this easier
24 | tags = tags.to_h
25 | tags = tags[:tags]
26 | # quick check to avoid having to iterate through all the tags to see if the one we need is there.
27 | temp_tags = tags.to_s
28 | if temp_tags.include?("aws:autoscaling:groupName")
29 | tags.each do |curtag|
30 | if curtag[:key] == "aws:autoscaling:groupName"
31 | @@autoscalegroup = curtag[:value]
32 | end
33 | end
34 | else
35 | @@autoscalegroup = "false"
36 | end
37 |
38 | def create_list
39 |
40 | if @@autoscalegroup != "false"
41 | # Pull all the instances
42 | asg = @@autoscaling.describe_auto_scaling_groups({
43 | auto_scaling_group_names: ["#{@@autoscalegroup}"]})
44 | # next line is hard coded for testing, can remove if we place this in the ASG and use lines above instead
45 | # asg = @@autoscaling.describe_auto_scaling_groups({
46 | # auto_scaling_group_names: ["test2"]})
47 | @@instancelist = asg.auto_scaling_groups.first.instances.map(&:instance_id)
48 | puts @@instancelist
49 |
50 |
51 | @@oldlist = {}
52 | @@instancelist.each do |instance|
53 | image = @@ec2.describe_instances(instance_ids: ["#{instance}"])
54 | image = image.reservations.first.instances.first.image_id
55 | if image == "#{@@image_id}"
56 | @@oldlist["#{instance}"] = "#{image}"
57 | end
58 | end
59 |
60 | @@newlist = {}
61 | @@instancelist.each do |instance|
62 | image = @@ec2.describe_instances(instance_ids: ["#{instance}"])
63 | image = image.reservations.first.instances.first.image_id
64 | if image != "#{@@image_id}"
65 | @@newlist["#{instance}"] = "#{image}"
66 | end
67 | end
68 |
69 |
70 | puts @@oldlist
71 | puts "---"
72 | puts @@newlist
73 | end
74 | end
75 |
76 |
77 | # Set Sinatra to production mode so it will accept outside http connections
78 | set :environment, :production
79 |
80 | # Start all the sinatra stuff
81 | get '/' do
82 | erb :index
83 | end
84 |
85 | __END__
86 | @@ layout
87 |
88 |
89 |
90 | Securosis Advanced Cloud Security Training Server
91 |
92 |
93 |
94 | <%= yield %>
95 |
96 |
97 |
98 | @@ index
99 | <% create_list %>
100 | Securosis Advanced Cloud Security Training Server
101 |
102 | I may be ugly, but I get the job done
103 |
104 | Current Instance ID: <%= @@instance_id %>
105 | Current Image ID: <%= @@image_id %>
106 |
107 |
108 | <% begin %>
109 | <% display_link = open('https://s3-us-west-2.amazonaws.com/advanced-cloudsec/config.txt') {|f| f.read } %>
110 | <% if (display_link[0..3] == "http") %>
111 | 
112 | <% else %>
113 | Fail 2 Sorry, no S3 service endpoint means no dynamite
114 | <% end %>
115 | <% rescue %>
116 | Fail 1 Sorry, no S3 service endpoint means no dynamite
117 | <% end %>
118 |
119 |
120 | <% if @@autoscalegroup != "false" %>
121 | Instances in ASG
122 | <% @@oldlist.each do |instance, image| %>
123 | Instance: <%= instance %>
Image <%= image %>
124 | <% end %>
125 | <% @@newlist.each do |instance, image| %>
126 | Instance: <%= instance %>
Image <%= image %>
127 | <% end %>
128 | <% else %>
129 | This instance is not in an auto scale group
130 | <% end %>
--------------------------------------------------------------------------------
/configure-logs.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | NAME=$(curl http://169.254.169.254/latest/meta-data/instance-id)
4 | # hostnamectl set-hostname $NAME
5 | # echo '/bin/hostname $NAME' >> /etc/rc.local
6 |
7 | echo '/bin/hostname $NAME'
8 | sudo systemctl restart network
9 |
10 | sudo service rsyslog restart
--------------------------------------------------------------------------------
/cred_scanner.py:
--------------------------------------------------------------------------------
1 | import os
2 | import re
3 | import sys
4 | import click
5 |
6 | @click.command()
7 | @click.option('--path', default='.', help='Path other than the local directory to scan')
8 | @click.option('--secret', is_flag=True, help='Also look for Secret Key patterns. This may result in many false matches due to the nature of secret keys.')
9 | def scan(path, secret):
10 | fail = False
11 | for dirname, dirnames, filenames in os.walk(path):
12 | # print path to all subdirectories first.
13 | for subdirname in dirnames:
14 | print(os.path.join(dirname, subdirname))
15 |
16 | # print path to all filenames.
17 | for filename in filenames:
18 | click.echo(os.path.join(dirname, filename))
19 | f = open(os.path.join(dirname, filename))
20 | if secret:
21 | pattern = re.compile('(?
19 | """
20 | Then the output should not contain:
21 | """
22 | 22/tcp
23 | """
24 | Scenario: Verify server is open on expected set of ports
25 | When I launch an "nmap" attack with:
26 | """
27 | nmap -p 4567
28 | """
29 | Then the output should match:
30 | """
31 | 4567/tcp\s+open
32 | """
--------------------------------------------------------------------------------
/packer.json:
--------------------------------------------------------------------------------
1 | {
2 | "builders": [{
3 | "type": "amazon-ebs",
4 | "access_key": "",
5 | "secret_key": "",
6 | "region": "us-west-2",
7 | "source_ami": "ami-082b5a644766e0e6f",
8 | "instance_type": "t2.micro",
9 | "ssh_username": "ec2-user",
10 | "ami_name": "cloudsec-training-{{timestamp}}"
11 | }],
12 |
13 | "provisioners": [
14 | {
15 | "type": "shell",
16 | "inline": ["sudo amazon-linux-extras install ansible2"]
17 | },
18 | {
19 | "type": "ansible-local",
20 | "playbook_file": "ansible-base.yml",
21 | "extra_arguments": [ "--verbose" ]
22 | },
23 | {
24 | "type": "file",
25 | "source": "cloudsec_advanced_server.rb",
26 | "destination": "/home/ec2-user/cloudsec_advanced_server.rb"
27 | },
28 | {
29 | "type": "file",
30 | "source": "configure-logs.sh",
31 | "destination": "/home/ec2-user/configure-logs.sh"
32 | }
33 | ]
34 | }
35 |
--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | this is a test
2 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | click
2 | boto3
3 | requests
4 | sh
--------------------------------------------------------------------------------
/rolling_update.rb:
--------------------------------------------------------------------------------
1 | # Workflow for rolling out an updated AMI in a rolling patch scenario on AWS
2 | # Copyright Securosis, LLC, 2016, all rights reserved
3 | # Note that credentials for this workflow rely on the current role of where it is running!
4 | # TODO automatically create qurantine security group if needed
5 |
6 |
7 | require 'aws-sdk'
8 | require 'json'
9 | require 'optparse'
10 |
11 |
12 | class AutoscaleActions
13 | def initialize
14 | # Initialize the needed service clients
15 | sts = Aws::STS::Client.new(region: "#{$region}")
16 | role = sts.assume_role({
17 | role_arn: "arn:aws:iam::#{$account_id}:role/SecOps",
18 | role_session_name: "cloudsec-jenkins",
19 | })
20 |
21 | @@ec2 = Aws::EC2::Client.new(credentials: role, region: "#{$region}")
22 | @@autoscaling = Aws::AutoScaling::Client.new(credentials: role, region: "#{$region}")
23 | end
24 |
25 | def get_autoscale_group_details(asg_name)
26 | # This method pulls the current launch configuration and image_id when given an auto scale group
27 | begin
28 | # Get details for the named ASG
29 | puts "Getting autoscale group details"
30 | asg_details = @@autoscaling.describe_auto_scaling_groups({
31 | auto_scaling_group_names: ["#{asg_name}"]})
32 | # Check to see if it was a valid name for the current region
33 | if asg_details.auto_scaling_groups == []
34 | puts "#{asg_name} is not a valid auto scale group in region #{$region}"
35 | exit
36 | else
37 | # Get the launch configuration name and then the associated image ID
38 | puts "Getting launch configuration name"
39 | launch_configuration_name = asg_details.auto_scaling_groups.first.launch_configuration_name
40 | puts "Getting launch configuration details"
41 | launch_config = @@autoscaling.describe_launch_configurations({launch_configuration_names: ["#{launch_configuration_name}"]})
42 | # get the current AMI
43 | current_image_id = launch_config.launch_configurations.first.image_id
44 | puts "Current launch configuration is #{launch_configuration_name} and image id is #{current_image_id}"
45 | return launch_configuration_name, current_image_id
46 | end
47 | puts "Current launch configuration is #{launch_configuration_name} and image id is #{current_image_id}"
48 | rescue Aws::AutoScaling::Errors::ServiceError => error
49 | puts "error encountered in get_autoscale_group_details: "
50 | puts "#{error.message}"
51 | end
52 | end
53 |
54 | def change_launchconfiguration_ami(launch_configuration_name, new_ami)
55 | # this method changes the AMI associated with a launch configuration.
56 | # It pulls the current configuration, then creates a new one with the updated AMI
57 | # then returns the new launch configuration to swap into an auto scale group.
58 | begin
59 | puts new_ami
60 | # pull the current configuration as an object
61 | puts launch_configuration_name
62 | launch_config = @@autoscaling.describe_launch_configurations({launch_configuration_names: ["#{launch_configuration_name}"]})
63 | # Confirm the new AMI is valid
64 | test = @@ec2.describe_images({image_ids: ["#{new_ami}"]})
65 | if test == []
66 | puts "Invalid AMI image ID, exiting."
67 | exit
68 | end
69 | # change to the new AMI
70 | launch_config.launch_configurations.first.image_id = new_ami
71 | curname = launch_config.launch_configurations.first.launch_configuration_name
72 | # See if the name has been previously modified by this workflow. If so, trim the end so it doesn't append the timestamp again
73 | if (/-[0-9]{14}TR$/x.match(curname) != nil)
74 | curname = curname[0..-19]
75 | end
76 | time = Time.now()
77 | time = time.strftime("%Y%m%d%H%M%S")
78 | newname = curname + "-" + time + "TR"
79 | # swap in the new name
80 | launch_config.launch_configurations.first.launch_configuration_name = newname
81 | # convert to hash to create the new launch config
82 | launch_config = launch_config.launch_configurations.first.to_h
83 | # Delete some things that will cause errors
84 | launch_config.delete(:launch_configuration_arn)
85 | launch_config.delete(:created_time)
86 | if launch_config[:kernel_id] == ""
87 | launch_config.delete(:kernel_id)
88 | end
89 | if launch_config[:ramdisk_id] == ""
90 | launch_config.delete(:ramdisk_id)
91 | end
92 | if launch_config[:key_name] == ""
93 | launch_config.delete(:key_name)
94 | end
95 | if launch_config[:block_device_mappings] == []
96 | launch_config.delete(:block_device_mappings)
97 | elsif launch_config[:block_device_mappings].first[:ebs].has_key?(:snapshot_id) == true
98 | launch_config[:block_device_mappings].first[:ebs].delete(:snapshot_id)
99 | end
100 | puts launch_config
101 | @@autoscaling.create_launch_configuration(launch_config)
102 | puts "New launch configuration created"
103 | return newname
104 | rescue Aws::AutoScaling::Errors::ServiceError => error
105 | puts "error encountered in method"
106 | puts "#{error.message}"
107 | end
108 | end
109 |
110 | def change_autoscale_launch_configuration(asg_name, launchconfiguration_name)
111 | # this method changes the launch configuration associated with an auto scale group
112 | begin
113 | # Swap in the launch configuration
114 | @@autoscaling.update_auto_scaling_group({auto_scaling_group_name: "#{asg_name}", launch_configuration_name: "#{launchconfiguration_name}"})
115 | rescue Aws::AutoScaling::Errors::ServiceError => error
116 | puts "error encountered in change_autoscale_launch_configuration"
117 | puts "#{error.message}"
118 | end
119 | end
120 |
121 | def self.rolling_autoscale_update(asg_name, old_ami, interval, batch_size, mode)
122 | # This method degrades, isolates, or terminates instances in an auto scale group using
123 | # the requested mode, time interval, and batch size.
124 | # Supported modes are degrade_health (mark instances as unhealthy and let the ASG manage the update),
125 | # terminate (rolling terminate the instances), detach_and_quarantine (detach the instances from the
126 | # ASG and put them in a quarantined security group)
127 | begin
128 | # Get a list of all the instances in the ASG
129 | auto_scale_description = @@autoscaling.describe_auto_scaling_groups({
130 | auto_scaling_group_names: ["#{asg_name}"]})
131 | instancelist = auto_scale_description.auto_scaling_groups.first.instances
132 | instancelist = instancelist.map(&:instance_id)
133 |
134 |
135 | # Ensure the min count is sufficient for the selected batch size. If not, reduce the batch size.
136 | instance_min = auto_scale_description.auto_scaling_groups.first.min_size
137 | if ((batch_size / 2) > instance_min)
138 | puts "Requested batch size of #{batch_size} may be too large for current auto scale group settings."
139 | batch_size = (batch_size / 2)
140 | puts "batch_size reduced to #{batch_size}"
141 | end
142 |
143 | # Make sure batch size is valid
144 | if batch_size <= 1
145 | batch_size = 1
146 | end
147 |
148 | # puts "Instances in the auto scale group:"
149 | # puts instancelist
150 | # Initialize the batch counter
151 | batch_counter = 1
152 | # Roll through the instances. If the AMI is the expired one, remove from the ASG using the desired method.
153 | instancelist.each do |curinstance|
154 | # Get the AMI for the instance and see if it is the one marked to remove
155 | instance = @@ec2.describe_instances({instance_ids: ["#{curinstance}"]})
156 | if instance.reservations.first.instances.first.image_id == old_ami
157 | unless ((instance.reservations.first.instances.first.state.name == "terminated") or (instance.reservations.first.instances.first.state.name == "shutting-down"))
158 | if batch_counter <= batch_size
159 | # add a 1 second delay to avoid API request limits
160 | sleep(1)
161 | print "Instance #{curinstance} running on old AMI. "
162 | if mode == "degrade_health"
163 | puts "Degrading health. Auto scale group will terminate the instance."
164 | # set the health status to unhealthy. The ASG will handle termination and replacement
165 | @@autoscaling.set_instance_health({instance_id: curinstance,
166 | health_status: "Unhealthy"})
167 | batch_counter += 1
168 | sleep(1)
169 | elsif mode == "terminate"
170 | puts "Terminating the instance."
171 | # Terminate the instance
172 | @@autoscaling.terminate_instance_in_auto_scaling_group({instance_id: curinstance})
173 | batch_counter += 1
174 | sleep(1)
175 | elsif mode == "detach_and_quarantine"
176 | puts "Detaching the instance from the auto scale group and setting Quarantine tag to Active."
177 | # Detach the instance from the ASG, then quarantine it using the current config setting, then
178 | # tag it somehow.
179 | # TODO need to pull quarantine settings. Need to determine what to tag it with.
180 | @@autoscaling.detach_instances({instance_ids: ["#{curinstance}", auto_scaling_group_name: asg_name]})
181 | # Note: to actually quarantine the instance you need to specify a quarantine security group and uncomment the line below
182 | # @@ec2.modify_instance_attribute(instance_id: curinstance, groups: ["#{sg-ead03c85}"])
183 | @@ec2.create_tags(resources: ["#{@instance_id}"], tags: [
184 | {
185 | key: "Quarantine",
186 | value: "Active",
187 | },
188 | ],)
189 | batch_counter += 1
190 | sleep(1)
191 | end
192 | else
193 | puts "Completed batch run of #{batch_size} and pausing for #{interval} seconds."
194 | sleep(interval)
195 | batch_counter = 0
196 | end
197 | end
198 | end
199 | end
200 | rescue Aws::AutoScaling::Errors::ServiceError => error
201 | puts "error encountered in rolling_autoscale_update"
202 | puts "#{error.message}"
203 | end
204 | return true
205 | end
206 |
207 | def manage_rolling_update(asg_name, old_ami, interval, batch_size, mode)
208 | # Supervisor method to keep process running until the auto scale group is free of instances on the old AMI.
209 | # Note that since we use the rolling update code in other areas, there are some overlaps between this meithod
210 | # and rolling_autoscale_update. These don't affect function but are not totally optimized.
211 |
212 | # Set the initial array and leave a placeholder value so the loop starts running
213 | instancelist = [1]
214 | until instancelist == []
215 | # Build a list of all the instances in the auto scale group
216 | auto_scale_description = @@autoscaling.describe_auto_scaling_groups({
217 | auto_scaling_group_names: ["#{asg_name}"]})
218 | instancelist = auto_scale_description.auto_scaling_groups.first.instances
219 | instancelist = instancelist.map(&:instance_id)
220 |
221 | # Check those instances to find which ones are running on the old image. Loop will exit if none
222 | instancelist = @@ec2.describe_instances({instance_ids: instancelist, filters: [
223 | {
224 | name: "image-id",
225 | values: ["#{old_ami}"],
226 | }]})
227 | instancelist = instancelist.reservations
228 | update = false
229 | update = self.class.rolling_autoscale_update(asg_name, old_ami, interval, batch_size, mode)
230 | sleep(1)
231 | until update == true
232 | sleep(1)
233 | end
234 | end
235 | puts "All running instances now based on the new AMI."
236 | end
237 |
238 | end
239 |
240 | # Set empty hash to hold command line options
241 | options = {}
242 | optparse = OptionParser.new do |opts|
243 | # opts.banner = "Usage: rolling_update.rb [options] [auto scale group name] [new AMI image ID]"
244 |
245 | options[:suppress] = false
246 | opts.on( '-y', '--yes', 'Suppress confirmation to proceed' ) do
247 | options[:suppress] = true
248 | end
249 |
250 | options[:region] = "us-west-2"
251 | opts.on( '-r', '--region REGION', 'Set region. Default is us-west-2' ) do |region|
252 | options[:region] = region
253 | end
254 |
255 | options[:account_id] = ""
256 | opts.on( '-a', '--account_id ACCOUNTID', 'Set the account ID for the AWS account. Default is none' ) do |account_id|
257 | options[:account_id] = account_id
258 | end
259 |
260 | options[:mode] = "degrade_health"
261 | opts.on( '-m', '--mode MODE', 'Set the mode for removing instances from the auto scale group. Default is degrade_health. Other options are terminate and detach_and_quarantine' ) do |method|
262 | options[:mode] = mode
263 | end
264 |
265 | options[:batch_size] = 1
266 | opts.on( '-b', '--batch SIZE', 'The number of instances to remove from the group during each round. Default is 5' ) do |size|
267 | options[:batch_size] = size
268 | end
269 |
270 | options[:interval] = 60
271 | opts.on( '-i', '--interval SECONDS', 'The amount of time to wait between each batch. Default is 60 seconds' ) do |interval|
272 | options[:interval] = interval
273 | end
274 |
275 | opts.on( '-h', '--help', 'Display this screen' ) do
276 | puts opts
277 | exit
278 | end
279 | end
280 |
281 |
282 | # Parse the command line options
283 | optparse.parse!
284 | # Set the region
285 | $region = options[:region]
286 | $account_id = options[:account_id]
287 | # Initialize the class for the auto scale group actions
288 | autoscale = AutoscaleActions.new()
289 |
290 | # Set the required variables based on the arguments. Validity is checked later in the application.
291 | asg_name = ARGV.shift
292 | new_ami = ARGV.shift
293 |
294 | # Require the user to manually approve execution unless the suppress option is set
295 | if options[:suppress] == false
296 | puts "WARNING! This application will significantly alter an auto scale group, changing the launch configuration, swapping out the AMI, and degrading or terminating running instances. Press Y to continue or any other key to exit."
297 | confirm = gets.chomp
298 | if confirm != "Y"
299 | exit
300 | end
301 | else
302 | puts "WARNING! This application is about to significantly alter an auto scale group, changing the launch configuration, swapping out the AMI, and degrading or terminating running instances."
303 | puts "These actions are not automatically reversable, and if you see this message it means you supressed manual confirmation."
304 | puts "So now it's too late, unless you kill this app *really quickly*."
305 | end
306 |
307 | # Start the rolling update process by pulling the existing AMI image id and launch configuration name
308 | vars = autoscale.get_autoscale_group_details(asg_name)
309 | launch_configuration_name = vars[0]
310 | old_ami = vars[1]
311 |
312 | # Create a new launch configuration that swaps in the new AMI image id, then update the auto scale group.
313 | puts "Updating the launch configuration and auto scale group to use the new AMI image ID."
314 | new_launchconfiguration_name = autoscale.change_launchconfiguration_ami("#{launch_configuration_name}", "#{new_ami}")
315 | autoscale.change_autoscale_launch_configuration(asg_name, new_launchconfiguration_name)
316 | puts "Auto scale group updated. Degraded and terminated instances will be replaced with the updated image."
317 | sleep(5)
318 |
319 |
320 | # Begin the rolling update process.
321 | puts "Beginning rolling update. Existing this program before completion may result in instances based on the old AMI remaining in service."
322 | autoscale.manage_rolling_update(asg_name, old_ami, options[:interval], options[:batch_size], options[:mode])
323 |
--------------------------------------------------------------------------------