├── .github ├── stale.yml └── workflows │ ├── check-aws-regions.yml │ ├── docker-build.yml │ └── smoke-test.yml ├── .gitignore ├── .rubocop.yml ├── .solargraph.yml ├── Dockerfile ├── Gemfile ├── LICENSE.txt ├── Rakefile ├── aws_recon.gemspec ├── bin ├── aws_recon ├── console └── setup ├── binstub └── aws_recon ├── lib ├── aws_recon.rb └── aws_recon │ ├── aws_recon.rb │ ├── collectors.rb │ ├── collectors │ ├── accessanalyzer.rb │ ├── acm.rb │ ├── apigateway.rb │ ├── apigatewayv2.rb │ ├── applicationautoscaling.rb │ ├── athena.rb │ ├── autoscaling.rb │ ├── backup.rb │ ├── cloudformation.rb │ ├── cloudfront.rb │ ├── cloudtrail.rb │ ├── cloudwatch.rb │ ├── cloudwatchlogs.rb │ ├── codebuild.rb │ ├── codepipeline.rb │ ├── configservice.rb │ ├── directconnect.rb │ ├── directoryservice.rb │ ├── dms.rb │ ├── dynamodb.rb │ ├── ec2.rb │ ├── ecr.rb │ ├── ecrpublic.rb │ ├── ecs.rb │ ├── efs.rb │ ├── eks.rb │ ├── elasticache.rb │ ├── elasticloadbalancing.rb │ ├── elasticloadbalancingv2.rb │ ├── elasticsearch.rb │ ├── emr.rb │ ├── firehose.rb │ ├── glue.rb │ ├── guardduty.rb │ ├── iam.rb │ ├── kafka.rb │ ├── kinesis.rb │ ├── kms.rb │ ├── lambda.rb │ ├── lightsail.rb │ ├── organizations.rb │ ├── rds.rb │ ├── redshift.rb │ ├── route53.rb │ ├── route53domains.rb │ ├── s3.rb │ ├── sagemaker.rb │ ├── secretsmanager.rb │ ├── securityhub.rb │ ├── servicequotas.rb │ ├── ses.rb │ ├── shield.rb │ ├── sns.rb │ ├── sqs.rb │ ├── ssm.rb │ ├── support.rb │ ├── transfer.rb │ ├── wafv2.rb │ ├── workspaces.rb │ └── xray.rb │ ├── lib │ ├── formatter.rb │ ├── mapper.rb │ └── patch.rb │ ├── options.rb │ ├── services.yaml │ └── version.rb ├── readme.md ├── test ├── aws_recon_test.rb └── test_helper.rb └── utils ├── aws ├── check_region_exclusions.rb └── regions.yaml ├── cloudformation └── aws-recon-cfn-template.yml └── terraform ├── cloudwatch.tf ├── ecs.tf ├── iam.tf ├── main.tf ├── output.tf ├── readme.md ├── s3.tf ├── vars.tf └── vpc.tf /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 30 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 5 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - pinned 8 | - security 9 | # Label to use when marking an issue as stale 10 | staleLabel: wontfix 11 | # Comment to post when marking an issue as stale. Set to `false` to disable 12 | markComment: > 13 | This issue has been automatically marked as stale because it has not had 14 | recent activity. It will be closed if no further activity occurs. Thank you 15 | for your contributions. 16 | # Comment to post when closing a stale issue. Set to `false` to disable 17 | closeComment: false 18 | -------------------------------------------------------------------------------- /.github/workflows/check-aws-regions.yml: -------------------------------------------------------------------------------- 1 | name: check-service-regions 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: '0 10 * * *' 7 | 8 | jobs: 9 | region-check: 10 | runs-on: ubuntu-20.04 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v2 14 | with: 15 | fetch-depth: 1 16 | - name: Check AWS service regions 17 | run: | 18 | cd utils/aws ; ruby check_region_exclusions.rb 19 | -------------------------------------------------------------------------------- /.github/workflows/docker-build.yml: -------------------------------------------------------------------------------- 1 | name: docker-build 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: build 7 | paths: 8 | - 'lib/aws_recon/version.rb' 9 | 10 | jobs: 11 | docker-build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v3 16 | with: 17 | fetch-depth: 1 18 | - name: Set up QEMU 19 | uses: docker/setup-qemu-action@v1 20 | - name: Set up Docker Buildx 21 | uses: docker/setup-buildx-action@v1 22 | - name: Login to DockerHub 23 | uses: docker/login-action@v1 24 | with: 25 | username: ${{ secrets.DOCKERHUB_USERNAME }} 26 | password: ${{ secrets.DOCKERHUB_TOKEN }} 27 | - name: Set version tag 28 | run: | 29 | echo "VERSION_TAG=$(grep VERSION lib/aws_recon/version.rb | awk -F\" '{print $2}')" >> $GITHUB_ENV 30 | - name: Build and push 31 | id: docker_build 32 | uses: docker/build-push-action@v2 33 | with: 34 | push: true 35 | platforms: linux/amd64,linux/arm64 36 | build-args: | 37 | VERSION=${{ env.VERSION_TAG }} 38 | tags: | 39 | darkbitio/aws_recon:${{ env.VERSION_TAG }} 40 | darkbitio/aws_recon:latest 41 | -------------------------------------------------------------------------------- /.github/workflows/smoke-test.yml: -------------------------------------------------------------------------------- 1 | name: smoke-test 2 | 3 | on: 4 | push: 5 | branches: main 6 | 7 | jobs: 8 | smoke-test: 9 | runs-on: ubuntu-20.04 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v2 13 | with: 14 | fetch-depth: 1 15 | - name: Set version tag 16 | run: | 17 | echo "VERSION_TAG=$(grep VERSION lib/aws_recon/version.rb | awk -F\" '{print $2}')" >> $GITHUB_ENV 18 | - name: Smoke Test :${{ env.VERSION_TAG }} 19 | run: | 20 | docker run -t --rm darkbitio/aws_recon:${{ env.VERSION_TAG }} aws_recon 21 | - name: Smoke Test :latest 22 | run: | 23 | docker run -t --rm darkbitio/aws_recon:latest aws_recon 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.json 3 | Gemfile.lock 4 | .rvmrc 5 | .ruby-gemset 6 | .ruby-version 7 | /.bundle/ 8 | /.yardoc 9 | /_yardoc/ 10 | /coverage/ 11 | /doc/ 12 | /pkg/ 13 | /spec/reports/ 14 | /tmp/ 15 | .terraform* 16 | terraform.tfstate* 17 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | # The behavior of RuboCop can be controlled via the .rubocop.yml 2 | # configuration file. It makes it possible to enable/disable 3 | # certain cops (checks) and to alter their behavior if they accept 4 | # any parameters. The file can be placed either in your home 5 | # directory or in some project directory. 6 | # 7 | # RuboCop will start looking for the configuration file in the directory 8 | # where the inspected file is and continue its way up to the root directory. 9 | # 10 | # See https://docs.rubocop.org/rubocop/configuration 11 | Layout/LineLength: 12 | Max: 100 13 | Style/FrozenStringLiteralComment: 14 | EnforcedStyle: always_true 15 | SafeAutoCorrect: true 16 | Style/ClassAndModuleChildren: 17 | Enabled: false 18 | Metrics/BlockLength: 19 | Enabled: false 20 | Metrics/MethodLength: 21 | Enabled: false 22 | Metrics/PerceivedComplexity: 23 | Enabled: false 24 | Metrics/CyclomaticComplexity: 25 | Enabled: false 26 | Metrics/AbcSize: 27 | Enabled: false 28 | -------------------------------------------------------------------------------- /.solargraph.yml: -------------------------------------------------------------------------------- 1 | --- 2 | include: 3 | - "**/*.rb" 4 | exclude: 5 | - spec/**/* 6 | - test/**/* 7 | - vendor/**/* 8 | - ".bundle/**/*" 9 | require: [] 10 | domains: [] 11 | reporters: 12 | - rubocop 13 | require_paths: [] 14 | plugins: [] 15 | max_files: 5000 16 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG RUBY_VERSION=2.7.3 2 | FROM ruby:${RUBY_VERSION}-alpine 3 | 4 | LABEL maintainer="Darkbit " 5 | 6 | # Supply AWS Recon version at build time 7 | ARG VERSION 8 | ARG USER=recon 9 | ARG GEM=aws_recon 10 | ARG BUNDLER_VERSION=2.1.4 11 | 12 | # Install new Bundler version 13 | RUN rm /usr/local/lib/ruby/gems/*/specifications/default/bundler-*.gemspec && \ 14 | gem uninstall bundler && \ 15 | gem install bundler -v ${BUNDLER_VERSION} 16 | 17 | # Install gem 18 | RUN gem install ${GEM} -v ${VERSION} 19 | 20 | # Create non-root user 21 | RUN addgroup -S ${USER} && \ 22 | adduser -S ${USER} \ 23 | -G ${USER} \ 24 | -s /bin/ash \ 25 | -h /${USER} 26 | 27 | # Copy binstub 28 | COPY binstub/${GEM} /usr/local/bundle/bin/ 29 | RUN chmod +x /usr/local/bundle/bin/${GEM} 30 | 31 | # Switch user 32 | USER ${USER} 33 | WORKDIR /${USER} 34 | 35 | CMD ["ash"] 36 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | git_source(:github) {|repo_name| "https://github.com/#{repo_name}" } 4 | 5 | # Specify your gem's dependencies in aws_recon.gemspec 6 | gemspec 7 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Darkbit, LLC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rake/testtask" 3 | 4 | Rake::TestTask.new(:test) do |t| 5 | t.libs << "test" 6 | t.libs << "lib" 7 | t.test_files = FileList["test/**/*_test.rb"] 8 | end 9 | 10 | task :default => :test 11 | -------------------------------------------------------------------------------- /aws_recon.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | lib = File.expand_path('lib', __dir__) 4 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 5 | require 'aws_recon/version' 6 | 7 | Gem::Specification.new do |spec| 8 | spec.name = 'aws_recon' 9 | spec.version = AwsRecon::VERSION 10 | spec.authors = ['Josh Larsen', 'Darkbit'] 11 | spec.required_ruby_version = '>= 2.6.0' 12 | spec.summary = 'A multi-threaded AWS security-focused inventory collection tool.' 13 | spec.description = 'AWS Recon is a command line tool to collect resources from an Amazon Web Services (AWS) account. The tool outputs JSON suitable for processing with other tools.' 14 | spec.homepage = 'https://github.com/darkbitio/aws-recon' 15 | spec.license = 'MIT' 16 | 17 | # Specify which files should be added to the gem when it is released. 18 | # The `git ls-files -z` loads the files in the RubyGem that have been added into git. 19 | spec.files = Dir.chdir(File.expand_path(__dir__)) do 20 | `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 21 | end 22 | spec.bindir = 'bin' 23 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 24 | spec.require_paths = ['lib'] 25 | 26 | spec.add_dependency 'aws-sdk', '~> 3.1' 27 | spec.add_dependency 'parallel', '~> 1.20.1' 28 | 29 | spec.add_development_dependency 'bundler', '~> 2.2.17' 30 | spec.add_development_dependency 'byebug', '~> 11.1' 31 | spec.add_development_dependency 'gem-release', '~> 2.1' 32 | spec.add_development_dependency 'minitest', '~> 5.0' 33 | spec.add_development_dependency 'pry', '~> 0.13.1' 34 | spec.add_development_dependency 'rake', '~> 12.3' 35 | spec.add_development_dependency 'rubocop', '~> 1.15' 36 | spec.add_development_dependency 'solargraph', '~> 0.40.4' 37 | end 38 | -------------------------------------------------------------------------------- /bin/aws_recon: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # for local testing 4 | $LOAD_PATH.unshift(File.expand_path(File.join('..', '..', 'lib'), __FILE__)) 5 | 6 | require 'aws_recon' 7 | 8 | AwsRecon::CLI.new.start(ARGV) 9 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "aws_recon" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start(__FILE__) 15 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /binstub/aws_recon: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # 4 | # Manually generated binstub 5 | # 6 | 7 | require "rubygems" 8 | require "bundler/setup" 9 | 10 | load Gem.bin_path("aws_recon", "aws_recon") 11 | -------------------------------------------------------------------------------- /lib/aws_recon.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module AwsRecon 4 | end 5 | 6 | require 'aws_recon/lib/patch.rb' 7 | String.include PolicyStringParser 8 | 9 | require 'parallel' 10 | require 'ostruct' 11 | require 'optparse' 12 | require 'yaml' 13 | require 'csv' 14 | require 'aws-sdk' 15 | require 'aws_recon/options.rb' 16 | require 'aws_recon/lib/mapper.rb' 17 | require 'aws_recon/lib/formatter.rb' 18 | require 'aws_recon/collectors.rb' 19 | 20 | require 'aws_recon/version' 21 | require 'aws_recon/aws_recon' 22 | -------------------------------------------------------------------------------- /lib/aws_recon/aws_recon.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | SERVICES_CONFIG_FILE = File.join(File.dirname(__FILE__), 'services.yaml').freeze 4 | 5 | module AwsRecon 6 | class CLI 7 | def initialize 8 | # parse options 9 | @options = Parser.parse ARGV.empty? ? %w[-h] : ARGV 10 | 11 | # timing 12 | @starting = Process.clock_gettime(Process::CLOCK_MONOTONIC) 13 | 14 | # AWS account id 15 | @account_id = Aws::STS::Client.new.get_caller_identity.account 16 | 17 | # AWS services 18 | @aws_services = YAML.safe_load(File.read(SERVICES_CONFIG_FILE), symbolize_names: true) 19 | 20 | # User config services 21 | if @options.config_file 22 | user_config = YAML.safe_load(File.read(@options.config_file), symbolize_names: true) 23 | 24 | @services = user_config[:services] 25 | @regions = user_config[:regions] 26 | else 27 | @regions = @options.regions 28 | @services = @options.services 29 | end 30 | 31 | # collection 32 | @resources = [] 33 | 34 | # formatter 35 | @formatter = Formatter.new 36 | 37 | return if @options.stream_output 38 | 39 | puts "\nStarting collection with #{@options.threads} threads...\n" 40 | end 41 | 42 | # 43 | # collector wrapper 44 | # 45 | def collect(service, region) 46 | mapper = Object.const_get(service.name) 47 | resources = mapper.new(@account_id, service.name, region, @options) 48 | 49 | collection = resources.collect.map do |resource| 50 | if @options.output_format == 'custom' 51 | @formatter.custom(@account_id, region, service, resource) 52 | else 53 | @formatter.aws(@account_id, region, service, resource) 54 | end 55 | end 56 | 57 | # write resources to stdout 58 | if @options.stream_output 59 | collection.each do |item| 60 | puts item.to_json 61 | end 62 | end 63 | 64 | # add resources to resources array for output to file 65 | @resources.concat(collection) if @options.output_file 66 | rescue Aws::Errors::ServiceError => e 67 | raise if @options.quit_on_exception 68 | 69 | puts "Ignoring exception: '#{e.message}'\n" 70 | end 71 | 72 | # 73 | # Format @resources as either JSON or JSONL 74 | # 75 | def formatted_json 76 | if @options.jsonl 77 | @resources.map { |r| JSON.generate(r) }.join("\n") 78 | else 79 | @resources.to_json 80 | end 81 | end 82 | 83 | # 84 | # main wrapper 85 | # 86 | def start(_args) 87 | # 88 | # global services 89 | # 90 | @aws_services.map { |x| OpenStruct.new(x) }.filter(&:global).each do |service| 91 | # user included this service in the args 92 | next unless @services.include?(service.name) 93 | 94 | # user did not exclude 'global' 95 | next unless @regions.include?('global') 96 | 97 | collect(service, 'global') 98 | end 99 | 100 | # 101 | # regional services 102 | # 103 | @regions.filter { |x| x != 'global' }.each do |region| 104 | Parallel.map(@aws_services.map { |x| OpenStruct.new(x) }.filter { |s| !s.global }.each, in_threads: @options.threads) do |service| 105 | # some services aren't available in some regions 106 | skip_region = service&.excluded_regions&.include?(region) 107 | 108 | # user included this region in the args 109 | next unless @regions.include?(region) && !skip_region 110 | 111 | # user included this service in the args 112 | next unless @services.include?(service.name) || @services.include?(service.alias) 113 | 114 | collect(service, region) 115 | end 116 | end 117 | rescue Interrupt # ctrl-c 118 | elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @starting 119 | 120 | puts "\nStopped early after #{elapsed.to_i} seconds.\n" 121 | ensure 122 | elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @starting 123 | 124 | puts "\nFinished in #{elapsed.to_i} seconds.\n\n" unless @options.stream_output 125 | 126 | # write output file 127 | if @options.output_file && !@options.s3 128 | puts "Saving resources to #{@options.output_file}.\n\n" 129 | 130 | File.write(@options.output_file, formatted_json) 131 | end 132 | 133 | # write output file to S3 bucket 134 | if @options.s3 135 | t = Time.now.utc 136 | 137 | s3_full_object_path = "AWSRecon/#{t.year}/#{t.month}/#{t.day}/#{@account_id}_aws_recon_#{t.to_i}.json.gz" 138 | 139 | begin 140 | # get bucket name (and region if not us-east-1) 141 | s3_bucket, s3_region = @options.s3.split(':') 142 | 143 | # build IO object and gzip it 144 | io = StringIO.new 145 | gzip_data = Zlib::GzipWriter.new(io) 146 | gzip_data.write(formatted_json) 147 | gzip_data.close 148 | 149 | # send it to S3 150 | s3_client = Aws::S3::Client.new(region: s3_region || 'us-east-1') 151 | s3_resource = Aws::S3::Resource.new(client: s3_client) 152 | obj = s3_resource.bucket(s3_bucket).object(s3_full_object_path) 153 | obj.put(body: io.string) 154 | 155 | puts "Saving resources to S3 s3://#{s3_bucket}/#{s3_full_object_path}\n\n" 156 | rescue Aws::S3::Errors::ServiceError => e 157 | puts "Error! - could not save output S3 bucket\n\n" 158 | puts "#{e.message} - #{e.code}\n" 159 | end 160 | end 161 | end 162 | end 163 | end 164 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # require all collectors 4 | Dir[File.join(__dir__, 'collectors', '*.rb')].sort.each { |file| require file } 5 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/accessanalyzer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect AccessAnalyzer resources 5 | # 6 | class AccessAnalyzer < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_analyzers 15 | # 16 | @client.list_analyzers.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | # analyzers 20 | response.analyzers.each do |analyzer| 21 | struct = OpenStruct.new(analyzer.to_h) 22 | struct.type = 'analyzer' 23 | resources.push(struct.to_h) 24 | end 25 | end 26 | 27 | resources 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/acm.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect ACM resources 5 | # 6 | class ACM < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_certificates 15 | # 16 | @client.list_certificates.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.certificate_summary_list.each do |cert| 20 | log(response.context.operation_name, cert.domain_name, page) 21 | 22 | struct = OpenStruct.new(cert.to_h) 23 | struct.type = 'certificate' 24 | struct.arn = cert.certificate_arn 25 | 26 | # describe_certificate 27 | struct.details = @client 28 | .describe_certificate({ certificate_arn: cert.certificate_arn }) 29 | .certificate.to_h 30 | 31 | resources.push(struct.to_h) 32 | end 33 | end 34 | 35 | resources 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/apigateway.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect API Gateway resources 5 | # 6 | class APIGateway < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # get_rest_apis 15 | # 16 | @client.get_rest_apis.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.items.each do |api| 20 | struct = OpenStruct.new(api.to_h) 21 | struct.type = 'api' 22 | struct.arn = api.id 23 | 24 | # get_authorizers 25 | struct.authorizers = @client.get_authorizers({ rest_api_id: api.id }).items.map(&:to_h) 26 | 27 | # get_stages 28 | struct.stages = @client.get_stages({ rest_api_id: api.id }).item.map(&:to_h) 29 | 30 | # get_models 31 | struct.models = @client.get_models({ rest_api_id: api.id }).items.map(&:to_h) 32 | 33 | # get_resources 34 | struct.resources = @client.get_resources({ rest_api_id: api.id }).items.map(&:to_h) 35 | 36 | resources.push(struct.to_h) 37 | end 38 | end 39 | 40 | # get_domain_names 41 | @client.get_domain_names.each_with_index do |response, page| 42 | log(response.context.operation_name, page) 43 | 44 | response.items.each do |domain| 45 | struct = OpenStruct.new(domain.to_h) 46 | struct.type = 'domain' 47 | struct.arn = domain.domain_name 48 | 49 | resources.push(struct.to_h) 50 | end 51 | end 52 | 53 | resources 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/apigatewayv2.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect API Gateway v2 resources 5 | # 6 | class ApiGatewayV2 < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # get_apis 15 | # 16 | @client.get_apis.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.items.each do |api| 20 | struct = OpenStruct.new(api.to_h) 21 | struct.type = 'api' 22 | struct.arn = api.api_id 23 | 24 | # get_authorizers 25 | struct.authorizers = @client.get_authorizers({ api_id: api.api_id }).items.map(&:to_h) 26 | 27 | # get_stages 28 | struct.stages = @client.get_stages({ api_id: api.api_id }).items.map(&:to_h) 29 | 30 | # get_models 31 | struct.models = @client.get_models({ api_id: api.api_id }).items.map(&:to_h) 32 | 33 | # get_deployments 34 | struct.deployments = @client.get_deployments({ api_id: api.api_id }).items.map(&:to_h) 35 | 36 | resources.push(struct.to_h) 37 | end 38 | end 39 | 40 | resources 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/applicationautoscaling.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect ApplicationAutoScaling resources 5 | # 6 | class ApplicationAutoScaling < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # DynamoDB auto-scaling policies 15 | # 16 | @client.describe_scaling_policies({ service_namespace: 'dynamodb' }).each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.scaling_policies.each do |policy| 20 | struct = OpenStruct.new(policy.to_h) 21 | struct.type = 'auto_scaling_policy' 22 | struct.arn = policy.policy_arn 23 | 24 | resources.push(struct.to_h) 25 | end 26 | end 27 | 28 | resources 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/athena.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Athena resources 5 | # 6 | class Athena < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_work_groups 15 | # 16 | @client.list_work_groups.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.work_groups.each do |workgroup| 20 | struct = OpenStruct.new(workgroup.to_h) 21 | struct.type = 'workgroup' 22 | struct.arn = "arn:aws:athena:#{@region}::workgroup/#{workgroup.name}" 23 | 24 | # get_work_group 25 | struct.details = @client.get_work_group({ work_group: workgroup.name }).to_h 26 | 27 | resources.push(struct.to_h) 28 | end 29 | end 30 | 31 | resources 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/autoscaling.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect AutoScaling resources 5 | # 6 | class AutoScaling < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_auto_scaling_groups 15 | # 16 | @client.describe_auto_scaling_groups.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.auto_scaling_groups.each do |asg| 20 | struct = OpenStruct.new(asg.to_h) 21 | struct.type = 'auto_scaling_group' 22 | struct.arn = asg.auto_scaling_group_arn 23 | struct.policies = [] 24 | 25 | # describe_policies 26 | @client.describe_policies({ auto_scaling_group_name: asg.auto_scaling_group_name }).each_with_index do |response, page| 27 | log(response.context.operation_name, page) 28 | 29 | response.scaling_policies.each do |policy| 30 | struct.policies.push(policy.to_h) 31 | end 32 | end 33 | 34 | resources.push(struct.to_h) 35 | end 36 | end 37 | 38 | resources 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/backup.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Backup resources 5 | # 6 | class Backup < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_backup_plans 15 | # 16 | @client.list_protected_resources.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.results.each do |resource| 20 | struct = OpenStruct.new(resource.to_h) 21 | struct.type = 'protected_resource' 22 | struct.arn = resource.resource_arn 23 | 24 | resources.push(struct.to_h) 25 | end 26 | end 27 | 28 | resources 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/cloudformation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect CloudFormation resources 5 | # 6 | class CloudFormation < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_stacks 15 | # 16 | @client.describe_stacks.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.stacks.each do |stack| 20 | struct = OpenStruct.new(stack.to_h) 21 | struct.type = 'stack' 22 | struct.arn = stack.stack_id 23 | 24 | # get_template 25 | struct.tempate = @client.get_template({ stack_name: stack.stack_name }).to_h 26 | log(response.context.operation_name, 'get_template', stack.stack_name) 27 | 28 | resources.push(struct.to_h) 29 | end 30 | end 31 | 32 | resources 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/cloudfront.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect CloudFront resources 5 | # 6 | class CloudFront < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_distributions 15 | # 16 | @client.list_distributions.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | # get_distribution 20 | response.distribution_list.items.each do |dist| 21 | struct = OpenStruct.new(dist.to_h) 22 | struct.type = 'distribution' 23 | struct.details = @client 24 | .get_distribution({ id: dist.id }) 25 | .distribution.to_h 26 | 27 | resources.push(struct.to_h) 28 | end 29 | end 30 | 31 | resources 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/cloudtrail.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect CloudTrail resources 5 | # 6 | class CloudTrail < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | # 13 | # describe_trails 14 | # 15 | @client.describe_trails.each_with_index do |response, page| 16 | log(response.context.operation_name, page) 17 | 18 | response.trail_list.each do |trail| 19 | # list_tags needs to call into home_region 20 | client = if @region != trail.home_region 21 | Aws::CloudTrail::Client.new({ region: trail.home_region }) 22 | else 23 | @client 24 | end 25 | 26 | struct = OpenStruct.new(trail.to_h) 27 | struct.tags = client.list_tags({ resource_id_list: [trail.trail_arn] }).resource_tag_list.first.tags_list.map(&:to_h) 28 | struct.type = 'cloud_trail' 29 | struct.event_selectors = client.get_event_selectors({ trail_name: trail.name }).to_h 30 | struct.status = client.get_trail_status({ name: trail.name }).to_h 31 | struct.arn = trail.trail_arn 32 | 33 | resources.push(struct.to_h) 34 | end 35 | end 36 | 37 | resources 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/cloudwatch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect CloudWatch resources 5 | # 6 | class CloudWatch < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_alarms 15 | # 16 | @client.describe_alarms.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.composite_alarms.each do |alarm| 20 | struct = OpenStruct.new(alarm.to_h) 21 | struct.type = 'composite_alarm' 22 | struct.arn = alarm.alarm_arn 23 | 24 | resources.push(struct.to_h) 25 | end 26 | 27 | response.metric_alarms.each do |alarm| 28 | struct = OpenStruct.new(alarm.to_h) 29 | struct.type = 'metric_alarm' 30 | struct.arn = alarm.alarm_arn 31 | struct.state_reason_data = alarm.state_reason_data&.parse_policy 32 | 33 | resources.push(struct.to_h) 34 | end 35 | end 36 | 37 | resources 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/cloudwatchlogs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect CloudWatchLogs resources 5 | # 6 | class CloudWatchLogs < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_log_groups 15 | # 16 | @client.describe_log_groups.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.log_groups.each do |log_group| 20 | struct = OpenStruct.new(log_group.to_h) 21 | struct.type = 'log_group' 22 | struct.metric_filters = [] 23 | 24 | # describe_metric_filters 25 | if log_group.metric_filter_count > 0 26 | @client.describe_metric_filters.each_with_index do |response, page| 27 | log(response.context.operation_name, log_group.log_group_name, page) 28 | 29 | response.metric_filters.each do |filter| 30 | struct.metric_filters.push(filter.to_h) 31 | end 32 | end 33 | end 34 | 35 | resources.push(struct.to_h) 36 | end 37 | end 38 | 39 | resources 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/codebuild.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect CodeBuild resources 5 | # 6 | class CodeBuild < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | # TODO: group projects in chucks to minimize batch_get calls 11 | # 12 | def collect 13 | resources = [] 14 | 15 | # 16 | # list_projects 17 | # 18 | @client.list_projects.each_with_index do |response, page| 19 | log(response.context.operation_name, page) 20 | 21 | # batch_get_projects 22 | response.projects.each do |project_name| 23 | @client.batch_get_projects({ names: [project_name] }).projects.each do |project| 24 | struct = OpenStruct.new(project.to_h) 25 | struct.type = 'project' 26 | 27 | resources.push(struct.to_h) 28 | end 29 | end 30 | end 31 | 32 | resources 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/codepipeline.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect CodePipeline resources 5 | # 6 | class CodePipeline < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_pipelines 15 | # 16 | begin 17 | @client.list_pipelines.each_with_index do |response, page| 18 | log(response.context.operation_name, page) 19 | 20 | # get_pipeline 21 | response.pipelines.each do |pipeline| 22 | resp = @client.get_pipeline(name: pipeline.name) 23 | struct = OpenStruct.new(resp.pipeline.to_h) 24 | struct.type = 'pipeline' 25 | struct.arn = resp.metadata.pipeline_arn 26 | 27 | resources.push(struct.to_h) 28 | end 29 | end 30 | rescue Aws::CodePipeline::Errors::ServiceError => e 31 | log_error(e.code) 32 | 33 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 34 | end 35 | 36 | resources 37 | end 38 | 39 | private 40 | 41 | # not an error 42 | def suppressed_errors 43 | %w[ 44 | AccessDeniedException 45 | ] 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/configservice.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Config resources 5 | # 6 | class ConfigService < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_config_rules 15 | # 16 | @client.describe_config_rules.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.config_rules.each do |rule| 20 | struct = OpenStruct.new(rule.to_h) 21 | struct.type = 'rule' 22 | struct.arn = rule.config_rule_arn 23 | 24 | # describe_config_rule_evaluation_status 25 | @client.describe_config_rule_evaluation_status({ config_rule_names: [rule.config_rule_name] }).each do |response| 26 | log(response.context.operation_name, rule.config_rule_name, page) 27 | 28 | response.config_rules_evaluation_status.each do |status| 29 | struct.evaluation_status = status.to_h 30 | end 31 | end 32 | 33 | resources.push(struct.to_h) 34 | end 35 | end 36 | 37 | # 38 | # describe_configuration_recorders 39 | # 40 | @client.describe_configuration_recorders.each_with_index do |response, page| 41 | log(response.context.operation_name, page) 42 | 43 | response.configuration_recorders.each do |recorder| 44 | struct = OpenStruct.new(recorder.to_h) 45 | struct.type = 'configuration_recorder' 46 | struct.arn = "arn:aws:config:#{@region}:#{@account}:configuration_recorder/#{recorder.name}" 47 | 48 | # describe_configuration_recorder_status (only accepts one recorder) 49 | @client.describe_configuration_recorder_status({ configuration_recorder_names: [recorder.name] }).each do |response| 50 | log(response.context.operation_name, recorder.name, page) 51 | 52 | response.configuration_recorders_status.each do |status| 53 | struct.status = status.to_h 54 | end 55 | end 56 | 57 | resources.push(struct.to_h) 58 | end 59 | end 60 | 61 | # 62 | # describe_delivery_channels 63 | # 64 | @client.describe_delivery_channels.each_with_index do |response, page| 65 | log(response.context.operation_name, page) 66 | 67 | response.delivery_channels.each do |channel| 68 | struct = OpenStruct.new(channel.to_h) 69 | struct.type = 'delivery_channel' 70 | struct.arn = "arn:aws:config:#{@region}:delivery_channel/#{channel.name}" 71 | 72 | # describe_delivery_channel_status (only accepts one channel) 73 | @client.describe_delivery_channel_status({ delivery_channel_names: [channel.name] }).each do |response| 74 | response.delivery_channels_status.each do |status| 75 | struct.status = status.to_h 76 | end 77 | end 78 | 79 | resources.push(struct.to_h) 80 | end 81 | end 82 | 83 | resources 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/directconnect.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect DirectConnect resources 5 | # 6 | class DirectConnect < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_connections 15 | # 16 | @client.describe_connections.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.connections.each do |connection| 20 | struct = OpenStruct.new(connection.to_h) 21 | struct.type = 'connection' 22 | struct.arn = "arn:aws:service:#{@service}::connection/#{connection.connection_id}" 23 | 24 | resources.push(struct.to_h) 25 | end 26 | end 27 | 28 | resources 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/directoryservice.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect DirectoryService resources 5 | # 6 | class DirectoryService < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | # TODO: confirm paging behavior 11 | # 12 | def collect 13 | resources = [] 14 | 15 | # 16 | # describe_directories 17 | # 18 | @client.describe_directories.each_with_index do |response, page| 19 | log(response.context.operation_name, page) 20 | 21 | response.directory_descriptions.each do |directory| 22 | struct = OpenStruct.new(directory.to_h) 23 | struct.type = 'directory' 24 | struct.arn = "arn:aws:#{@service}:#{@region}::directory/#{directory.directory_id}" 25 | 26 | resources.push(struct.to_h) 27 | end 28 | end 29 | 30 | resources 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/dms.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect DMS resources 5 | # 6 | class DatabaseMigrationService < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_replication_instances 15 | # 16 | @client.describe_replication_instances.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.replication_instances.each do |instance| 20 | struct = OpenStruct.new(instance.to_h) 21 | struct.type = 'replication_instance' 22 | struct.arn = "arn:aws:#{@service}:#{@region}::replication_instance/#{instance.replication_instance_identifier}" 23 | 24 | resources.push(struct.to_h) 25 | end 26 | end 27 | 28 | resources 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/dynamodb.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect DynamodDB resources 5 | # 6 | class DynamoDB < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_limits 15 | # 16 | @client.describe_limits.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | struct = OpenStruct.new(response) 20 | struct.type = 'limits' 21 | struct.arn = "arn:aws:dynamodb:#{@region}:#{@account}/limits" 22 | 23 | resources.push(struct.to_h) 24 | end 25 | 26 | # 27 | # list_tables 28 | # 29 | @client.list_tables.each_with_index do |response, page| 30 | log(response.context.operation_name, page) 31 | 32 | # describe_table 33 | response.table_names.each do |table_name| 34 | struct = OpenStruct.new(@client.describe_table({ table_name: table_name }).table.to_h) 35 | struct.type = 'table' 36 | struct.arn = struct.table_arn 37 | struct.continuous_backups_description = @client.describe_continuous_backups({ table_name: table_name }).continuous_backups_description.to_h 38 | 39 | resources.push(struct.to_h) 40 | end 41 | end 42 | 43 | resources 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/ec2.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect EC2 resources 5 | # 6 | class EC2 < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | image_params = { 13 | filters: [{ name: 'state', values: ['available'] }], 14 | owners: ['self'] 15 | } 16 | snapshot_params = { 17 | owner_ids: ['self'] 18 | } 19 | 20 | # global calls 21 | if @region == 'global' 22 | 23 | # 24 | # get_account_attributes 25 | # 26 | @client.describe_account_attributes.each do |response| 27 | log(response.context.operation_name) 28 | 29 | struct = OpenStruct.new 30 | struct.attributes = response.account_attributes.map(&:to_h) 31 | struct.type = 'account' 32 | struct.arn = "arn:aws:ec2::#{@account}:attributes/account_attributes" 33 | 34 | resources.push(struct.to_h) 35 | end 36 | end 37 | 38 | # regional calls 39 | if @region != 'global' 40 | # 41 | # get_ebs_encryption_by_default 42 | # 43 | @client.get_ebs_encryption_by_default.each do |response| 44 | log(response.context.operation_name) 45 | 46 | struct = OpenStruct.new(response.to_h) 47 | struct.type = 'ebs_encryption_settings' 48 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:settings/ebs_encryption_settings" 49 | 50 | resources.push(struct.to_h) 51 | end 52 | 53 | # 54 | # describe_instances 55 | # 56 | @client.describe_instances.each_with_index do |response, page| 57 | log(response.context.operation_name, page) 58 | 59 | # reservations 60 | response.reservations.each_with_index do |reservation, rpage| 61 | log(response.context.operation_name, 'reservations', rpage) 62 | 63 | # instances 64 | reservation.instances.each do |instance| 65 | struct = OpenStruct.new(instance.to_h) 66 | struct.type = 'instance' 67 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:instance/#{instance.instance_id}" # no true ARN 68 | struct.reservation_id = reservation.reservation_id 69 | 70 | # collect instance user_data 71 | if @options.collect_user_data 72 | user_data_raw = @client.describe_instance_attribute({ 73 | attribute: 'userData', 74 | instance_id: instance.instance_id 75 | }).user_data.to_h[:value] 76 | 77 | # don't save non-string user_data 78 | if user_data_raw 79 | user_data = Base64.decode64(user_data_raw) 80 | 81 | struct.user_data = user_data if user_data.force_encoding('UTF-8').ascii_only? 82 | end 83 | end 84 | 85 | resources.push(struct.to_h) 86 | end 87 | end 88 | end 89 | 90 | # 91 | # describe_vpcs 92 | # 93 | @client.describe_vpcs.each_with_index do |response, page| 94 | log(response.context.operation_name, page) 95 | 96 | response.vpcs.each do |vpc| 97 | struct = OpenStruct.new(vpc.to_h) 98 | struct.type = 'vpc' 99 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:vpc/#{vpc.vpc_id}" # no true ARN 100 | struct.flow_logs = @client 101 | .describe_flow_logs({ filter: [{ 102 | name: 'resource-id', 103 | values: [vpc.vpc_id] 104 | }] }) 105 | .flow_logs.first.to_h 106 | 107 | resources.push(struct.to_h) 108 | end 109 | end 110 | 111 | # 112 | # describe_security_groups 113 | # 114 | @client.describe_security_groups.each_with_index do |response, page| 115 | log(response.context.operation_name, page) 116 | 117 | response.security_groups.each do |security_group| 118 | struct = OpenStruct.new(security_group.to_h) 119 | struct.type = 'security_group' 120 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:security_group/#{security_group.group_id}" # no true ARN 121 | 122 | resources.push(struct.to_h) 123 | end 124 | end 125 | 126 | # 127 | # describe_network_interfaces 128 | # 129 | @client.describe_network_interfaces.each_with_index do |response, page| 130 | log(response.context.operation_name, page) 131 | 132 | response.network_interfaces.each do |network_interface| 133 | struct = OpenStruct.new(network_interface.to_h) 134 | struct.type = 'network_interface' 135 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:network_interface/#{network_interface.network_interface_id}" # no true ARN 136 | 137 | resources.push(struct.to_h) 138 | end 139 | end 140 | 141 | # 142 | # describe_network_acls 143 | # 144 | @client.describe_network_acls.each_with_index do |response, page| 145 | log(response.context.operation_name, page) 146 | 147 | response.network_acls.each do |network_acl| 148 | struct = OpenStruct.new(network_acl.to_h) 149 | struct.type = 'network_acl' 150 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:network_acl/#{network_acl.network_acl_id}" # no true ARN 151 | 152 | resources.push(struct.to_h) 153 | end 154 | end 155 | 156 | # 157 | # describe_subnets 158 | # 159 | @client.describe_subnets.each_with_index do |response, page| 160 | log(response.context.operation_name, page) 161 | 162 | response.subnets.each do |subnet| 163 | struct = OpenStruct.new(subnet.to_h) 164 | struct.type = 'subnet' 165 | struct.arn = subnet.subnet_arn 166 | 167 | resources.push(struct.to_h) 168 | end 169 | end 170 | 171 | # 172 | # describe_addresses 173 | # 174 | @client.describe_addresses.each_with_index do |response, page| 175 | log(response.context.operation_name, page) 176 | 177 | response.addresses.each do |address| 178 | struct = OpenStruct.new(address.to_h) 179 | struct.type = 'eip_address' 180 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:eip_address/#{address.allocation_id}" # no true ARN 181 | 182 | resources.push(struct.to_h) 183 | end 184 | end 185 | 186 | # 187 | # describe_nat_gateways 188 | # 189 | @client.describe_nat_gateways.each_with_index do |response, page| 190 | log(response.context.operation_name, page) 191 | 192 | response.nat_gateways.each do |gateway| 193 | struct = OpenStruct.new(gateway.to_h) 194 | struct.type = 'nat_gateway' 195 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:nat_gateway/#{gateway.nat_gateway_id}" # no true ARN 196 | 197 | resources.push(struct.to_h) 198 | end 199 | end 200 | 201 | # 202 | # describe_internet_gateways 203 | # 204 | @client.describe_internet_gateways.each_with_index do |response, page| 205 | log(response.context.operation_name, page) 206 | 207 | response.internet_gateways.each do |gateway| 208 | struct = OpenStruct.new(gateway.to_h) 209 | struct.type = 'internet_gateway' 210 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:internet_gateway/#{gateway.internet_gateway_id}" # no true ARN 211 | 212 | resources.push(struct.to_h) 213 | end 214 | end 215 | 216 | # 217 | # describe_route_tables 218 | # 219 | @client.describe_route_tables.each_with_index do |response, page| 220 | log(response.context.operation_name, page) 221 | 222 | response.route_tables.each do |table| 223 | struct = OpenStruct.new(table.to_h) 224 | struct.type = 'route_table' 225 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:route_table/#{table.route_table_id}" # no true ARN 226 | 227 | resources.push(struct.to_h) 228 | end 229 | end 230 | 231 | # 232 | # describe_images 233 | # 234 | @client.describe_images(image_params).each_with_index do |response, page| 235 | log(response.context.operation_name, page) 236 | 237 | response.images.each do |image| 238 | struct = OpenStruct.new(image.to_h) 239 | struct.type = 'image' 240 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:image/#{image.image_id}" # no true ARN 241 | 242 | resources.push(struct.to_h) 243 | end 244 | end 245 | 246 | # 247 | # describe_snapshots 248 | # 249 | @client.describe_snapshots(snapshot_params).each_with_index do |response, page| 250 | log(response.context.operation_name, page) 251 | 252 | response.snapshots.each do |snapshot| 253 | struct = OpenStruct.new(snapshot.to_h) 254 | struct.type = 'snapshot' 255 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:snapshot/#{snapshot.snapshot_id}" # no true ARN 256 | struct.create_volume_permissions = @client.describe_snapshot_attribute({ 257 | attribute: 'createVolumePermission', 258 | snapshot_id: snapshot.snapshot_id 259 | }).create_volume_permissions.map(&:to_h) 260 | 261 | resources.push(struct.to_h) 262 | end 263 | end 264 | 265 | # 266 | # describe_flow_logs 267 | # 268 | @client.describe_flow_logs.each_with_index do |response, page| 269 | log(response.context.operation_name, page) 270 | 271 | response.flow_logs.each do |flow_log| 272 | struct = OpenStruct.new(flow_log.to_h) 273 | struct.type = 'flow_log' 274 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:flow_log/#{flow_log.flow_log_id}" # no true ARN 275 | 276 | resources.push(struct.to_h) 277 | end 278 | end 279 | 280 | # 281 | # describe_volumes 282 | # 283 | @client.describe_volumes.each_with_index do |response, page| 284 | log(response.context.operation_name, page) 285 | 286 | response.volumes.each do |volume| 287 | struct = OpenStruct.new(volume.to_h) 288 | struct.type = 'volume' 289 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:volume/#{volume.volume_id}" # no true ARN 290 | 291 | resources.push(struct.to_h) 292 | end 293 | end 294 | 295 | # 296 | # describe_vpn_gateways 297 | # 298 | @client.describe_vpn_gateways.each_with_index do |response, page| 299 | log(response.context.operation_name, page) 300 | 301 | response.vpn_gateways.each do |gateway| 302 | struct = OpenStruct.new(gateway.to_h) 303 | struct.type = 'vpn_gateway' 304 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:vpn_gateway/#{gateway.vpn_gateway_id}" # no true ARN 305 | 306 | resources.push(struct.to_h) 307 | end 308 | end 309 | 310 | # 311 | # describe_vpc_peering_connections 312 | # 313 | @client.describe_vpc_peering_connections.each_with_index do |response, page| 314 | log(response.context.operation_name, page) 315 | 316 | response.vpc_peering_connections.each do |peer| 317 | struct = OpenStruct.new(peer.to_h) 318 | struct.type = 'peering_connection' 319 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:peering_connection/#{peer.vpc_peering_connection_id}" # no true ARN 320 | 321 | resources.push(struct.to_h) 322 | end 323 | end 324 | 325 | # 326 | # describe_vpc_endpoints 327 | # 328 | @client.describe_vpc_endpoints.each_with_index do |response, page| 329 | log(response.context.operation_name, page) 330 | 331 | response.vpc_endpoints.each do |point| 332 | struct = OpenStruct.new(point.to_h) 333 | struct.type = 'vpc_endpoint' 334 | struct.arn = "arn:aws:ec2:#{@region}:#{@account}:vpc_endpoint/#{point.vpc_endpoint_id}" # no true ARN 335 | 336 | resources.push(struct.to_h) 337 | end 338 | end 339 | 340 | # 341 | # describe_managed_prefix_lists 342 | # 343 | begin 344 | @client.describe_managed_prefix_lists.each_with_index do |response, page| 345 | log(response.context.operation_name, page) 346 | 347 | response.prefix_lists.each do |list| 348 | struct = OpenStruct.new(list.to_h) 349 | struct.type = 'prefix_list' 350 | struct.arn = list.prefix_list_arn 351 | 352 | resources.push(struct.to_h) 353 | end 354 | end 355 | rescue Aws::EC2::Errors::ServiceError => e 356 | log_error(e.code) 357 | 358 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 359 | end 360 | end 361 | 362 | resources 363 | end 364 | 365 | private 366 | 367 | def suppressed_errors 368 | %w[ 369 | InvalidAction 370 | ] 371 | end 372 | end 373 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/ecr.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect ECR resources 5 | # 6 | class ECR < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_repositories 15 | # 16 | @client.describe_repositories.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.repositories.each do |repo| 20 | struct = OpenStruct.new(repo.to_h) 21 | struct.type = 'repository' 22 | struct.arn = repo.repository_arn 23 | struct.policy = @client 24 | .get_repository_policy({ repository_name: repo.repository_name }).policy_text.parse_policy 25 | 26 | struct.images = [] 27 | # 28 | # describe images 29 | # 30 | @client.list_images( {repository_name: repo.repository_name}).image_ids.each_with_index do | image, page | 31 | log(response.context.operation_name, 'list_images', page) 32 | image_hash = image.to_h 33 | # 34 | # describe image scan results 35 | # 36 | result = @client.describe_image_scan_findings({ repository_name: repo.repository_name, image_id: { image_digest: image.image_digest, image_tag: image.image_tag } }) 37 | image_hash["image_scan_status"] = result.image_scan_status.to_h 38 | image_hash["image_scan_findings"] = result.image_scan_findings.to_h 39 | 40 | rescue Aws::ECR::Errors::ScanNotFoundException => e 41 | # No scan result for this image. No action needed 42 | ensure 43 | struct.images << image_hash 44 | end 45 | rescue Aws::ECR::Errors::ServiceError => e 46 | log_error(e.code) 47 | 48 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 49 | ensure 50 | resources.push(struct.to_h) 51 | end 52 | end 53 | 54 | resources 55 | end 56 | 57 | private 58 | 59 | # not an error 60 | def suppressed_errors 61 | %w[ 62 | RepositoryPolicyNotFoundException, 63 | ScanNotFoundException 64 | ] 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/ecrpublic.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect ECRPublic resources 5 | # 6 | class ECRPublic < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_repositories 15 | # 16 | 17 | puts(@client.describe_repositories) 18 | @client.describe_repositories.each_with_index do |response, page| 19 | log(response.context.operation_name, page) 20 | 21 | response.repositories.each do |repo| 22 | struct = OpenStruct.new(repo.to_h) 23 | struct.type = "repository" 24 | struct.arn = repo.repository_arn 25 | struct.policy = @client 26 | .get_repository_policy({ repository_name: repo.repository_name }).policy_text.parse_policy 27 | 28 | struct.images = [] 29 | # 30 | # describe images 31 | # 32 | @client.describe_images({ repository_name: repo.repository_name }).image_details.each_with_index do |image, page| 33 | log(response.context.operation_name, "describe_images", page) 34 | image_hash = image.to_h 35 | struct.images << image_hash 36 | end 37 | rescue Aws::ECR::Errors::ServiceError => e 38 | log_error(e.code) 39 | 40 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 41 | ensure 42 | resources.push(struct.to_h) 43 | end 44 | end 45 | 46 | resources 47 | end 48 | 49 | private 50 | 51 | # not an error 52 | def suppressed_errors 53 | %w[ 54 | RepositoryPolicyNotFoundException, 55 | ScanNotFoundException 56 | ] 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/ecs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect ECS resources 5 | # 6 | class ECS < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_clusters 15 | # 16 | @client.list_clusters.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.cluster_arns.each do |cluster| 20 | struct = OpenStruct.new(@client.describe_clusters({ clusters: [cluster] }).clusters.first.to_h) 21 | struct.type = 'cluster' 22 | struct.arn = cluster 23 | struct.tasks = [] 24 | 25 | # list_tasks 26 | @client.list_tasks({ cluster: cluster }).each_with_index do |response, page| 27 | log(response.context.operation_name, 'list_tasks', page) 28 | 29 | # describe_tasks 30 | response.task_arns.each do |task_arn| 31 | @client.describe_tasks({ cluster: cluster, tasks: [task_arn] }).tasks.each do |task| 32 | struct.tasks.push(task) 33 | end 34 | end 35 | end 36 | 37 | resources.push(struct.to_h) 38 | end 39 | end 40 | 41 | resources 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/efs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect EFS resources 5 | # 6 | class EFS < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_file_systems 15 | # 16 | @client.describe_file_systems.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.file_systems.each do |filesystem| 20 | struct = OpenStruct.new(filesystem.to_h) 21 | struct.type = 'filesystem' 22 | struct.arn = filesystem.file_system_arn 23 | 24 | resources.push(struct.to_h) 25 | end 26 | end 27 | 28 | resources 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/eks.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect EKS resources 5 | # 6 | class EKS < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_clusters 15 | # 16 | @client.list_clusters.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | # describe_cluster 20 | response.clusters.each do |cluster| 21 | struct = OpenStruct.new(@client.describe_cluster({ name: cluster }).cluster.to_h) 22 | struct.type = 'cluster' 23 | struct.nodegroups = [] 24 | 25 | # list_nodegroups 26 | @client.list_nodegroups({ cluster_name: cluster }).each_with_index do |response, page| 27 | log(response.context.operation_name, 'list_nodegroups', page) 28 | 29 | # describe_nodegroup 30 | response.nodegroups.each do |nodegroup| 31 | struct.nodegroups.push(@client.describe_nodegroup({ cluster_name: cluster, nodegroup_name: nodegroup }).nodegroup.to_h) 32 | end 33 | end 34 | 35 | resources.push(struct.to_h) 36 | end 37 | end 38 | 39 | resources 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/elasticache.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect ElastiCache resources 5 | # 6 | class ElastiCache < Mapper 7 | def collect 8 | resources = [] 9 | 10 | # 11 | # describe_cache_clusters 12 | # 13 | @client.describe_cache_clusters.each_with_index do |response, page| 14 | log(response.context.operation_name, page) 15 | 16 | response.cache_clusters.each do |cluster| 17 | struct = OpenStruct.new(cluster.to_h) 18 | struct.type = 'cluster' 19 | struct.arn = cluster.arn 20 | 21 | resources.push(struct.to_h) 22 | end 23 | end 24 | 25 | resources 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/elasticloadbalancing.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect ELB resources 5 | # 6 | class ElasticLoadBalancing < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_load_balancers 15 | # 16 | @client.describe_load_balancers.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.load_balancer_descriptions.each do |elb| 20 | struct = OpenStruct.new(elb.to_h) 21 | struct.type = 'load_balancer' 22 | struct.arn = elb.dns_name 23 | struct.load_balancer_version = 'v1' 24 | 25 | # describe_load_balancer_policies 26 | struct.policies = @client 27 | .describe_load_balancer_policies({ load_balancer_name: elb.load_balancer_name }) 28 | .policy_descriptions.map(&:to_h) 29 | 30 | # describe_load_balancer_attributes 31 | struct.attributes = @client 32 | .describe_load_balancer_attributes({ load_balancer_name: elb.load_balancer_name }) 33 | .load_balancer_attributes.to_h 34 | 35 | # describe_tags 36 | struct.tags = @client 37 | .describe_tags({ load_balancer_names: [elb.load_balancer_name] }) 38 | .tag_descriptions.map(&:tags) 39 | .flatten.map(&:to_h) 40 | 41 | resources.push(struct.to_h) 42 | end 43 | end 44 | 45 | resources 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/elasticloadbalancingv2.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect ELBv2 resources 5 | # 6 | class ElasticLoadBalancingV2 < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_load_balancers 15 | # 16 | @client.describe_load_balancers.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.load_balancers.each do |elb| 20 | struct = OpenStruct.new(elb.to_h) 21 | struct.type = 'load_balancer' 22 | struct.arn = elb.load_balancer_arn 23 | struct.load_balancer_version = 'v2' 24 | struct.load_balancer_type = elb.type 25 | struct.listeners = [] 26 | struct.target_groups = [] 27 | 28 | # describe_load_balancer_attributes 29 | struct.attributes = @client 30 | .describe_load_balancer_attributes({ load_balancer_arn: elb.load_balancer_arn }) 31 | .attributes.map(&:to_h) 32 | 33 | # describe_tags 34 | struct.tags = @client 35 | .describe_tags({ resource_arns: [elb.load_balancer_arn] }) 36 | .tag_descriptions.map(&:tags) 37 | .flatten.map(&:to_h) 38 | 39 | # describe_listeners 40 | @client.describe_listeners({ load_balancer_arn: elb.load_balancer_arn }).each_with_index do |response, _page| 41 | log(response.context.operation_name, page) 42 | 43 | response.listeners.each do |listener| 44 | struct.listeners.push(listener.to_h) 45 | end 46 | end 47 | 48 | # describe_target_groups 49 | @client.describe_target_groups({ load_balancer_arn: elb.load_balancer_arn }).each_with_index do |response, page| 50 | log(response.context.operation_name, page) 51 | 52 | response.target_groups.each do |target_group| 53 | tg = OpenStruct.new(target_group.to_h) 54 | 55 | # describe_target_health 56 | tg.health_descriptions = @client 57 | .describe_target_health({ target_group_arn: target_group.target_group_arn }) 58 | .target_health_descriptions.map(&:to_h) 59 | 60 | struct.target_groups.push(tg.to_h) 61 | end 62 | end 63 | 64 | resources.push(struct.to_h) 65 | end 66 | end 67 | 68 | resources 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/elasticsearch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect ElasticSearch resources 5 | # 6 | class ElasticsearchService < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_domain_names 15 | # 16 | @client.list_domain_names.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.domain_names.each do |domain| 20 | log(response.context.operation_name, 'describe_elasticsearch_domain', page) 21 | 22 | # describe_elasticsearch_domains 23 | struct = OpenStruct.new(@client.describe_elasticsearch_domain({ domain_name: domain.domain_name }).domain_status.to_h) 24 | struct.type = 'domain' 25 | struct.access_policies = struct.access_policies&.parse_policy 26 | 27 | resources.push(struct.to_h) 28 | end 29 | end 30 | 31 | resources 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/emr.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect EMR resources 5 | # 6 | class EMR < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # get_block_public_access_configuration 15 | # 16 | begin 17 | @client.get_block_public_access_configuration.each do |response| 18 | log(response.context.operation_name) 19 | 20 | struct = OpenStruct.new(response.block_public_access_configuration.to_h) 21 | struct.type = 'configuration' 22 | struct.arn = "arn:aws:emr:#{@region}:#{@account}/block_public_access_configuration" 23 | 24 | resources.push(struct.to_h) 25 | end 26 | rescue Aws::EMR::Errors::ServiceError => e 27 | log_error(e.code) 28 | 29 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 30 | end 31 | 32 | # 33 | # list_clusters 34 | # 35 | @client.list_clusters.each_with_index do |response, page| 36 | log(response.context.operation_name, page) 37 | 38 | response.clusters.each do |cluster| 39 | log(response.context.operation_name, cluster.id) 40 | 41 | struct = OpenStruct.new(@client.describe_cluster({ cluster_id: cluster.id }).cluster.to_h) 42 | struct.type = 'cluster' 43 | struct.arn = cluster.cluster_arn 44 | 45 | resources.push(struct.to_h) 46 | end 47 | end 48 | 49 | # 50 | # list_security_configurations 51 | # 52 | @client.list_security_configurations.each_with_index do |response, page| 53 | log(response.context.operation_name, page) 54 | 55 | response.security_configurations.each do |security_configuration| 56 | log(response.context.operation_name, security_configuration.name) 57 | 58 | struct = OpenStruct.new(@client.describe_security_configuration({ name: security_configuration.name }).security_configuration.parse_policy) 59 | struct.type = 'security_configuration' 60 | struct.arn = "arn:aws:emr:#{@region}:#{@account}:security-configuration/#{security_configuration.name}" # no true ARN 61 | resources.push(struct.to_h) 62 | end 63 | end 64 | 65 | resources 66 | end 67 | 68 | private 69 | 70 | def suppressed_errors 71 | %w[ 72 | InvalidRequestException 73 | ] 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/firehose.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Firehose resources 5 | # 6 | class Firehose < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | # TODO: test live 11 | # TODO: confirm paging behavior 12 | # 13 | def collect 14 | resources = [] 15 | 16 | # 17 | # list_delivery_streams 18 | # 19 | @client.list_delivery_streams.each_with_index do |response, page| 20 | log(response.context.operation_name, page) 21 | 22 | # describe_delivery_stream 23 | response.delivery_stream_names.each do |stream| 24 | struct = OpenStruct.new(@client.describe_delivery_stream({ delivery_stream_name: stream }).delivery_stream_description.to_h) 25 | struct.type = 'stream' 26 | struct.arn = struct.delivery_stream_arn 27 | 28 | resources.push(struct.to_h) 29 | end 30 | end 31 | 32 | resources 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/glue.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Glue resources 5 | # 6 | class Glue < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | # 13 | # get_data_catalog_encryption_settings 14 | # 15 | @client.get_data_catalog_encryption_settings.each_with_index do |response, page| 16 | log(response.context.operation_name, page) 17 | 18 | struct = OpenStruct.new(response.to_h) 19 | struct.type = 'catalog_encryption_settings' 20 | struct.arn = "arn:aws:glue:#{@region}:#{@account}:data-catalog-encryption-settings" # no true ARN 21 | resources.push(struct.to_h) 22 | end 23 | 24 | # 25 | # get_security_configurations 26 | # 27 | @client.get_security_configurations.each_with_index do |response, page| 28 | log(response.context.operation_name, page) 29 | 30 | response.security_configurations.each do |security_configuration| 31 | struct = OpenStruct.new(security_configuration.to_h) 32 | struct.type = 'security_configuration' 33 | struct.arn = "arn:aws:glue:#{@region}:#{@account}:security-configuration/#{security_configuration.name}" # no true ARN 34 | resources.push(struct.to_h) 35 | end 36 | end 37 | 38 | # 39 | # get_databases 40 | # 41 | @client.get_databases.each_with_index do |response, page| 42 | log(response.context.operation_name, page) 43 | 44 | response.database_list.each do |database| 45 | struct = OpenStruct.new(database.to_h) 46 | struct.type = 'database' 47 | struct.arn = "arn:aws:glue:#{@region}:#{@account}:database/#{database.name}" 48 | 49 | # 50 | # get_tables 51 | # 52 | tables = @client.get_tables({ database_name: database.name }) 53 | struct.tables = tables.to_h 54 | 55 | resources.push(struct.to_h) 56 | end 57 | end 58 | 59 | # 60 | # get_jobs 61 | # 62 | @client.get_jobs.each_with_index do |response, page| 63 | log(response.context.operation_name, page) 64 | 65 | response.jobs.each do |job| 66 | struct = OpenStruct.new(job.to_h) 67 | struct.type = 'job' 68 | struct.arn = "arn:aws:glue:#{@region}:#{@account}:job/#{job.name}" 69 | resources.push(struct.to_h) 70 | end 71 | end 72 | 73 | # 74 | # get_dev_endpoints 75 | # 76 | @client.get_dev_endpoints.each_with_index do |response, page| 77 | log(response.context.operation_name, page) 78 | 79 | response.dev_endpoints.each do |dev_endpoint| 80 | struct = OpenStruct.new(dev_endpoint.to_h) 81 | struct.type = 'dev_endpoint' 82 | struct.arn = "arn:aws:glue:#{@region}:#{@account}:devEndpoint/#{dev_endpoint.endpoint_name}" 83 | resources.push(struct.to_h) 84 | end 85 | end 86 | 87 | # 88 | # get_crawlers 89 | # 90 | @client.get_crawlers.each_with_index do |response, page| 91 | log(response.context.operation_name, page) 92 | 93 | response.crawlers.each do |crawler| 94 | struct = OpenStruct.new(crawler.to_h) 95 | struct.type = 'crawler' 96 | struct.arn = "arn:aws:glue:#{@region}:#{@account}:crawler/#{crawler.name}" 97 | resources.push(struct.to_h) 98 | end 99 | end 100 | 101 | # 102 | # get_connections 103 | # 104 | @client.get_connections.each_with_index do |response, page| 105 | log(response.context.operation_name, page) 106 | 107 | response.connection_list.each do |connection| 108 | struct = OpenStruct.new(connection.to_h) 109 | struct.type = 'connection' 110 | struct.arn = "arn:aws:glue:#{@region}:#{@account}:connection/#{connection.name}" 111 | resources.push(struct.to_h) 112 | end 113 | end 114 | resources 115 | end 116 | end 117 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/guardduty.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect GuardDuty resources 5 | # 6 | class GuardDuty < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_detectors 15 | # 16 | @client.list_detectors.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.detector_ids.each do |detector| 20 | log(response.context.operation_name, 'get_detector', detector) 21 | 22 | # get_detector 23 | struct = OpenStruct.new(@client.get_detector({ detector_id: detector }).to_h) 24 | struct.type = 'detector' 25 | struct.arn = "arn:aws:guardduty:#{@region}:#{@account}:detector/#{detector}" 26 | 27 | # get_findings_statistics (only active findings) 28 | struct.findings_statistics = @client.get_findings_statistics({ 29 | detector_id: detector, 30 | finding_statistic_types: ['COUNT_BY_SEVERITY'], 31 | finding_criteria: finding_criteria 32 | }).finding_statistics.to_h 33 | # get_findings_statistics (only active findings older than 7 days) 34 | struct.findings_statistics_aged_short = @client.get_findings_statistics({ 35 | detector_id: detector, 36 | finding_statistic_types: ['COUNT_BY_SEVERITY'], 37 | finding_criteria: finding_criteria(7) 38 | }).finding_statistics.to_h 39 | # get_findings_statistics (only active findings older than 30 days) 40 | struct.findings_statistics_aged_long = @client.get_findings_statistics({ 41 | detector_id: detector, 42 | finding_statistic_types: ['COUNT_BY_SEVERITY'], 43 | finding_criteria: finding_criteria(30) 44 | }).finding_statistics.to_h 45 | 46 | # get_master_account 47 | struct.master_account = @client.get_master_account({ detector_id: detector }).master.to_h 48 | 49 | resources.push(struct.to_h) 50 | end 51 | end 52 | 53 | resources 54 | end 55 | 56 | private 57 | 58 | def finding_criteria(days = 1) 59 | criteria = { 60 | criterion: { 61 | 'service.archived': { eq: ['false'] } 62 | } 63 | } 64 | 65 | if days > 1 66 | days_ago = (Time.now.to_f * 1000).to_i - (60 * 60 * 24 * 1000 * days) # with miliseconds 67 | 68 | criteria = { 69 | criterion: { 70 | 'service.archived': { eq: ['false'] }, 71 | 'updatedAt': { less_than: days_ago } 72 | } 73 | } 74 | end 75 | 76 | criteria 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/iam.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect IAM resources 5 | # 6 | class IAM < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # get_account_authorization_details 15 | # list_mfa_devices 16 | # list_ssh_public_keys 17 | # 18 | opts = { 19 | filter: %w[User Role Group LocalManagedPolicy AWSManagedPolicy] 20 | } 21 | @client.get_account_authorization_details(opts).each_with_index do |response, page| 22 | log(response.context.operation_name, page) 23 | 24 | # users 25 | response.user_detail_list.each do |user| 26 | struct = OpenStruct.new(user.to_h) 27 | struct.type = 'user' 28 | struct.mfa_devices = @client.list_mfa_devices({ user_name: user.user_name }).mfa_devices.map(&:to_h) 29 | struct.ssh_keys = @client.list_ssh_public_keys({ user_name: user.user_name }).ssh_public_keys.map(&:to_h) 30 | struct.user_policy_list = if user.user_policy_list 31 | user.user_policy_list.map do |p| 32 | { 33 | policy_name: p.policy_name, 34 | policy_document: p.policy_document.parse_policy 35 | } 36 | end 37 | end 38 | 39 | resources.push(struct.to_h) 40 | end 41 | 42 | # groups 43 | response.group_detail_list.each do |group| 44 | struct = OpenStruct.new(group.to_h) 45 | struct.type = 'group' 46 | struct.group_policy_list = if group.group_policy_list 47 | group.group_policy_list.map do |p| 48 | { 49 | policy_name: p.policy_name, 50 | policy_document: p.policy_document.parse_policy 51 | } 52 | end 53 | end 54 | 55 | resources.push(struct.to_h) 56 | end 57 | 58 | # roles 59 | response.role_detail_list.each do |role| 60 | struct = OpenStruct.new(role.to_h) 61 | struct.type = 'role' 62 | struct.assume_role_policy_document = role.assume_role_policy_document.parse_policy 63 | struct.role_policy_list = if role.role_policy_list 64 | role.role_policy_list.map do |p| 65 | { 66 | policy_name: p.policy_name, 67 | policy_document: p.policy_document.parse_policy 68 | } 69 | end 70 | end 71 | 72 | resources.push(struct.to_h) 73 | end 74 | 75 | # polices 76 | response.policies.each do |policy| 77 | struct = OpenStruct.new(policy.to_h) 78 | struct.type = 'policy' 79 | struct.policy_version_list = if policy.policy_version_list 80 | policy.policy_version_list.map do |p| 81 | { 82 | version_id: p.version_id, 83 | document: p.document.parse_policy, 84 | is_default_version: p.is_default_version, 85 | create_date: p.create_date 86 | } 87 | end 88 | end 89 | 90 | resources.push(struct.to_h) 91 | end 92 | end 93 | 94 | # 95 | # list_instance_profiles 96 | # 97 | @client.list_instance_profiles.each_with_index do |response, page| 98 | log(response.context.operation_name, page) 99 | 100 | # instance_profiles 101 | response.instance_profiles.each do |profile| 102 | struct = OpenStruct.new(profile.to_h) 103 | struct.type = 'instance_profile' 104 | struct.arn = profile.arn 105 | struct.roles = [] 106 | 107 | profile.roles&.each do |role| 108 | role.assume_role_policy_document = role.assume_role_policy_document.parse_policy 109 | struct.roles.push(role.to_h) 110 | end 111 | 112 | resources.push(struct.to_h) 113 | end 114 | end 115 | 116 | # 117 | # get_account_password_policy 118 | # 119 | begin 120 | @client.get_account_password_policy.each do |response| 121 | log(response.context.operation_name) 122 | 123 | struct = OpenStruct.new(response.password_policy.to_h) 124 | struct.type = 'password_policy' 125 | struct.arn = "arn:aws:iam::#{@account}:account_password_policy/global" 126 | 127 | resources.push(struct.to_h) 128 | end 129 | rescue Aws::IAM::Errors::ServiceError => e 130 | log_error(e.code) 131 | 132 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 133 | end 134 | 135 | # 136 | # get_account_summary 137 | # 138 | @client.get_account_summary.each do |response| 139 | log(response.context.operation_name) 140 | 141 | struct = OpenStruct.new(response.summary_map) 142 | struct.type = 'account_summary' 143 | struct.arn = "arn:aws:iam::#{@account}:account_summary/global" 144 | 145 | resources.push(struct.to_h) 146 | end 147 | 148 | # 149 | # list_server_certificates 150 | # 151 | @client.list_server_certificates.each_with_index do |response, page| 152 | log(response.context.operation_name, page) 153 | 154 | response.server_certificate_metadata_list.each do |cert| 155 | struct = OpenStruct.new(cert) 156 | struct.type = 'server_certificate' 157 | struct.arn = cert.arn 158 | 159 | resources.push(struct.to_h) 160 | end 161 | end 162 | 163 | # 164 | # list_virtual_mfa_devices 165 | # 166 | @client.list_virtual_mfa_devices.each_with_index do |response, page| 167 | log(response.context.operation_name, page) 168 | 169 | response.virtual_mfa_devices.each do |mfa_device| 170 | struct = OpenStruct.new(mfa_device.to_h) 171 | struct.type = 'virtual_mfa_device' 172 | struct.arn = mfa_device.serial_number 173 | 174 | resources.push(struct.to_h) 175 | end 176 | end 177 | 178 | # 179 | # generate_credential_report 180 | # 181 | unless @options.skip_credential_report 182 | status = 'STARTED' 183 | interval = 5 184 | 185 | # wait for report to generate 186 | while status != 'COMPLETE' 187 | @client.generate_credential_report.each do |response| 188 | log(response.context.operation_name) 189 | status = response.state 190 | end 191 | 192 | sleep interval unless status == 'COMPLETE' 193 | end 194 | end 195 | 196 | # 197 | # get_credential_report 198 | # 199 | begin 200 | @client.get_credential_report.each do |response| 201 | log(response.context.operation_name) 202 | 203 | struct = OpenStruct.new 204 | struct.type = 'credential_report' 205 | struct.arn = "arn:aws:iam::#{@account}:credential_report/global" 206 | struct.content = CSV.parse(response.content, headers: :first_row).map(&:to_h) 207 | struct.report_format = response.report_format 208 | struct.generated_time = response.generated_time 209 | 210 | resources.push(struct.to_h) 211 | end 212 | rescue Aws::IAM::Errors::ServiceError => e 213 | log_error(e.code) 214 | 215 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 216 | end 217 | 218 | resources 219 | end 220 | 221 | private 222 | 223 | # not an error 224 | def suppressed_errors 225 | %w[ 226 | ReportNotPresent 227 | NoSuchEntity 228 | ] 229 | end 230 | end 231 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/kafka.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Kafka resources 5 | # 6 | class Kafka < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | # TODO: test live 11 | # 12 | def collect 13 | resources = [] 14 | 15 | # 16 | # list_clusters 17 | # 18 | @client.list_clusters.each_with_index do |response, page| 19 | log(response.context.operation_name, page) 20 | 21 | response.cluster_info_list.each do |cluster| 22 | struct = OpenStruct.new(cluster.to_h) 23 | struct.type = 'cluster' 24 | struct.arn = cluster.cluster_arn 25 | 26 | resources.push(struct.to_h) 27 | end 28 | end 29 | 30 | resources 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/kinesis.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Kinesis resources 5 | # 6 | class Kinesis < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_streams 15 | # 16 | @client.list_streams.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | # describe_stream 20 | response.stream_names.each do |stream| 21 | struct = OpenStruct.new(@client.describe_stream({ stream_name: stream }).stream_description.to_h) 22 | struct.type = 'stream' 23 | struct.arn = struct.stream_arn 24 | 25 | resources.push(struct.to_h) 26 | end 27 | end 28 | 29 | resources 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/kms.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect KMS resources 5 | # 6 | class KMS < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_keys 15 | # 16 | @client.list_keys.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | # describe_key 20 | response.keys.each do |key| 21 | log(response.context.operation_name, 'describe_key', page) 22 | struct = OpenStruct.new(@client 23 | .describe_key({ key_id: key.key_id }) 24 | .key_metadata.to_h) 25 | struct.type = 'key' 26 | struct.grants = [] 27 | 28 | # get_key_rotation_status 29 | log(response.context.operation_name, 'get_key_rotation_status') 30 | # The default master key rotation status can't be queried 31 | begin 32 | struct.rotation_enabled = @client 33 | .get_key_rotation_status({ key_id: key.key_id }) 34 | .key_rotation_enabled 35 | rescue Aws::KMS::Errors::ServiceError => e 36 | log_error(e.code) 37 | 38 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 39 | end 40 | 41 | # list_grants 42 | @client.list_grants({ key_id: key.key_id }).each_with_index do |response, page| 43 | log(response.context.operation_name, 'list_grants', page) 44 | 45 | response.grants.each do |grant| 46 | struct.grants.push(grant.to_h) 47 | end 48 | end 49 | 50 | # get_key_policy - 'default' is the only valid policy 51 | log(response.context.operation_name, 'get_key_policy') 52 | struct.policy = @client 53 | .get_key_policy({ key_id: key.key_id, policy_name: 'default' }) 54 | .policy.parse_policy 55 | 56 | # list_aliases 57 | log(response.context.operation_name, 'list_aliases') 58 | struct.aliases = @client 59 | .list_aliases({ key_id: key.key_id }) 60 | .aliases.map(&:to_h) 61 | 62 | resources.push(struct.to_h) 63 | end 64 | end 65 | 66 | resources 67 | end 68 | 69 | private 70 | 71 | # not an error 72 | def suppressed_errors 73 | %w[ 74 | AccessDeniedException 75 | ] 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/lambda.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Lambda resources 5 | # 6 | class Lambda < Mapper 7 | def collect 8 | resources = [] 9 | 10 | # 11 | # list_functions 12 | # 13 | @client.list_functions.each_with_index do |response, page| 14 | log(response.context.operation_name, page) 15 | 16 | response.functions.each do |function| 17 | struct = OpenStruct.new(function) 18 | struct.type = 'function' 19 | struct.arn = function.function_arn 20 | struct.vpc_config = function.vpc_config.to_h 21 | struct.tracing_config = function.tracing_config.to_h 22 | struct.layers = function.layers ? function.layers.map(&:to_h) : [] 23 | struct.policy = @client.get_policy({ function_name: function.function_name }).policy.parse_policy 24 | 25 | rescue Aws::Lambda::Errors::ResourceNotFoundException => e 26 | log_error(e.code) 27 | ensure 28 | resources.push(struct.to_h) 29 | end 30 | end 31 | 32 | # 33 | # list_layers 34 | # 35 | @client.list_layers.each_with_index do |response, page| 36 | log(response.context.operation_name, page) 37 | 38 | response.layers.each do |layer| 39 | struct = OpenStruct.new(layer) 40 | struct.type = 'layer' 41 | struct.arn = layer.layer_arn 42 | struct.latest_matching_version = layer.latest_matching_version.to_h 43 | 44 | # list_layer_versions 45 | struct.versions = @client.list_layer_versions({ layer_name: layer.layer_name }).layer_versions.map(&:to_h) 46 | log(response.context.operation_name, 'list_layer_versions') 47 | 48 | resources.push(struct.to_h) 49 | end 50 | end 51 | 52 | resources 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/lightsail.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Lightsail resources 5 | # 6 | class Lightsail < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # get_instances 15 | # 16 | @client.get_instances.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.instances.each do |instance| 20 | struct = OpenStruct.new(instance.to_h) 21 | struct.type = 'instance' 22 | 23 | resources.push(struct.to_h) 24 | end 25 | end 26 | 27 | # 28 | # get_load_balancers 29 | # 30 | @client.get_load_balancers.each_with_index do |response, page| 31 | log(response.context.operation_name, page) 32 | 33 | response.load_balancers.each do |load_balancer| 34 | struct = OpenStruct.new(load_balancer.to_h) 35 | struct.type = 'load_balancer' 36 | 37 | resources.push(struct.to_h) 38 | end 39 | end 40 | 41 | resources 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/organizations.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Org resources 5 | # 6 | class Organizations < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_organization 15 | # 16 | begin 17 | @client.describe_organization.each do |response| 18 | log(response.context.operation_name) 19 | 20 | struct = OpenStruct.new(response.organization.to_h) 21 | struct.type = 'organization' 22 | 23 | resources.push(struct.to_h) 24 | end 25 | rescue Aws::Organizations::Errors::ServiceError => e 26 | log_error(e.code) 27 | 28 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 29 | end 30 | 31 | # 32 | # list_handshakes_for_account 33 | # 34 | @client.list_handshakes_for_account.each_with_index do |response, page| 35 | log(response.context.operation_name, page) 36 | 37 | response.handshakes.each do |handshake| 38 | struct = OpenStruct.new(handshake.to_h) 39 | struct.type = 'handshake' 40 | 41 | resources.push(struct.to_h) 42 | end 43 | end 44 | 45 | # 46 | # list_policies 47 | # 48 | begin 49 | @client.list_policies({ filter: 'SERVICE_CONTROL_POLICY' }).each_with_index do |response, page| 50 | log(response.context.operation_name, page) 51 | 52 | response.policies.each do |policy| 53 | struct = OpenStruct.new(policy.to_h) 54 | struct.type = 'service_control_policy' 55 | struct.content = @client.describe_policy({ policy_id: policy.id }).policy.content.parse_policy 56 | 57 | resources.push(struct.to_h) 58 | end 59 | end 60 | rescue Aws::Organizations::Errors::ServiceError => e 61 | log_error(e.code) 62 | 63 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 64 | end 65 | 66 | resources 67 | end 68 | 69 | private 70 | 71 | # not an error 72 | def suppressed_errors 73 | %w[ 74 | AccessDeniedException 75 | AWSOrganizationsNotInUseException 76 | ] 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/rds.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect RDS Resources 5 | # 6 | class RDS < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | # describe_db_engine_versions is skipped with @options.skip_slow 11 | # 12 | def collect 13 | resources = [] 14 | 15 | # 16 | # describe_db_clusters 17 | # 18 | @client.describe_db_clusters.each_with_index do |response, page| 19 | log(response.context.operation_name, page) 20 | 21 | response.db_clusters.each do |cluster| 22 | log(response.context.operation_name, cluster.db_cluster_identifier) 23 | 24 | struct = OpenStruct.new(cluster.to_h) 25 | struct.type = 'db_cluster' 26 | struct.arn = cluster.db_cluster_arn 27 | 28 | resources.push(struct.to_h) 29 | end 30 | end 31 | 32 | # 33 | # describe_db_instances 34 | # 35 | @client.describe_db_instances.each_with_index do |response, page| 36 | log(response.context.operation_name, page) 37 | 38 | response.db_instances.each do |instance| 39 | log(response.context.operation_name, instance.db_instance_identifier) 40 | 41 | struct = OpenStruct.new(instance.to_h) 42 | struct.type = 'db_instance' 43 | struct.arn = instance.db_instance_arn 44 | struct.parent_id = instance.db_cluster_identifier 45 | 46 | # TODO: describe_db_snapshots here (with public flag) 47 | 48 | resources.push(struct.to_h) 49 | end 50 | end 51 | 52 | # 53 | # describe_db_snapshots 54 | # 55 | @client.describe_db_snapshots.each_with_index do |response, page| 56 | log(response.context.operation_name, page) 57 | 58 | response.db_snapshots.each do |snapshot| 59 | log(response.context.operation_name, snapshot.db_snapshot_identifier) 60 | 61 | struct = OpenStruct.new(snapshot.to_h) 62 | struct.type = 'db_snapshot' 63 | struct.arn = snapshot.db_snapshot_arn 64 | struct.parent_id = snapshot.db_instance_identifier 65 | 66 | resources.push(struct.to_h) 67 | end 68 | end 69 | 70 | # 71 | # describe_db_cluster_snapshots 72 | # 73 | @client.describe_db_cluster_snapshots.each_with_index do |response, page| 74 | log(response.context.operation_name, page) 75 | 76 | response.db_cluster_snapshots.each do |snapshot| 77 | log(response.context.operation_name, snapshot.db_cluster_snapshot_identifier) 78 | 79 | struct = OpenStruct.new(snapshot.to_h) 80 | struct.type = 'db_cluster_snapshot' 81 | struct.arn = snapshot.db_cluster_snapshot_arn 82 | struct.parent_id = snapshot.db_cluster_identifier 83 | 84 | resources.push(struct.to_h) 85 | end 86 | end 87 | 88 | # 89 | # describe_db_engine_versions 90 | # 91 | ### unless @options.skip_slow 92 | ### @client.describe_db_engine_versions.each_with_index do |response, page| 93 | ### log(response.context.operation_name, page) 94 | 95 | ### response.db_engine_versions.each do |version| 96 | ### struct = OpenStruct.new(version.to_h) 97 | ### struct.type = 'db_engine_version' 98 | 99 | ### resources.push(struct.to_h) 100 | ### end 101 | ### end 102 | ### end 103 | 104 | resources 105 | end 106 | end 107 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/redshift.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Redshift resources 5 | # 6 | class Redshift < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_clusters 15 | # 16 | @client.describe_clusters.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.clusters.each do |cluster| 20 | struct = OpenStruct.new(cluster.to_h) 21 | struct.type = 'cluster' 22 | struct.arn = cluster.cluster_identifier 23 | struct.logging_status = @client.describe_logging_status({ cluster_identifier: cluster.cluster_identifier }).to_h 24 | 25 | resources.push(struct.to_h) 26 | end 27 | end 28 | 29 | # 30 | # describe_cluster_subnet_groups 31 | # 32 | @client.describe_cluster_subnet_groups.each_with_index do |response, page| 33 | log(response.context.operation_name, page) 34 | 35 | response.cluster_subnet_groups.each do |group| 36 | struct = OpenStruct.new(group.to_h) 37 | struct.type = 'subnet_group' 38 | struct.arn = group.cluster_subnet_group_name 39 | 40 | resources.push(struct.to_h) 41 | end 42 | end 43 | 44 | resources 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/route53.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Route53 resources 5 | # 6 | class Route53 < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_hosted_zones 15 | # 16 | @client.list_hosted_zones.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.hosted_zones.each do |zone| 20 | struct = OpenStruct.new(zone.to_h) 21 | struct.type = 'zone' 22 | struct.arn = "arn:aws:route53:#{@region}:#{@account}:zone/#{zone.name}" 23 | struct.logging_config = @client 24 | .list_query_logging_configs({ hosted_zone_id: zone.id }) 25 | .query_logging_configs.first.to_h 26 | 27 | resources.push(struct.to_h) 28 | end 29 | end 30 | 31 | resources 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/route53domains.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Route53 Domain resources 5 | # 6 | class Route53Domains < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_domains 15 | # 16 | @client.list_domains.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.domains.each do |domain| 20 | struct = OpenStruct.new(domain.to_h) 21 | struct.type = 'domain' 22 | struct.arn = "arn:aws:#{@service}:#{@region}::domain/#{domain.domain_name}" 23 | 24 | resources.push(struct.to_h) 25 | end 26 | end 27 | 28 | resources 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/s3.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect S3 Resources 5 | # 6 | class S3 < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | # Since S3 is a global service, the bucket operation calls 11 | # can be parallelized. 12 | # 13 | def collect 14 | resources = [] 15 | 16 | # 17 | # list_buckets 18 | # 19 | @client.list_buckets.each_with_index do |response, page| 20 | log(response.context.operation_name, page) 21 | 22 | Parallel.map(response.buckets.each, in_threads: @options.threads) do |bucket| 23 | @thread = Parallel.worker_number 24 | log(response.context.operation_name, bucket.name) 25 | 26 | struct = OpenStruct.new(bucket) 27 | struct.type = 'bucket' 28 | struct.arn = "arn:aws:s3:::#{bucket.name}" 29 | 30 | # check bucket region constraint 31 | location = @client.get_bucket_location({ bucket: bucket.name }).location_constraint 32 | 33 | # if you use a region other than the us-east-1 endpoint 34 | # to create a bucket, you must set the location_constraint 35 | # bucket parameter to the same region. (https://docs.aws.amazon.com/general/latest/gr/s3.html) 36 | client = if location.empty? 37 | struct.location = 'us-east-1' 38 | @client 39 | else 40 | location = 'eu-west-1' if location == 'EU' 41 | struct.location = location 42 | Aws::S3::Client.new({ region: location }) 43 | end 44 | 45 | operations = [ 46 | { func: 'get_bucket_acl', key: 'acl', field: nil }, 47 | { func: 'get_bucket_encryption', key: 'encryption', field: 'server_side_encryption_configuration' }, 48 | { func: 'get_bucket_replication', key: 'replication', field: 'replication_configuration' }, 49 | { func: 'get_bucket_policy', key: 'policy', field: 'policy' }, 50 | { func: 'get_bucket_policy_status', key: 'public', field: 'policy_status' }, 51 | { func: 'get_public_access_block', key: 'public_access_block', field: 'public_access_block_configuration' }, 52 | { func: 'get_object_lock_configuration', key: 'object_lock_configuration', field: 'object_lock_configuration' }, 53 | { func: 'get_bucket_tagging', key: 'tagging', field: nil }, 54 | { func: 'get_bucket_logging', key: 'logging', field: 'logging_enabled' }, 55 | { func: 'get_bucket_versioning', key: 'versioning', field: nil }, 56 | { func: 'get_bucket_website', key: 'website', field: nil }, 57 | { func: 'get_bucket_ownership_controls', key: 'ownership_controls', field: 'ownership_controls' } 58 | ] 59 | 60 | operations.each do |operation| 61 | op = OpenStruct.new(operation) 62 | 63 | resp = client.send(op.func, { bucket: bucket.name }) 64 | 65 | struct[op.key] = if op.key == 'policy' 66 | resp.policy.string.parse_policy 67 | else 68 | op.field ? resp.send(op.field).to_h : resp.to_h 69 | end 70 | 71 | rescue Aws::S3::Errors::ServiceError => e 72 | log_error(bucket.name, op.func, e.code) 73 | 74 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 75 | end 76 | 77 | resources.push(struct.to_h) 78 | 79 | rescue Aws::S3::Errors::NoSuchBucket 80 | # skip missing bucket 81 | end 82 | end 83 | 84 | resources 85 | end 86 | 87 | private 88 | 89 | # not an error 90 | def suppressed_errors 91 | %w[ 92 | AccessDenied 93 | ServerSideEncryptionConfigurationNotFoundError 94 | NoSuchBucketPolicy 95 | NoSuchTagSet 96 | NoSuchWebsiteConfiguration 97 | ReplicationConfigurationNotFoundError 98 | NoSuchPublicAccessBlockConfiguration 99 | ObjectLockConfigurationNotFoundError 100 | OwnershipControlsNotFoundError 101 | ] 102 | end 103 | end 104 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/sagemaker.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect SageMaker Resources 5 | # 6 | class SageMaker < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_notebook_instances 15 | # 16 | @client.list_notebook_instances.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.notebook_instances.each do |instance| 20 | struct = OpenStruct.new(@client.describe_notebook_instance({ 21 | notebook_instance_name: instance.notebook_instance_name 22 | }).to_h) 23 | struct.type = 'notebook_instance' 24 | struct.arn = instance.notebook_instance_arn 25 | 26 | resources.push(struct.to_h) 27 | end 28 | end 29 | 30 | # 31 | # list_endpoints 32 | # 33 | @client.list_endpoints.each_with_index do |response, page| 34 | log(response.context.operation_name, page) 35 | 36 | response.endpoints.each do |instance| 37 | struct = OpenStruct.new(@client.describe_endpoint({ 38 | endpoint_name: instance.endpoint_name 39 | }).to_h) 40 | struct.type = 'endpoint' 41 | struct.arn = instance.endpoint_arn 42 | 43 | resources.push(struct.to_h) 44 | end 45 | end 46 | 47 | resources 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/secretsmanager.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Secrets Manager resources 5 | # 6 | class SecretsManager < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_auto_scaling_groups 15 | # 16 | @client.list_secrets.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.secret_list.each_with_index do |secret, i| 20 | log(response.context.operation_name, i) 21 | 22 | struct = OpenStruct.new(secret.to_h) 23 | struct.type = 'secret' 24 | 25 | resources.push(struct.to_h) 26 | end 27 | end 28 | 29 | resources 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/securityhub.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Security Hub resources 5 | # 6 | class SecurityHub < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_hub 15 | # 16 | begin 17 | @client.describe_hub.each do |response| 18 | log(response.context.operation_name) 19 | 20 | struct = OpenStruct.new(response.to_h) 21 | struct.type = 'hub' 22 | struct.arn = response.hub_arn 23 | 24 | resources.push(struct.to_h) 25 | end 26 | rescue Aws::SecurityHub::Errors::ServiceError => e 27 | log_error(e.code) 28 | 29 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 30 | end 31 | 32 | resources 33 | end 34 | 35 | private 36 | 37 | # not an error 38 | def suppressed_errors 39 | %w[ 40 | InvalidAccessException 41 | ] 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/servicequotas.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect ServiceQuota resources 5 | # 6 | class ServiceQuotas < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_service_quotas 15 | # 16 | # TODO: expand to more services as needed 17 | # 18 | # service_codes = %w[autoscaling ec2 ecr eks elasticloadbalancing fargate iam vpc] 19 | service_codes = %w[ec2 eks iam] 20 | 21 | service_codes.each do |service| 22 | @client.list_service_quotas({ service_code: service }).each_with_index do |response, page| 23 | log(response.context.operation_name, service, page) 24 | 25 | response.quotas.each do |quota| 26 | struct = OpenStruct.new(quota.to_h) 27 | struct.type = 'quota' 28 | struct.arn = quota.quota_arn 29 | 30 | resources.push(struct.to_h) 31 | end 32 | end 33 | rescue Aws::ServiceQuotas::Errors::ServiceError => e 34 | log_error(e.code, service) 35 | 36 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 37 | end 38 | 39 | resources 40 | end 41 | 42 | private 43 | 44 | # not an error 45 | def suppressed_errors 46 | %w[ 47 | NoSuchResourceException 48 | ] 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/ses.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect SES resources 5 | # 6 | class SES < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_identities 15 | # 16 | @client.list_identities.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.identities.each do |identity| 20 | struct = OpenStruct.new 21 | struct.type = 'identity' 22 | struct.arn = "arn:aws:ses:#{@region}:#{@account}:identity/#{identity}" 23 | 24 | # get_identity_dkim_attributes 25 | struct.dkim_attributes = @client.get_identity_dkim_attributes({ identities: [identity] }).dkim_attributes[identity].to_h 26 | 27 | resources.push(struct.to_h) 28 | end 29 | end 30 | 31 | resources 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/shield.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Shield resources 5 | # 6 | class Shield < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_subscription 15 | # 16 | @client.describe_subscription.each do |response| 17 | log(response.context.operation_name) 18 | 19 | struct = OpenStruct.new(response.subscription.to_h) 20 | struct.type = 'subscription' 21 | struct.arn = "arn:aws:shield:#{@region}:#{@account}:subscription" 22 | 23 | resources.push(struct.to_h) 24 | end 25 | 26 | # 27 | # describe_emergency_contact_settings 28 | # 29 | @client.describe_emergency_contact_settings.each do |response| 30 | log(response.context.operation_name) 31 | 32 | struct = OpenStruct.new 33 | struct.type = 'contact_list' 34 | struct.arn = "arn:aws:shield:#{@region}:#{@account}:contact_list" 35 | struct.contacts = response&.emergency_contact_list&.map(&:to_h) 36 | 37 | resources.push(struct.to_h) 38 | end 39 | 40 | # 41 | # list_protections 42 | # 43 | @client.list_protections.each_with_index do |response, page| 44 | log(response.context.operation_name, page) 45 | 46 | # describe_protection 47 | response.protections.each do |protection| 48 | struct = OpenStruct.new(@client.describe_protection({ protection_id: protection.id }).protection.to_h) 49 | struct.type = 'protection' 50 | struct.arn = protection.resource_arn 51 | 52 | resources.push(struct.to_h) 53 | end 54 | end 55 | 56 | resources 57 | rescue Aws::Shield::Errors::ServiceError => e 58 | log_error(e.code) 59 | 60 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 61 | 62 | [] # no access or service isn't enabled 63 | end 64 | 65 | private 66 | 67 | # not an error 68 | def suppressed_errors 69 | %w[ 70 | ResourceNotFoundException 71 | ] 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/sns.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect SNS resources 5 | # 6 | class SNS < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_topics 15 | # 16 | @client.list_topics.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.topics.each do |topic| 20 | log(response.context.operation_name, topic.topic_arn, page) 21 | 22 | # get_topic_attributes 23 | struct = OpenStruct.new(@client.get_topic_attributes({ topic_arn: topic.topic_arn }).attributes.to_h) 24 | struct.type = 'topic' 25 | struct.arn = topic.topic_arn 26 | struct.policy = struct.delete_field('Policy').parse_policy 27 | struct.effective_delivery_policy = struct.delete_field('EffectiveDeliveryPolicy').parse_policy 28 | struct.subscriptions = [] 29 | 30 | # list_subscriptions_by_topic 31 | @client.list_subscriptions_by_topic({ topic_arn: topic.topic_arn }).each_with_index do |response, page| 32 | log(response.context.operation_name, topic.topic_arn, page) 33 | 34 | response.subscriptions.each do |sub| 35 | struct.subscriptions.push(sub.to_h) 36 | end 37 | end 38 | 39 | resources.push(struct.to_h) 40 | end 41 | end 42 | 43 | resources 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/sqs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect SQS resources 5 | # 6 | class SQS < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_queues 15 | # 16 | @client.list_queues.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.queue_urls.each do |queue| 20 | log(response.context.operation_name, queue.downcase.split('/').last, page) 21 | 22 | # get_queue_attributes 23 | struct = OpenStruct.new(@client.get_queue_attributes({ queue_url: queue, attribute_names: ['All'] }).attributes.to_h) 24 | struct.type = 'queue' 25 | struct.arn = struct.QueueArn 26 | struct.policy = struct.Policy ? struct.delete_field('Policy').parse_policy : nil 27 | 28 | resources.push(struct.to_h) 29 | end 30 | end 31 | 32 | resources 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/ssm.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect SSM resources 5 | # 6 | class SSM < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_instance_information 15 | # 16 | @client.describe_instance_information.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.instance_information_list.each do |instance| 20 | struct = OpenStruct.new(instance.to_h) 21 | struct.type = 'instance' 22 | struct.arn = instance.instance_id 23 | 24 | resources.push(struct.to_h) 25 | end 26 | end 27 | 28 | # 29 | # describe_parameters 30 | # 31 | @client.describe_parameters.each_with_index do |response, page| 32 | log(response.context.operation_name, page) 33 | 34 | response.parameters.each do |parameter| 35 | struct = OpenStruct.new(parameter.to_h) 36 | struct.string_type = parameter.type 37 | struct.type = 'parameter' 38 | struct.arn = "arn:aws:#{@service}:#{@region}:#{@account}:parameter:#{parameter.name}" 39 | 40 | resources.push(struct.to_h) 41 | end 42 | end 43 | 44 | resources 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/support.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Support resources 5 | # 6 | class Support < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # describe_trusted_advisor_checks 15 | # 16 | @client.describe_trusted_advisor_checks({ language: 'en' }).each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.checks.each do |check| 20 | struct = OpenStruct.new(check.to_h) 21 | struct.type = 'trusted_advisor_check' 22 | struct.arn = "arn:aws:support::trusted_advisor_check/#{check.id}" 23 | 24 | # describe_trusted_advisor_check_result 25 | struct.result = @client.describe_trusted_advisor_check_result({ check_id: check.id }).result.to_h 26 | log(response.context.operation_name, 'describe_trusted_advisor_check_result', check.id) 27 | 28 | resources.push(struct.to_h) 29 | end 30 | end 31 | 32 | resources 33 | rescue Aws::Support::Errors::ServiceError => e 34 | log_error(e.code) 35 | 36 | raise e unless suppressed_errors.include?(e.code) && !@options.quit_on_exception 37 | 38 | [] # no Support subscription 39 | end 40 | 41 | private 42 | 43 | # not an error 44 | def suppressed_errors 45 | %w[ 46 | AccessDeniedException 47 | SubscriptionRequiredException 48 | ] 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/transfer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect Transfer resources 5 | # 6 | class Transfer < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # list_servers 15 | # 16 | @client.list_servers.each_with_index do |response, page| 17 | log(response.context.operation_name, page) 18 | 19 | response.servers.each do |server| 20 | struct = OpenStruct.new(server.to_h) 21 | struct.type = 'server' 22 | 23 | resources.push(struct.to_h) 24 | end 25 | end 26 | 27 | resources 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/wafv2.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect WAFv2 resources 5 | # 6 | class WAFV2 < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | # TODO: resolve scope (e.g. CLOUDFRONT supported?) 11 | # 12 | def collect 13 | resources = [] 14 | 15 | # 16 | # list_web_acls 17 | # 18 | # %w[CLOUDFRONT REGIONAL].each do |scope| 19 | %w[REGIONAL].each do |scope| 20 | @client.list_web_acls({ scope: scope }).each_with_index do |response, page| 21 | log(response.context.operation_name, page) 22 | 23 | response.web_acls.each do |acl| 24 | struct = OpenStruct.new(acl.to_h) 25 | struct.type = 'web_acl' 26 | 27 | params = { 28 | name: acl.name, 29 | scope: scope, 30 | id: acl.id 31 | } 32 | 33 | # get_web_acl 34 | @client.get_web_acl(params).each do |r| 35 | struct.arn = r.web_acl.arn 36 | struct.details = r.web_acl 37 | end 38 | 39 | # list_resources_for_web_acl 40 | @client.list_resources_for_web_acl({ web_acl_arn: acl.arn }).each do |r| 41 | struct.resources = r.resource_arns.map(&:to_h) 42 | end 43 | 44 | resources.push(struct.to_h) 45 | end 46 | end 47 | end 48 | 49 | resources 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/workspaces.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect WorkSpaces resources 5 | # 6 | class WorkSpaces < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | # 13 | # describe_workspaces 14 | # 15 | @client.describe_workspaces.each_with_index do |response, page| 16 | log(response.context.operation_name, page) 17 | 18 | response.workspaces.each do |workspace| 19 | struct = OpenStruct.new(workspace.to_h) 20 | struct.type = 'workspace' 21 | struct.arn = "arn:aws:workspaces:#{@region}::workspace/#{workspace.workspace_id}" 22 | 23 | resources.push(struct.to_h) 24 | end 25 | end 26 | 27 | resources 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/aws_recon/collectors/xray.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Collect XRay resources 5 | # 6 | class XRay < Mapper 7 | # 8 | # Returns an array of resources. 9 | # 10 | def collect 11 | resources = [] 12 | 13 | # 14 | # get_encryption_config 15 | # 16 | struct = OpenStruct.new 17 | struct.config = @client.get_encryption_config.encryption_config.to_h 18 | struct.type = 'config' 19 | struct.arn = "arn:aws:xray:#{@region}:#{@account}/config" 20 | 21 | resources.push(struct.to_h) 22 | 23 | resources 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/aws_recon/lib/formatter.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Customize resource format/shape 3 | # 4 | class Formatter 5 | # 6 | # Custom 7 | # 8 | def custom(account_id, region, service, resource) 9 | { 10 | account: account_id, 11 | name: resource[:arn], 12 | service: service.name, 13 | region: region, 14 | asset_type: resource[:type], 15 | resource: { data: resource, version: 'v1' }, 16 | timestamp: Time.now.utc 17 | } 18 | end 19 | 20 | # 21 | # Standard AWS 22 | # 23 | def aws(account_id, region, service, resource) 24 | { 25 | account: account_id, 26 | service: service.name, 27 | region: region, 28 | resource: resource, 29 | timestamp: Time.now.utc 30 | } 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/aws_recon/lib/mapper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Generic wrapper for service clients. 5 | # 6 | # Paging 7 | # ------ 8 | # The AWS Ruby SDK has a built-in enumerator in the client response 9 | # object that automatically handles paging large requests. Confirm 10 | # the AWS SDK client call returns a Aws::PageableResponse to take 11 | # advantage of automatic paging. 12 | # 13 | # Retries 14 | # ------- 15 | # The AWS Ruby SDK performs retries automatically. We change the 16 | # retry_limit to 9 (10 total requests) and the retry_backoff 17 | # to add 5 seconds delay on each retry for a total max of 55 seconds. 18 | # 19 | class Mapper 20 | # Services that use us-east-1 endpoint only: 21 | # Organizations 22 | # Route53Domains 23 | # Shield 24 | # S3 (unless the bucket was created in another region) 25 | SINGLE_REGION_SERVICES = %w[route53domains s3 shield support organizations].freeze 26 | 27 | def initialize(account, service, region, options) 28 | @account = account 29 | @service = service 30 | @region = region 31 | @options = options 32 | @thread = Parallel.worker_number || 0 33 | 34 | # build the client interface 35 | module_name = "Aws::#{service}::Client" 36 | 37 | # incremental delay on retries (seconds) 38 | retry_delay = 5 39 | 40 | # default is 3 retries, with 15 second sleep in between 41 | # reset to 9 retries, with incremental backoff 42 | client_options = { 43 | retry_mode: 'legacy', # legacy, standard, or adaptive 44 | retry_limit: 9, # only legacy 45 | retry_backoff: ->(context) { sleep(retry_delay * context.retries + 1) }, # only legacy 46 | http_read_timeout: 10 47 | } 48 | 49 | # regional service 50 | client_options.merge!({ region: region }) unless region == 'global' 51 | 52 | # single region services 53 | client_options.merge!({ region: 'us-east-1' }) if SINGLE_REGION_SERVICES.include?(service.downcase) # rubocop:disable Layout/LineLength 54 | 55 | # debug with wire trace 56 | client_options.merge!({ http_wire_trace: true }) if @options.debug 57 | 58 | @client = Object.const_get(module_name).new(client_options) 59 | end 60 | 61 | private 62 | 63 | def _msg(msg) 64 | base_msg = ["t#{@thread}", @region, @service] 65 | base_msg.concat(msg) 66 | end 67 | 68 | def log(*msg) 69 | return unless @options.verbose 70 | 71 | puts _msg(msg).map(&:to_s).join('.') 72 | end 73 | 74 | def log_error(*msg) 75 | return unless @options.verbose 76 | 77 | puts _msg(msg).map(&:to_s).join('.') 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/aws_recon/lib/patch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Parse and unescape AWS policy document string 5 | # 6 | module PolicyStringParser 7 | def parse_policy 8 | JSON.parse(CGI.unescape(self)) 9 | rescue StandardError 10 | nil 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/aws_recon/options.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Command line options parser 5 | # 6 | class Parser 7 | DEFAULT_CONFIG_FILE = nil 8 | DEFAULT_OUTPUT_FILE = File.expand_path(File.join(Dir.pwd, 'output.json')).freeze 9 | DEFAULT_S3_PATH = nil 10 | SERVICES_CONFIG_FILE = File.join(File.dirname(__FILE__), 'services.yaml').freeze 11 | DEFAULT_FORMAT = 'aws' 12 | DEFAULT_THREADS = 8 13 | MAX_THREADS = 128 14 | 15 | Options = Struct.new( 16 | :regions, 17 | :services, 18 | :config_file, 19 | :s3, 20 | :output_file, 21 | :output_format, 22 | :threads, 23 | :jsonl, 24 | :collect_user_data, 25 | :skip_slow, 26 | :skip_credential_report, 27 | :stream_output, 28 | :verbose, 29 | :quit_on_exception, 30 | :debug 31 | ) 32 | 33 | def self.parse(options) 34 | begin 35 | unless (options & ['-h', '--help']).any? 36 | aws_regions = ['global'].concat(Aws::EC2::Client.new.describe_regions.regions.map(&:region_name)) 37 | end 38 | rescue Aws::Errors::ServiceError => e 39 | warn "\nAWS Error: #{e.code}\n\n" 40 | exit(1) 41 | end 42 | 43 | aws_services = YAML.load(File.read(SERVICES_CONFIG_FILE), symbolize_names: true) 44 | 45 | args = Options.new( 46 | aws_regions, 47 | aws_services.map { |service| service[:name] }, 48 | DEFAULT_CONFIG_FILE, 49 | DEFAULT_S3_PATH, 50 | DEFAULT_OUTPUT_FILE, 51 | DEFAULT_FORMAT, 52 | DEFAULT_THREADS, 53 | false, 54 | false, 55 | false, 56 | false, 57 | false, 58 | false, 59 | false, 60 | false 61 | ) 62 | 63 | opt_parser = OptionParser.new do |opts| 64 | opts.banner = "\n\x1b[32mAWS Recon\x1b[0m - AWS Inventory Collector (#{AwsRecon::VERSION})\n\nUsage: aws_recon [options]" 65 | 66 | # regions 67 | opts.on('-r', '--regions [REGIONS]', 'Regions to scan, separated by comma (default: all)') do |regions| 68 | next if regions.downcase == 'all' 69 | 70 | args.regions = args.regions.filter { |region| regions.split(',').include?(region) } 71 | end 72 | 73 | # regions to skip 74 | opts.on('-n', '--not-regions [REGIONS]', 'Regions to skip, separated by comma (default: none)') do |regions| 75 | next if regions.downcase == 'all' 76 | 77 | args.regions = args.regions.filter { |region| !regions.split(',').include?(region) } 78 | end 79 | 80 | # services 81 | opts.on('-s', '--services [SERVICES]', 'Services to scan, separated by comma (default: all)') do |services| 82 | next if services.downcase == 'all' 83 | 84 | svcs = services.split(',') 85 | args.services = aws_services.map { |service| service[:name] if svcs.include?(service[:name]) || svcs.include?(service[:alias]) }.compact # rubocop:disable Layout/LineLength 86 | end 87 | 88 | # services to skip 89 | opts.on('-x', '--not-services [SERVICES]', 'Services to skip, separated by comma (default: none)') do |services| 90 | next if services.downcase == 'all' 91 | 92 | svcs = services.split(',') 93 | args.services = aws_services.map { |service| service[:name] unless svcs.include?(service[:name]) || svcs.include?(service[:alias]) }.compact # rubocop:disable Layout/LineLength 94 | end 95 | 96 | # config file 97 | opts.on('-c', '--config [CONFIG]', 'Specify config file for services & regions (e.g. config.yaml)') do |config| 98 | args.config_file = config 99 | end 100 | 101 | # write output file to S3 bucket 102 | opts.on('-b', '--s3-bucket [BUCKET:REGION]', 'Write output file to S3 bucket (default: \'\')') do |bucket_with_region| 103 | args.stream_output = false 104 | args.s3 = bucket_with_region 105 | end 106 | 107 | # output file 108 | opts.on('-o', '--output [OUTPUT]', 'Specify output file (default: output.json)') do |output| 109 | args.output_file = File.expand_path(File.join(Dir.pwd, output)) 110 | end 111 | 112 | # output format 113 | opts.on('-f', '--format [FORMAT]', 'Specify output format (default: aws)') do |f| 114 | args.output_format = f.downcase if %w[aws custom].include?(f.downcase) 115 | end 116 | 117 | # threads 118 | opts.on('-t', '--threads [THREADS]', "Specify max threads (default: #{Parser::DEFAULT_THREADS}, max: 128)") do |threads| 119 | args.threads = threads.to_i if (0..Parser::MAX_THREADS).include?(threads.to_i) 120 | end 121 | 122 | # output NDJSON/JSONL format 123 | opts.on('-l', '--json-lines', 'Output NDJSON/JSONL format (default: false)') do 124 | args.jsonl = true 125 | end 126 | 127 | # collect EC2 instance user data 128 | opts.on('-u', '--user-data', 'Collect EC2 instance user data (default: false)') do 129 | args.collect_user_data = true 130 | end 131 | 132 | # skip slow operations 133 | opts.on('-z', '--skip-slow', 'Skip slow operations (default: false)') do 134 | args.skip_slow = true 135 | end 136 | 137 | # skip generating IAM credential report 138 | opts.on('-g', '--skip-credential-report', 'Skip generating IAM credential report (default: false)') do 139 | args.skip_credential_report = true 140 | end 141 | 142 | # stream output (forces JSON lines, doesn't output handled warnings or errors ) 143 | opts.on('-j', '--stream-output', 'Stream JSON lines to stdout (default: false)') do 144 | args.output_file = nil 145 | args.verbose = false 146 | args.debug = false 147 | args.stream_output = true 148 | end 149 | 150 | # verbose 151 | opts.on('-v', '--verbose', 'Output client progress and current operation') do 152 | args.verbose = true unless args.stream_output 153 | end 154 | 155 | # re-raise exceptions 156 | opts.on('-q', '--quit-on-exception', 'Stop collection if an API error is encountered (default: false)') do 157 | args.quit_on_exception = true 158 | end 159 | 160 | # debug 161 | opts.on('-d', '--debug', 'Output debug with wire trace info') do 162 | unless args.stream_output 163 | args.debug = true 164 | args.verbose = true 165 | end 166 | end 167 | 168 | opts.on('-h', '--help', 'Print this help information') do 169 | puts opts 170 | exit 171 | end 172 | end 173 | 174 | opt_parser.parse!(options) 175 | args 176 | end 177 | end 178 | -------------------------------------------------------------------------------- /lib/aws_recon/services.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Organizations 3 | global: true 4 | alias: organizations 5 | - name: AccessAnalyzer 6 | alias: aa 7 | - name: ApplicationAutoScaling 8 | alias: aas 9 | - name: Backup 10 | alias: backup 11 | - name: ConfigService 12 | alias: config 13 | - name: CodeBuild 14 | alias: codebuild 15 | - name: CodePipeline 16 | alias: codepipeline 17 | - name: AutoScaling 18 | alias: autoscaling 19 | - name: CloudTrail 20 | alias: cloudtrail 21 | - name: CloudFront 22 | alias: cloudfront 23 | - name: EC2 24 | global: true 25 | alias: ec2 26 | - name: EC2 27 | alias: ec2 28 | - name: EKS 29 | alias: eks 30 | - name: ECS 31 | alias: ecs 32 | - name: ElasticLoadBalancing 33 | alias: elb 34 | - name: ElasticLoadBalancingV2 35 | alias: elbv2 36 | - name: ElastiCache 37 | alias: elasticache 38 | - name: EMR 39 | alias: emr 40 | - name: Glue 41 | alias: glue 42 | - name: IAM 43 | global: true 44 | alias: iam 45 | - name: Lambda 46 | alias: lambda 47 | - name: S3 48 | global: true 49 | alias: s3 50 | - name: RDS 51 | alias: rds 52 | - name: ECR 53 | alias: ecr 54 | - name: ECRPublic 55 | alias: ecrpublic 56 | excluded_regions: 57 | - af-south-1 58 | - ap-east-1 59 | - ap-northeast-1 60 | - ap-northeast-2 61 | - ap-northeast-3 62 | - ap-south-1 63 | - ap-southeast-1 64 | - ap-southeast-2 65 | - ca-central-1 66 | - eu-central-1 67 | - eu-north-1 68 | - eu-south-1 69 | - eu-west-1 70 | - eu-west-2 71 | - eu-west-3 72 | - me-south-1 73 | - sa-east-1 74 | - us-east-2 75 | - us-west-1 76 | - us-west-2 77 | - af-south-1 78 | - ap-east-1 79 | - eu-south-1 80 | - me-south-1 81 | - name: DynamoDB 82 | alias: dynamodb 83 | - name: KMS 84 | alias: kms 85 | - name: Kinesis 86 | alias: kinesis 87 | - name: Redshift 88 | alias: redshift 89 | - name: ElasticsearchService 90 | alias: es 91 | - name: APIGateway 92 | alias: apigateway 93 | - name: ApiGatewayV2 94 | alias: apigatewayv2 95 | - name: Route53 96 | alias: route53 97 | - name: Route53Domains 98 | global: true 99 | alias: route53domains 100 | - name: SQS 101 | alias: sqs 102 | - name: ACM 103 | alias: acm 104 | - name: SNS 105 | alias: sns 106 | - name: Shield 107 | global: true 108 | alias: shield 109 | - name: CloudFormation 110 | alias: cloudformation 111 | - name: SES 112 | alias: ses 113 | excluded_regions: 114 | - ap-east-1 115 | - name: CloudWatch 116 | alias: cloudwatch 117 | - name: CloudWatchLogs 118 | alias: cloudwatchlogs 119 | - name: Kafka 120 | alias: kafka 121 | - name: SecretsManager 122 | alias: secretsmanager 123 | - name: SecurityHub 124 | alias: securityhub 125 | - name: Support 126 | global: true 127 | alias: support 128 | - name: SSM 129 | alias: ssm 130 | - name: GuardDuty 131 | alias: guardduty 132 | - name: Athena 133 | alias: athena 134 | - name: EFS 135 | alias: efs 136 | - name: Firehose 137 | alias: firehose 138 | - name: Lightsail 139 | alias: lightsail 140 | excluded_regions: 141 | - af-south-1 142 | - ap-east-1 143 | - ap-northeast-3 144 | - eu-south-1 145 | - me-south-1 146 | - sa-east-1 147 | - us-west-1 148 | - name: WorkSpaces 149 | alias: workspaces 150 | excluded_regions: 151 | - ap-east-1 152 | - ap-northeast-3 153 | - eu-north-1 154 | - eu-south-1 155 | - eu-west-3 156 | - me-south-1 157 | - us-east-2 158 | - us-west-1 159 | - name: SageMaker 160 | alias: sagemaker 161 | - name: ServiceQuotas 162 | alias: servicequotas 163 | - name: Transfer 164 | alias: transfer 165 | - name: DirectConnect 166 | alias: directconnect 167 | - name: DirectoryService 168 | alias: ds 169 | - name: DatabaseMigrationService 170 | alias: dms 171 | - name: XRay 172 | alias: xray 173 | - name: WAFV2 174 | alias: wafv2 175 | excluded_regions: 176 | - ap-northeast-3 177 | -------------------------------------------------------------------------------- /lib/aws_recon/version.rb: -------------------------------------------------------------------------------- 1 | module AwsRecon 2 | VERSION = "0.5.33" 3 | end 4 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | [![Docker Pulls](https://img.shields.io/docker/pulls/darkbitio/aws_recon?logo=docker)](https://hub.docker.com/r/darkbitio/aws_recon) 2 | [![Gem Version](https://badge.fury.io/rb/aws_recon.svg)](https://rubygems.org/gems/aws_recon) 3 | [![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/joshlarsen/aws-recon/smoke-test/main)](https://github.com/joshlarsen/aws-recon/actions?query=branch%3Amain) 4 | [![AWS Service Regions](https://github.com/joshlarsen/aws-recon/actions/workflows/check-aws-regions.yml/badge.svg?branch=main&event=schedule)](https://github.com/joshlarsen/aws-recon/actions/workflows/check-aws-regions.yml) 5 | 6 | # AWS Recon 7 | 8 | A multi-threaded AWS security-focused inventory collection tool written in Ruby. 9 | 10 | This tool was created to facilitate efficient collection of a large amount of AWS resource attributes and metadata. It aims to collect nearly everything that is relevant to the security configuration and posture of an AWS environment. 11 | 12 | Existing tools (e.g. [AWS Config](https://aws.amazon.com/config)) that do some form of resource collection lack the coverage and specificity to accurately measure security posture (e.g. detailed resource attribute data, fully parsed policy documents, and nested resource relationships). 13 | 14 | AWS Recon handles collection from large accounts by taking advantage of automatic retries (either due to network reliability or API throttling), automatic paging of large responses (> 100 resources per API call), and multi-threading parallel requests to speed up collection. 15 | 16 | ## Project Goals 17 | 18 | - More complete resource coverage than available tools (especially for ECS & EKS) 19 | - More granular resource detail, including nested related resources in the output 20 | - Flexible output (console, JSON lines, plain JSON, file, S3 bucket, and standard out) 21 | - Efficient (multi-threaded, rate limited, automatic retries, and automatic result paging) 22 | - Easy to maintain and extend 23 | 24 | ## Awesome companies using AWS Recon\*\* 25 | 26 | - [Netflix](https://www.netflix.com/) 27 | - [HashiCorp](https://www.hashicorp.com/) 28 | - [Workday](https://www.workday.com/) 29 | - [Stripe](https://stripe.com/) 30 | - [PayPal](https://paypal.com/) 31 | - [Typeform](https://typeform.com/) 32 | - [Amazon Web Services](https://aws.amazon.com/) 33 | - [Plaid](https://plaid.com/) 34 | - [Expel](https://expel.io/) 35 | - [Mozilla](https://www.mozilla.org/) 36 | - [Bugcrowd](https://www.bugcrowd.com/) 37 | - [Dropbox](https://www.dropbox.com/) 38 | - [Pinterest](https://www.pinterest.com/) 39 | - [HackerOne](https://www.hackerone.com/) 40 | - [MuleSoft](https://www.mulesoft.com/) 41 | - [Slack](https://slack.com/) 42 | - [Drata](https://drata.com/) 43 | - [Google](https://www.google.com/) 44 | - [Sophos](https://www.sophos.com/) 45 | - [Sumo Logic](https://www.sumologic.com/) 46 | - [Coalfile](https://www.coalfire.com/) 47 | - [Xero](https://www.xero.com/) 48 | 49 | > \*\* usage does not imply endorsement 50 | 51 | ## Setup 52 | 53 | ### Requirements 54 | 55 | AWS Recon needs an AWS account role or credentials with `ReadOnlyAccess`. Full `AdministratorAccess` is over-privileged, but will work as well. The `SecurityAudit` policy is **not** sufficient as it omits access to many services. 56 | 57 | #### Running via Docker 58 | 59 | Use Docker version 19.x or above to run the pre-built image without having to install anything. 60 | 61 | #### Running locally via Ruby 62 | 63 | If you already have Ruby installed (2.6.x or 2.7.x), you may want to install the Ruby gem. 64 | 65 | ### Installation 66 | 67 | AWS Recon can be run locally via a Docker container or by installing the Ruby gem. 68 | 69 | To run via a Docker a container, pass the necessary AWS credentials into the Docker `run` command. For example: 70 | 71 | ``` 72 | $ docker run -t --rm \ 73 | -e AWS_REGION \ 74 | -e AWS_ACCESS_KEY_ID \ 75 | -e AWS_SECRET_ACCESS_KEY \ 76 | -e AWS_SESSION_TOKEN \ 77 | -v $(pwd)/output.json:/recon/output.json \ 78 | darkbitio/aws_recon:latest \ 79 | aws_recon -v -s EC2 -r global,us-east-1,us-east-2 80 | ``` 81 | 82 | To run locally, first install the gem: 83 | 84 | ``` 85 | $ gem install aws_recon 86 | Fetching aws_recon-0.5.17.gem 87 | Fetching aws-sdk-3.0.1.gem 88 | Fetching parallel-1.20.1.gem 89 | ... 90 | Successfully installed aws-sdk-3.0.1 91 | Successfully installed parallel-1.20.1 92 | Successfully installed aws_recon-0.5.17 93 | ``` 94 | 95 | Or add it to your Gemfile using `bundle`: 96 | 97 | ``` 98 | $ bundle add aws_recon 99 | Fetching gem metadata from https://rubygems.org/ 100 | Resolving dependencies... 101 | ... 102 | Using aws-sdk 3.0.1 103 | Using parallel-1.20.1 104 | Using aws_recon 0.5.17 105 | ``` 106 | 107 | ## Usage 108 | 109 | AWS Recon will leverage any AWS credentials (see [requirements](#requirements)) currently available to the environment it runs in. If you are collecting from multiple accounts, you may want to leverage something like [aws-vault](https://github.com/99designs/aws-vault) to manage different credentials. 110 | 111 | ``` 112 | $ aws-vault exec profile -- aws_recon 113 | ``` 114 | 115 | Plain environment variables will work fine too. 116 | 117 | ``` 118 | $ AWS_PROFILE= aws_recon 119 | ``` 120 | 121 | To run from a Docker container using `aws-vault` managed credentials (output to stdout): 122 | 123 | ``` 124 | $ aws-vault exec -- docker run -t --rm \ 125 | -e AWS_REGION \ 126 | -e AWS_ACCESS_KEY_ID \ 127 | -e AWS_SECRET_ACCESS_KEY \ 128 | -e AWS_SESSION_TOKEN \ 129 | darkbitio/aws_recon:latest \ 130 | aws_recon -j -s EC2 -r global,us-east-1,us-east-2 131 | ``` 132 | 133 | To run from a Docker container using `aws-vault` managed credentials and output to a file, you will need to satisfy a couple of requirements. First, Docker needs access to bind mount the path you specify (or a parent path above). Second, you need to create an empty file to save the output into (e.g. `output.json`). This is because only that one file is mounted into the Docker container at run time. For example: 134 | 135 | Create an empty file. 136 | 137 | ``` 138 | $ touch output.json 139 | ``` 140 | 141 | Run the `aws_recon` container, specifying the output file. 142 | 143 | ``` 144 | $ aws-vault exec -- docker run -t --rm \ 145 | -e AWS_REGION \ 146 | -e AWS_ACCESS_KEY_ID \ 147 | -e AWS_SECRET_ACCESS_KEY \ 148 | -e AWS_SESSION_TOKEN \ 149 | -v $(pwd)/output.json:/recon/output.json \ 150 | darkbitio/aws_recon:latest \ 151 | aws_recon -s EC2 -v -r global,us-east-1,us-east-2 152 | ``` 153 | 154 | You may want to use the `-v` or `--verbose` flag initially to see status and activity while collection is running. 155 | 156 | In verbose mode, the console output will show: 157 | 158 | ``` 159 | ... 160 | ``` 161 | 162 | The `t` prefix indicates which thread a particular request is running under. Region, service, and operation indicate which request operation is currently in progress and where. 163 | 164 | ``` 165 | $ aws_recon -v 166 | 167 | t0.global.EC2.describe_account_attributes 168 | t2.global.S3.list_buckets 169 | t3.global.Support.describe_trusted_advisor_checks 170 | t2.global.S3.list_buckets.acl 171 | t5.ap-southeast-1.WorkSpaces.describe_workspaces 172 | t6.ap-northeast-1.Lightsail.get_instances 173 | ... 174 | t2.us-west-2.WorkSpaces.describe_workspaces 175 | t1.us-east-2.Lightsail.get_instances 176 | t4.ap-southeast-1.Firehose.list_delivery_streams 177 | t7.ap-southeast-1.Lightsail.get_instances 178 | t0.ap-south-1.Lightsail.get_instances 179 | t1.us-east-2.Lightsail.get_load_balancers 180 | t7.ap-southeast-2.WorkSpaces.describe_workspaces 181 | t2.eu-west-3.SageMaker.list_notebook_instances 182 | t3.eu-west-2.SageMaker.list_notebook_instances 183 | 184 | Finished in 46 seconds. Saving resources to output.json. 185 | ``` 186 | 187 | #### Example command line options 188 | 189 | ``` 190 | # collect S3 and EC2 global resources, as well as us-east-1 and us-east-2 191 | 192 | $ AWS_PROFILE= aws_recon -s S3,EC2 -r global,us-east-1,us-east-2 193 | ``` 194 | 195 | ``` 196 | # collect S3 and EC2 global resources, as well as us-east-1 and us-east-2 197 | 198 | $ AWS_PROFILE= aws_recon --services S3,EC2 --regions global,us-east-1,us-east-2 199 | ``` 200 | 201 | ``` 202 | # save output to S3 bucket 203 | 204 | $ AWS_PROFILE= aws_recon \ 205 | --services S3,EC2 \ 206 | --regions global,us-east-1,us-east-2 \ 207 | --verbose \ 208 | --s3-bucket my-recon-bucket 209 | ``` 210 | 211 | ``` 212 | # save output to S3 bucket with a home region other than us-east-1 213 | 214 | $ AWS_PROFILE= aws_recon \ 215 | --services S3,EC2 \ 216 | --regions global,us-east-1,us-east-2 \ 217 | --verbose \ 218 | --s3-bucket my-recon-bucket:us-west-2 219 | ``` 220 | 221 | Example [OpenCSPM](https://github.com/OpenCSPM/opencspm) formatted (NDJSON) output. 222 | 223 | ``` 224 | $ AWS_PROFILE= aws_recon -l \ 225 | -s S3,EC2 \ 226 | -r global,us-east-1,us-east-2 \ 227 | -f custom 228 | ``` 229 | 230 | or 231 | 232 | ``` 233 | $ AWS_PROFILE= aws_recon -j \ 234 | -s S3,EC2 \ 235 | -r global,us-east-1,us-east-2 \ 236 | -f custom > output.json 237 | ``` 238 | 239 | #### Errors 240 | 241 | API exceptions related to permissions are silently ignored in most cases. These errors are usually due to one of these cases: 242 | 243 | - using a role without sufficient permissions 244 | - querying an account with SCPs in place that prevent usage of certain services 245 | - trying to query a service that isn't enabled/available in your region/account 246 | 247 | In `verbose` mode, you will see exception logs in the output: 248 | 249 | ``` 250 | t2.us-east-1.EC2.describe_subnets.0 251 | t4.us-east-1.SSM.describe_instance_information.0 252 | t6.us-east-1.SecurityHub.InvalidAccessException <----- 253 | t2.us-east-1.EC2.describe_addresses.0 254 | t4.us-east-1.SSM.describe_parameters.0 255 | t1.us-east-1.GuardDuty.list_detectors.0 256 | ``` 257 | 258 | Use the `-q` command line option to re-raise these exceptions so troubleshooting access issues is easier. 259 | 260 | ``` 261 | Traceback (most recent call last): 262 | arn:aws:sts::1234567890:assumed-role/role/my-audit-role is not authorized to perform: 263 | codepipeline:GetPipeline on resource: arn:aws:codepipeline:us-west-2:1234567890:pipeline 264 | (Aws::CodePipeline::Errors::AccessDeniedException) 265 | ``` 266 | 267 | The exact API operation that triggered the exception is indicated on the last line of the stack trace. If you can't resolve the necessary access, you should exclude those services with `-x` or `--not-services`, or leave off the `-q` option so the collection can continue. 268 | 269 | ### Threads 270 | 271 | AWS Recon uses multiple threads to try to overcome some of the I/O challenges of performing many API calls to endpoints all over the world. 272 | 273 | For global services like IAM, Shield, and Support, requests are not multi-threaded. The S3 module is multi-threaded since each bucket requires several additional calls to collect complete metadata. 274 | 275 | For regional services, a thread (up to the thread limit) is spawned for each service in a region. By default, up to 8 threads will be used. If your account has resources spread across many regions, you may see a speed improvement by increasing threads with `-t X`, where `X` is the number of threads. 276 | 277 | ### Performance 278 | 279 | AWS Recon will make a minimum of ~2,000 API calls in a new/empty account, just to query the supported services in all 20 standard (non-GovCloud, non-China) regions. It is very likely to encounter API rate-limiting (throttling) on large accounts if you enable more threads than the default (8). 280 | 281 | Recon will automatically backoff and respect the retry limits in the API response. If you observe long pauses during collection, this is likely what is happening. Retry collection with the `-d` or `--debug` option to observe the wire trace and see if you're being throttled. Consider using fewer threads or requesting higher rate limits from AWS if you are regularly getting rate-limited. 282 | 283 | ### Options 284 | 285 | Most users will want to limit collection to relevant services and regions. Running without any exclusions will attempt to collect all resources from all regions enabled for the account. 286 | 287 | ``` 288 | $ aws_recon -h 289 | 290 | AWS Recon - AWS Inventory Collector (0.5.17) 291 | 292 | Usage: aws_recon [options] 293 | -r, --regions [REGIONS] Regions to scan, separated by comma (default: all) 294 | -n, --not-regions [REGIONS] Regions to skip, separated by comma (default: none) 295 | -s, --services [SERVICES] Services to scan, separated by comma (default: all) 296 | -x, --not-services [SERVICES] Services to skip, separated by comma (default: none) 297 | -c, --config [CONFIG] Specify config file for services & regions (e.g. config.yaml) 298 | -b, --s3-bucket [BUCKET:REGION] Write output file to S3 bucket (default: '') 299 | -o, --output [OUTPUT] Specify output file (default: output.json) 300 | -f, --format [FORMAT] Specify output format (default: aws) 301 | -t, --threads [THREADS] Specify max threads (default: 8, max: 128) 302 | -l, --json-lines Output NDJSON/JSONL format (default: false) 303 | -u, --user-data Collect EC2 instance user data (default: false) 304 | -z, --skip-slow Skip slow operations (default: false) 305 | -g, --skip-credential-report Skip generating IAM credential report (default: false) 306 | -j, --stream-output Stream JSON lines to stdout (default: false) 307 | -v, --verbose Output client progress and current operation 308 | -q, --quit-on-exception Stop collection if an API error is encountered (default: false) 309 | -d, --debug Output debug with wire trace info 310 | -h, --help Print this help information 311 | 312 | ``` 313 | 314 | #### Output 315 | 316 | Output is always some form of JSON - either JSON lines or plain JSON. The output is either written to a file (the default), or written to stdout (with `-j`). 317 | 318 | When writing to an S3 bucket, the JSON output is automatically compressed with `gzip`. 319 | 320 | ## Support for Manually Enabled Regions 321 | 322 | If you have enabled **manually enabled regions**: 323 | 324 | - me-south-1 - Middle East (Bahrain) 325 | - af-south-1 - Africa (Cape Town) 326 | - ap-east-1 - Asia Pacific (Hong Kong) 327 | - eu-south-1 - Europe (Milan) 328 | 329 | and you are using STS to assume a role into an account, you will need to [enable v2 STS tokens](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) in the account you are assuming the role **from** to be able to run AWS Recon against those regions. 330 | 331 | > Version 1 tokens are valid only in AWS Regions that are available by default. These tokens do not work in manually enabled Regions, such as Asia Pacific (Hong Kong). Version 2 tokens are valid in all Regions. However, version 2 tokens are longer and might affect systems where you temporarily store tokens. 332 | 333 | If you are using a static access key/secret, you can collect from these regions regardless of STS token version. 334 | 335 | ## Supported Services & Resources 336 | 337 | Current "coverage" by service is listed below. The services without coverage will eventually be added. PRs are certainly welcome. :) 338 | 339 | AWS Recon aims to collect all resources and metadata that are relevant in determining the security posture of your AWS account(s). However, it does not actually examine the resources for security posture - that is the job of other tools that take the output of AWS Recon as input. 340 | 341 | - [x] AccessAnalyzer 342 | - [x] AdvancedShield 343 | - [x] ApplicationAutoScaling 344 | - [x] Athena 345 | - [x] Backup 346 | - [x] GuardDuty 347 | - [ ] Macie 348 | - [x] Systems Manager 349 | - [x] Trusted Advisor 350 | - [x] ACM 351 | - [x] API Gateway 352 | - [x] AutoScaling 353 | - [x] CodePipeline 354 | - [x] CodeBuild 355 | - [x] CloudFormation 356 | - [x] CloudFront 357 | - [x] CloudWatch 358 | - [x] CloudWatch Logs 359 | - [x] CloudTrail 360 | - [x] Config 361 | - [x] DirectoryService 362 | - [x] DirectConnect 363 | - [x] DMS 364 | - [x] DynamoDB 365 | - [x] EC2 366 | - [x] ECR 367 | - [x] ECRPublic 368 | - [x] ECS 369 | - [x] EFS 370 | - [x] EKS 371 | - [x] ELB 372 | - [x] EMR 373 | - [x] Elasticsearch 374 | - [x] ElastiCache 375 | - [x] Firehose 376 | - [ ] FMS 377 | - [ ] Glacier 378 | - [x] Glue 379 | - [x] IAM 380 | - [x] KMS 381 | - [x] Kafka 382 | - [x] Kinesis 383 | - [x] Lambda 384 | - [x] Lightsail 385 | - [x] Organizations 386 | - [x] RDS 387 | - [x] Redshift 388 | - [x] Route53 389 | - [x] Route53Domains 390 | - [x] S3 391 | - [x] SageMaker 392 | - [x] SES 393 | - [x] SecretsManager 394 | - [x] SecurityHub 395 | - [x] ServiceQuotas 396 | - [x] Shield 397 | - [x] SNS 398 | - [x] SQS 399 | - [x] Transfer 400 | - [x] VPC 401 | - [ ] WAF 402 | - [x] WAFv2 403 | - [x] Workspaces 404 | - [x] Xray 405 | 406 | ### Additional Coverage 407 | 408 | One of the primary motivations for AWS Recon was to build a tool that is easy to maintain and extend. If you feel like coverage could be improved for a particular service, we would welcome PRs to that effect. Anyone with a moderate familiarity with Ruby will be able to mimic the pattern used by the existing collectors to query a specific service and add the results to the resource collection. 409 | 410 | ### Development 411 | 412 | Clone this repository: 413 | 414 | ``` 415 | $ git clone git@github.com:darkbitio/aws-recon.git 416 | $ cd aws-recon 417 | ``` 418 | 419 | Create a sticky gemset if using RVM: 420 | 421 | ``` 422 | $ rvm use 2.7.2@aws_recon_dev --create --ruby-version 423 | ``` 424 | 425 | Run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. 426 | 427 | To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). 428 | 429 | ### TODO 430 | 431 | - [ ] Test coverage with AWS SDK stubbed resources 432 | 433 | ## Kudos 434 | 435 | AWS Recon was inspired by the excellent work of the people and teams behind these tools: 436 | 437 | - CloudMapper [https://github.com/duo-labs/cloudmapper](https://github.com/duo-labs/cloudmapper) 438 | - Prowler [https://github.com/toniblyx/prowler](https://github.com/toniblyx/prowler) 439 | - CloudSploit [https://github.com/cloudsploit/scans](https://github.com/cloudsploit/scans) 440 | -------------------------------------------------------------------------------- /test/aws_recon_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class AwsReconTest < Minitest::Test 4 | def test_that_it_has_a_version_number 5 | refute_nil ::AwsRecon::VERSION 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__) 2 | require "aws_recon" 3 | 4 | require "minitest/autorun" 5 | -------------------------------------------------------------------------------- /utils/aws/check_region_exclusions.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # 4 | # Check regional service availability against services.yaml exclusions. 5 | # 6 | # AWS updates the regional service table daily. By checking regional service 7 | # coverage, we can identify regions that should be excluded from AWS Recon 8 | # checks. We exclude non-supported regions because service APIs handle non- 9 | # availability differently. Some will respond with an error that can be handled 10 | # by the errors defined in the AWS Ruby SDK client. Others will fail at the 11 | # network level (i.e. there is no API endpoint even available). We could handle 12 | # those errors and silently fail, but we choose not to so we can identify cases 13 | # where there is a lack of service availability in a particular region. 14 | # 15 | require 'net/http' 16 | require 'json' 17 | require 'yaml' 18 | 19 | TS = Time.now.to_i 20 | # AWS Regional services table 21 | URL = "https://api.regional-table.region-services.aws.a2z.com/index.json?timestamp=#{TS}000" 22 | 23 | service_to_query = ARGV[0] 24 | region_exclusion_mistmatch = nil 25 | 26 | # 27 | # load current AWS Recon regions 28 | # 29 | recon_services = YAML.safe_load(File.read('../../lib/aws_recon/services.yaml')) 30 | abort('Errors loading AWS Recon services') unless recon_services.is_a?(Array) 31 | 32 | # 33 | # load current AWS regions (non-gov, non-cn) 34 | # 35 | regions = YAML.safe_load(File.read('regions.yaml')) 36 | abort('Errors loading regions') unless regions['Regions'] 37 | 38 | all_regions = regions['Regions'].map { |r| r['RegionName'] } 39 | 40 | # 41 | # get service/price list from AWS 42 | # 43 | uri = URI(URL) 44 | res = Net::HTTP.get_response(uri) 45 | abort('Error loading AWS services from API') unless res.code == '200' 46 | 47 | map = {} 48 | 49 | # 50 | # load service region availability 51 | # 52 | data = res.body 53 | json = JSON.parse(data) 54 | 55 | # 56 | # query regions for a single service 57 | # 58 | if service_to_query 59 | single_service_regions = [] 60 | 61 | json['prices'].each do |p| 62 | single_service_regions << p['id'].split(':').last 63 | end 64 | 65 | single_service_regions.uniq.sort.each { |r| puts r } 66 | 67 | exit 0 68 | end 69 | 70 | # iterate through AWS provided services & regions 71 | json['prices'].each do |p| 72 | at = p['attributes'] 73 | service_name = at['aws:serviceName'] 74 | service_id, service_region = p['id'].split(':') 75 | 76 | # skip this service unless AWS Recon already has exclusions 77 | next unless recon_services.filter { |s| s['alias'] == service_id }&.length&.positive? 78 | 79 | if map.key?(service_name) 80 | map[service_name]['regions'] << service_region 81 | else 82 | map[service_name] = { 83 | 'id' => service_id, 84 | 'regions' => [service_region] 85 | } 86 | end 87 | end 88 | 89 | # iterate through the services AWS Recon knows about 90 | map.sort.each do |k, v| 91 | service_excluded_regions = all_regions.reject { |r| v['regions'].include?(r) } 92 | 93 | aws_recon_service = recon_services.filter { |s| s['alias'] == v['id'] }&.first 94 | aws_recon_service_excluded_regions = aws_recon_service['excluded_regions'] || [] 95 | 96 | # move on if AWS Recon region exclusions match AWS service region exclusions 97 | next unless service_excluded_regions.sort != aws_recon_service_excluded_regions.sort 98 | 99 | region_exclusion_mistmatch = true 100 | 101 | puts "#{k} (#{v['id']})" 102 | 103 | # determine the direction of the exclusion mismatch 104 | if (service_excluded_regions - aws_recon_service_excluded_regions).length.positive? 105 | puts " + missing region exclusion: #{(service_excluded_regions - aws_recon_service_excluded_regions).join(', ')}" 106 | else 107 | puts " - unnecessary region exclusion: #{(aws_recon_service_excluded_regions - service_excluded_regions).join(', ')}" 108 | end 109 | end 110 | 111 | # exit code 1 if we have any mismatches 112 | exit 1 if region_exclusion_mistmatch 113 | -------------------------------------------------------------------------------- /utils/aws/regions.yaml: -------------------------------------------------------------------------------- 1 | Regions: 2 | - Endpoint: ec2.af-south-1.amazonaws.com 3 | RegionName: af-south-1 4 | - Endpoint: ec2.eu-north-1.amazonaws.com 5 | RegionName: eu-north-1 6 | - Endpoint: ec2.ap-south-1.amazonaws.com 7 | RegionName: ap-south-1 8 | - Endpoint: ec2.eu-west-3.amazonaws.com 9 | RegionName: eu-west-3 10 | - Endpoint: ec2.eu-west-2.amazonaws.com 11 | RegionName: eu-west-2 12 | - Endpoint: ec2.eu-south-1.amazonaws.com 13 | RegionName: eu-south-1 14 | - Endpoint: ec2.eu-west-1.amazonaws.com 15 | RegionName: eu-west-1 16 | - Endpoint: ec2.ap-northeast-3.amazonaws.com 17 | RegionName: ap-northeast-3 18 | - Endpoint: ec2.ap-northeast-2.amazonaws.com 19 | RegionName: ap-northeast-2 20 | - Endpoint: ec2.me-south-1.amazonaws.com 21 | RegionName: me-south-1 22 | - Endpoint: ec2.ap-northeast-1.amazonaws.com 23 | RegionName: ap-northeast-1 24 | - Endpoint: ec2.sa-east-1.amazonaws.com 25 | RegionName: sa-east-1 26 | - Endpoint: ec2.ca-central-1.amazonaws.com 27 | RegionName: ca-central-1 28 | - Endpoint: ec2.ap-east-1.amazonaws.com 29 | RegionName: ap-east-1 30 | - Endpoint: ec2.ap-southeast-1.amazonaws.com 31 | RegionName: ap-southeast-1 32 | - Endpoint: ec2.ap-southeast-2.amazonaws.com 33 | RegionName: ap-southeast-2 34 | - Endpoint: ec2.eu-central-1.amazonaws.com 35 | RegionName: eu-central-1 36 | - Endpoint: ec2.us-east-1.amazonaws.com 37 | RegionName: us-east-1 38 | - Endpoint: ec2.us-east-2.amazonaws.com 39 | RegionName: us-east-2 40 | - Endpoint: ec2.us-west-1.amazonaws.com 41 | RegionName: us-west-1 42 | - Endpoint: ec2.us-west-2.amazonaws.com 43 | RegionName: us-west-2 44 | -------------------------------------------------------------------------------- /utils/cloudformation/aws-recon-cfn-template.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Description: 'Deploys AWS Recon inventory collection resources, scheduled ECS task and corresponding IAM roles and policies.' 3 | Resources: 4 | AWSReconVPC: 5 | Type: AWS::EC2::VPC 6 | Properties: 7 | CidrBlock: '10.75.0.0/27' 8 | Tags: 9 | - Key: Name 10 | Value: aws-recon-CFN 11 | AWSReconSubnet: 12 | Type: AWS::EC2::Subnet 13 | Properties: 14 | CidrBlock: '10.75.0.0/28' 15 | VpcId: !Ref AWSReconVPC 16 | Tags: 17 | - Key: Name 18 | Value: aws-recon-CFN 19 | DependsOn: AWSReconVPC 20 | AWSReconSecurityGroup: 21 | Type: AWS::EC2::SecurityGroup 22 | Properties: 23 | GroupDescription: AWS Recon collection egress 24 | VpcId: !Ref AWSReconVPC 25 | SecurityGroupEgress: 26 | - IpProtocol: -1 27 | FromPort: 0 28 | ToPort: 0 29 | CidrIp: 0.0.0.0/0 30 | Tags: 31 | - Key: Name 32 | Value: aws-recon-CFN 33 | AWSReconInternetGateway: 34 | Type: AWS::EC2::InternetGateway 35 | Properties: 36 | Tags: 37 | - Key: Name 38 | Value: aws-recon-CFN 39 | AWSReconInternetGatewayAttachment: 40 | Type: AWS::EC2::VPCGatewayAttachment 41 | Properties: 42 | InternetGatewayId: !Ref AWSReconInternetGateway 43 | VpcId: !Ref AWSReconVPC 44 | AWSReconEgressRouteTable: 45 | Type: AWS::EC2::RouteTable 46 | Properties: 47 | VpcId: !Ref AWSReconVPC 48 | Tags: 49 | - Key: Name 50 | Value: aws-recon-CFN 51 | AWSReconSubnetRouteTableAssociation: 52 | Type: AWS::EC2::SubnetRouteTableAssociation 53 | Properties: 54 | SubnetId: !Ref AWSReconSubnet 55 | RouteTableId: !Ref AWSReconEgressRouteTable 56 | AWSReconEgressRoute: 57 | Type: AWS::EC2::Route 58 | Properties: 59 | DestinationCidrBlock: '0.0.0.0/0' 60 | GatewayId: !Ref AWSReconInternetGateway 61 | RouteTableId: !Ref AWSReconEgressRouteTable 62 | AWSReconECSCluster: 63 | Type: AWS::ECS::Cluster 64 | Properties: 65 | ClusterName: aws-recon-CFN 66 | CapacityProviders: 67 | - FARGATE 68 | Tags: 69 | - Key: Name 70 | Value: aws-recon-CFN 71 | DependsOn: AWSReconSubnet 72 | AWSReconECSTask: 73 | Type: AWS::ECS::TaskDefinition 74 | Properties: 75 | Family: aws-recon-CFN 76 | RequiresCompatibilities: 77 | - FARGATE 78 | NetworkMode: awsvpc 79 | Cpu: 1024 80 | Memory: 2048 81 | TaskRoleArn: !Ref AWSReconECSTaskRole 82 | ExecutionRoleArn: !Ref AWSReconECSExecutionRole 83 | ContainerDefinitions: 84 | - Name: aws-recon-CFN 85 | Image: 'darkbitio/aws_recon:latest' 86 | EntryPoint: 87 | - 'aws_recon' 88 | - '--verbose' 89 | - '--format' 90 | - 'custom' 91 | AWSReconECSTaskRole: 92 | Type: AWS::IAM::Role 93 | Properties: 94 | RoleName: aws-recon-ecs-task-role 95 | ManagedPolicyArns: 96 | - 'arn:aws:iam::aws:policy/ReadOnlyAccess' 97 | Policies: 98 | - PolicyName: AWSReconECSTaskRole 99 | PolicyDocument: 100 | Version: '2012-10-17' 101 | Statement: 102 | - Effect: Allow 103 | Action: 's3:PutObject' 104 | Resource: 'arn:aws:s3:::CHANGEME/*' 105 | AssumeRolePolicyDocument: 106 | Version: '2012-10-17' 107 | Statement: 108 | - Effect: Allow 109 | Principal: 110 | Service: 111 | - ecs.amazonaws.com 112 | - ecs-tasks.amazonaws.com 113 | Action: 'sts:AssumeRole' 114 | AWSReconECSExecutionRole: 115 | Type: AWS::IAM::Role 116 | Properties: 117 | RoleName: aws-recon-ecs-execution-role 118 | Policies: 119 | - PolicyName: AWSReconECSTaskExecutionPolicy 120 | PolicyDocument: 121 | Version: '2012-10-17' 122 | Statement: 123 | - Effect: Allow 124 | Action: 125 | - 'ecr:GetAuthorizationToken' 126 | - 'ecr:BatchCheckLayerAvailability' 127 | - 'ecr:GetDownloadUrlForLayer' 128 | - 'ecr:BatchGetImage' 129 | - 'logs:CreateLogStream' 130 | - 'logs:PutLogEvents' 131 | Resource: '*' 132 | AssumeRolePolicyDocument: 133 | Version: '2012-10-17' 134 | Statement: 135 | - Effect: Allow 136 | Principal: 137 | Service: 138 | - ecs-tasks.amazonaws.com 139 | Action: 'sts:AssumeRole' 140 | AWSReconCloudWatchEventsRole: 141 | Type: AWS::IAM::Role 142 | Properties: 143 | RoleName: aws-recon-events-role 144 | AssumeRolePolicyDocument: 145 | Version: '2012-10-17' 146 | Statement: 147 | - Effect: Allow 148 | Principal: 149 | Service: 150 | - events.amazonaws.com 151 | Action: 'sts:AssumeRole' 152 | -------------------------------------------------------------------------------- /utils/terraform/cloudwatch.tf: -------------------------------------------------------------------------------- 1 | # https://www.terraform.io/docs/providers/aws/r/cloudwatch_event_rule.html 2 | resource "aws_cloudwatch_event_rule" "default" { 3 | name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}" 4 | description = "AWS Recon scheduled task" 5 | schedule_expression = var.schedule_expression 6 | } 7 | 8 | # https://www.terraform.io/docs/providers/aws/r/cloudwatch_event_target.html 9 | resource "aws_cloudwatch_event_target" "default" { 10 | target_id = aws_ecs_task_definition.aws_recon_task.id 11 | arn = aws_ecs_cluster.aws_recon.arn 12 | rule = aws_cloudwatch_event_rule.default.name 13 | role_arn = aws_iam_role.cw_events.arn 14 | 15 | ecs_target { 16 | launch_type = "FARGATE" 17 | task_definition_arn = aws_ecs_task_definition.aws_recon_task.arn 18 | platform_version = "LATEST" 19 | 20 | network_configuration { 21 | assign_public_ip = true 22 | security_groups = [aws_security_group.sg.id] 23 | subnets = [aws_subnet.subnet.id] 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /utils/terraform/ecs.tf: -------------------------------------------------------------------------------- 1 | resource "aws_ecs_cluster" "aws_recon" { 2 | name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}" 3 | capacity_providers = [local.ecs_task_provider] 4 | } 5 | 6 | resource "aws_ecs_task_definition" "aws_recon_task" { 7 | family = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}" 8 | task_role_arn = aws_iam_role.aws_recon_role.arn 9 | execution_role_arn = aws_iam_role.ecs_task_execution.arn 10 | requires_compatibilities = [local.ecs_task_provider] 11 | network_mode = "awsvpc" 12 | cpu = 1024 13 | memory = 2048 14 | 15 | container_definitions = jsonencode([ 16 | { 17 | name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}" 18 | image = "${var.aws_recon_container_name}:${var.aws_recon_container_version}" 19 | assign_public_ip = true 20 | entryPoint = [ 21 | "aws_recon", 22 | "--verbose", 23 | "--format", 24 | "custom", 25 | "--json-lines", 26 | "--s3-bucket", 27 | "${aws_s3_bucket.aws_recon.bucket}:${data.aws_region.current.name}", 28 | "--regions", 29 | join(",", var.aws_regions) 30 | ] 31 | logConfiguration = { 32 | logDriver = "awslogs" 33 | options = { 34 | awslogs-group = aws_cloudwatch_log_group.aws_recon.name, 35 | awslogs-region = data.aws_region.current.name, 36 | awslogs-stream-prefix = "ecs" 37 | } 38 | } 39 | } 40 | ]) 41 | } 42 | 43 | resource "aws_cloudwatch_log_group" "aws_recon" { 44 | name = "/ecs/${var.aws_recon_base_name}-${random_id.aws_recon.hex}" 45 | retention_in_days = var.retention_period 46 | } 47 | 48 | locals { 49 | ecs_task_provider = "FARGATE" 50 | } 51 | -------------------------------------------------------------------------------- /utils/terraform/iam.tf: -------------------------------------------------------------------------------- 1 | # 2 | # IAM policies and roles for ECS and CloudWatch execution 3 | # 4 | resource "aws_iam_role" "aws_recon_role" { 5 | name = local.aws_recon_task_role_name 6 | assume_role_policy = data.aws_iam_policy_document.aws_recon_task_execution_assume_role_policy.json 7 | } 8 | 9 | data "aws_iam_policy_document" "aws_recon_task_execution_assume_role_policy" { 10 | statement { 11 | actions = ["sts:AssumeRole"] 12 | 13 | principals { 14 | type = "Service" 15 | identifiers = [ 16 | "ecs.amazonaws.com", 17 | "ecs-tasks.amazonaws.com" 18 | ] 19 | } 20 | } 21 | } 22 | 23 | resource "aws_iam_role_policy_attachment" "aws_recon_task_execution" { 24 | role = aws_iam_role.aws_recon_role.name 25 | policy_arn = data.aws_iam_policy.aws_recon_task_execution.arn 26 | } 27 | 28 | resource "aws_iam_role_policy" "aws_recon" { 29 | name = local.bucket_write_policy_name 30 | role = aws_iam_role.aws_recon_role.id 31 | 32 | policy = jsonencode({ 33 | Version = "2012-10-17" 34 | Id = "${var.aws_recon_base_name}-bucket-write" 35 | Statement = [ 36 | { 37 | Sid = "AWSReconS3PutObject" 38 | Effect = "Allow" 39 | Action = "s3:PutObject" 40 | Resource = [ 41 | "${aws_s3_bucket.aws_recon.arn}/*" 42 | ] 43 | } 44 | ] 45 | }) 46 | } 47 | 48 | data "aws_iam_policy" "aws_recon_task_execution" { 49 | arn = "arn:aws:iam::aws:policy/ReadOnlyAccess" 50 | } 51 | 52 | resource "aws_iam_role" "ecs_task_execution" { 53 | name = local.ecs_task_execution_role_name 54 | assume_role_policy = data.aws_iam_policy_document.ecs_task_execution_assume_role_policy.json 55 | 56 | tags = { 57 | Name = local.ecs_task_execution_role_name 58 | } 59 | } 60 | 61 | data "aws_iam_policy_document" "ecs_task_execution_assume_role_policy" { 62 | statement { 63 | actions = ["sts:AssumeRole"] 64 | 65 | principals { 66 | type = "Service" 67 | identifiers = ["ecs-tasks.amazonaws.com"] 68 | } 69 | } 70 | } 71 | 72 | # ECS task execution 73 | resource "aws_iam_policy" "ecs_task_execution" { 74 | name = local.ecs_task_execution_policy_name 75 | policy = data.aws_iam_policy.ecs_task_execution.policy 76 | } 77 | 78 | resource "aws_iam_role_policy_attachment" "ecs_task_execution" { 79 | role = aws_iam_role.ecs_task_execution.name 80 | policy_arn = aws_iam_policy.ecs_task_execution.arn 81 | } 82 | 83 | data "aws_iam_policy" "ecs_task_execution" { 84 | arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" 85 | } 86 | 87 | # CloudWatch Events 88 | resource "aws_iam_role" "cw_events" { 89 | name = local.cw_events_role_name 90 | assume_role_policy = data.aws_iam_policy_document.cw_events_assume_role_policy.json 91 | } 92 | 93 | data "aws_iam_policy_document" "cw_events_assume_role_policy" { 94 | statement { 95 | actions = ["sts:AssumeRole"] 96 | 97 | principals { 98 | type = "Service" 99 | identifiers = ["events.amazonaws.com"] 100 | } 101 | } 102 | } 103 | 104 | resource "aws_iam_policy" "cw_events" { 105 | name = local.cw_events_policy_name 106 | policy = data.aws_iam_policy.cw_events.policy 107 | } 108 | 109 | resource "aws_iam_role_policy_attachment" "cw_events" { 110 | role = aws_iam_role.cw_events.name 111 | policy_arn = aws_iam_policy.cw_events.arn 112 | } 113 | 114 | data "aws_iam_policy" "cw_events" { 115 | arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceEventsRole" 116 | } 117 | 118 | locals { 119 | bucket_write_policy_name = "${var.aws_recon_base_name}-bucket-write-policy" 120 | ecs_task_execution_role_name = "${var.aws_recon_base_name}-ecs-task-execution-role" 121 | ecs_task_execution_policy_name = "${var.aws_recon_base_name}-ecs-task-execution-policy" 122 | cw_events_policy_name = "${var.aws_recon_base_name}-cw-events-policy" 123 | cw_events_role_name = "${var.aws_recon_base_name}-cw-events-role" 124 | aws_recon_task_role_name = "${var.aws_recon_base_name}-exec-role" 125 | } 126 | -------------------------------------------------------------------------------- /utils/terraform/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | aws = { 4 | source = "hashicorp/aws" 5 | version = "~> 3.0" 6 | } 7 | } 8 | } 9 | 10 | # Configure the AWS Provider 11 | provider "aws" { 12 | region = "us-east-2" 13 | } 14 | -------------------------------------------------------------------------------- /utils/terraform/output.tf: -------------------------------------------------------------------------------- 1 | output "aws_recon_ecs_cluster" { 2 | value = aws_ecs_cluster.aws_recon.name 3 | } 4 | 5 | output "aws_recon_ecs_scheduled_task" { 6 | value = aws_cloudwatch_event_rule.default.name 7 | } 8 | 9 | output "aws_recon_s3_bucket" { 10 | value = aws_s3_bucket.aws_recon.bucket 11 | } 12 | 13 | output "aws_recon_task_manual_run_command" { 14 | value = "\nOne-off task run command:\n\naws ecs run-task --task-definition ${aws_ecs_task_definition.aws_recon_task.family} --cluster ${aws_ecs_cluster.aws_recon.name} --launch-type FARGATE --network-configuration \"awsvpcConfiguration={subnets=[${aws_subnet.subnet.id}],securityGroups=[${aws_security_group.sg.id}],assignPublicIp=ENABLED}\"\n" 15 | } 16 | -------------------------------------------------------------------------------- /utils/terraform/readme.md: -------------------------------------------------------------------------------- 1 | ## Terraform Setup 2 | 3 | This is an example module that can be used in its current form or modified for your specific environment. It builds the minimum components necessary to collect inventory on a schedule running AWS Recon as a Fargate scheduled task. 4 | 5 | ### Requirements 6 | 7 | Before running this Terraform module, adjust your region accordingly in `main.tf`. 8 | 9 | ### What is created? 10 | 11 | This Terraform example will deploy the following resources: 12 | 13 | - an S3 bucket to store compressed JSON output files 14 | - an IAM role for ECS task execution 15 | - a Security Group for the ECS cluster/task 16 | - a VPC and NGW for the ECS cluster/task 17 | - an ECS/Fargate cluster 18 | - an ECS task definition to run AWS Recon collection 19 | - a CloudWatch event rule to trigger the ECS task 20 | - a CloudTrail log group for ECS task logs 21 | -------------------------------------------------------------------------------- /utils/terraform/s3.tf: -------------------------------------------------------------------------------- 1 | resource "aws_s3_bucket" "aws_recon" { 2 | bucket = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}-${data.aws_iam_account_alias.current.id}" 3 | acl = "private" 4 | force_destroy = true 5 | 6 | lifecycle_rule { 7 | id = "expire-after-${var.retention_period}-days" 8 | enabled = true 9 | 10 | expiration { 11 | days = var.retention_period 12 | } 13 | } 14 | } 15 | 16 | resource "random_id" "aws_recon" { 17 | byte_length = 6 18 | } 19 | 20 | data "aws_iam_account_alias" "current" {} 21 | -------------------------------------------------------------------------------- /utils/terraform/vars.tf: -------------------------------------------------------------------------------- 1 | variable "aws_recon_base_name" { 2 | type = string 3 | default = "aws-recon" 4 | } 5 | 6 | variable "aws_recon_container_name" { 7 | type = string 8 | default = "darkbitio/aws_recon" 9 | } 10 | 11 | variable "aws_recon_container_version" { 12 | type = string 13 | default = "latest" 14 | } 15 | 16 | variable "aws_regions" { 17 | type = list(any) 18 | default = [ 19 | "global", 20 | # "af-south-1", 21 | # "ap-east-1", 22 | # "ap-northeast-1", 23 | # "ap-northeast-2", 24 | # "ap-northeast-3", 25 | # "ap-south-1", 26 | # "ap-southeast-1", 27 | # "ap-southeast-2", 28 | # "ca-central-1", 29 | # "eu-central-1", 30 | # "eu-north-1", 31 | # "eu-south-1", 32 | # "eu-west-1", 33 | # "eu-west-2", 34 | # "eu-west-3", 35 | # "me-south-1", 36 | # "sa-east-1", 37 | "us-east-1", 38 | "us-east-2", 39 | "us-west-1", 40 | "us-west-2", 41 | ] 42 | } 43 | 44 | # must be one of: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365 45 | variable "retention_period" { 46 | type = number 47 | default = 30 48 | } 49 | 50 | variable "schedule_expression" { 51 | type = string 52 | default = "cron(4 * * * ? *)" 53 | } 54 | 55 | variable "base_subnet_cidr" { 56 | type = string 57 | default = "10.76.0.0/16" 58 | } 59 | -------------------------------------------------------------------------------- /utils/terraform/vpc.tf: -------------------------------------------------------------------------------- 1 | 2 | # Create a VPC 3 | resource "aws_vpc" "vpc" { 4 | cidr_block = local.cidr_block 5 | tags = { 6 | Name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}" 7 | } 8 | } 9 | 10 | # Create subnet 11 | resource "aws_subnet" "subnet" { 12 | vpc_id = aws_vpc.vpc.id 13 | cidr_block = local.subnet_cidr_block 14 | availability_zone = data.aws_availability_zones.available.names[0] 15 | 16 | tags = { 17 | Name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}-public" 18 | } 19 | } 20 | 21 | resource "aws_security_group" "sg" { 22 | name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}" 23 | description = "Allow AWS Recon collection egress" 24 | vpc_id = aws_vpc.vpc.id 25 | 26 | egress { 27 | from_port = 0 28 | to_port = 0 29 | protocol = "-1" 30 | cidr_blocks = ["0.0.0.0/0"] 31 | } 32 | 33 | tags = { 34 | Name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}" 35 | } 36 | } 37 | 38 | resource "aws_internet_gateway" "igw" { 39 | vpc_id = aws_vpc.vpc.id 40 | 41 | tags = { 42 | Name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}" 43 | } 44 | } 45 | 46 | resource "aws_route_table" "rt" { 47 | vpc_id = aws_vpc.vpc.id 48 | 49 | route { 50 | cidr_block = "0.0.0.0/0" 51 | gateway_id = aws_internet_gateway.igw.id 52 | } 53 | 54 | tags = { 55 | Name = "${var.aws_recon_base_name}-${random_id.aws_recon.hex}" 56 | } 57 | } 58 | 59 | resource "aws_route_table_association" "rt_association" { 60 | subnet_id = aws_subnet.subnet.id 61 | route_table_id = aws_route_table.rt.id 62 | } 63 | 64 | locals { 65 | cidr_block = var.base_subnet_cidr 66 | subnet_cidr_block = cidrsubnet(local.cidr_block, 8, 0) 67 | } 68 | 69 | data "aws_region" "current" {} 70 | 71 | data "aws_availability_zones" "available" { 72 | state = "available" 73 | } 74 | --------------------------------------------------------------------------------