├── cloudtrail_anomaly ├── aws │ ├── __init__.py │ ├── s3.py │ ├── orgs.py │ ├── iam.py │ └── athena.py ├── __init__.py ├── __about__.py └── cli.py ├── requirements.txt ├── setup.py ├── .gitignore └── README.md /cloudtrail_anomaly/aws/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | click 3 | click-log 4 | cloudaux 5 | iso8601 6 | PyYAML 7 | pytz 8 | -------------------------------------------------------------------------------- /cloudtrail_anomaly/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | log = logging.getLogger('cloudtrail-anomaly') 4 | -------------------------------------------------------------------------------- /cloudtrail_anomaly/aws/s3.py: -------------------------------------------------------------------------------- 1 | from cloudtrail_anomaly import log 2 | 3 | 4 | def read_data_from_s3(bucket, key, cloudaux=None): 5 | """Read the s3 file""" 6 | log.debug('Downloading and reading S3 file s3://{}/{}'.format(bucket, key)) 7 | s3_response_object = cloudaux.call( 8 | 's3.client.get_object', 9 | Bucket=bucket, 10 | Key=key) 11 | 12 | data = s3_response_object['Body'].read() 13 | 14 | return data.decode('utf-8').replace('"', '').split('\n') 15 | -------------------------------------------------------------------------------- /cloudtrail_anomaly/__about__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | 3 | __all__ = [ 4 | "__title__", "__summary__", "__uri__", "__version__", "__author__", 5 | "__email__", "__license__", "__copyright__", 6 | ] 7 | 8 | __title__ = "cloudtrail-anomaly" 9 | __summary__ = ("CloudTrail Anomaly") 10 | __uri__ = "https://github.com/netflix-skunkworks/cloudtrail-anomaly" 11 | 12 | __version__ = "0.0.1" 13 | 14 | __author__ = "Will Bengtson" 15 | 16 | __license__ = "Apache License, Version 2.0" 17 | __copyright__ = "Copyright 2019 {0}".format(__author__) 18 | -------------------------------------------------------------------------------- /cloudtrail_anomaly/aws/orgs.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from botocore.exceptions import ClientError 3 | 4 | from cloudtrail_anomaly import log 5 | 6 | 7 | def get_accounts_from_orgs(cloudaux=None): 8 | """Return the account""" 9 | 10 | log.info('Getting accounts from organizations') 11 | response = cloudaux.call( 12 | 'organizations.client.list_accounts' 13 | ) 14 | 15 | accounts = [] 16 | while True: 17 | next_token = response.get('NextToken', None) 18 | for account in response.get('Accounts', []): 19 | accounts.append(account['Id']) 20 | if next_token: 21 | response = cloudaux.call( 22 | 'organizations.client.list_accounts', 23 | NextToken=next_token) 24 | else: 25 | break 26 | 27 | return accounts 28 | -------------------------------------------------------------------------------- /cloudtrail_anomaly/aws/iam.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from botocore.exceptions import ClientError 3 | 4 | from cloudtrail_anomaly import log 5 | 6 | 7 | def get_roles_in_account(cloudaux=None): 8 | """Return the roles in the account""" 9 | 10 | log.info('Getting roles from {}'.format(cloudaux.conn_details['account_number'])) 11 | response = cloudaux.call( 12 | 'iam.client.list_roles', 13 | PathPrefix='/', 14 | MaxItems=100) 15 | 16 | roles = {} 17 | while True: 18 | next_token = response.get('Marker', None) 19 | for role in response.get('Roles', []): 20 | roles[role['Arn']] = role 21 | if next_token: 22 | response = cloudaux.call( 23 | 'iam.client.list_roles', 24 | MaxItems=100, 25 | Marker=next_token) 26 | else: 27 | break 28 | 29 | return roles 30 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Example Python application, using the paved path.""" 2 | 3 | try: # for pip >= 10 4 | from pip._internal.req import parse_requirements 5 | except ImportError: # for pip <= 9.0.3 6 | from pip.req import parse_requirements 7 | try: # for pip >= 10 8 | from pip._internal.download import PipSession 9 | except ImportError: # for pip <= 9.0.3 10 | from pip.download import PipSession 11 | 12 | from setuptools import setup 13 | 14 | 15 | # Gather install requirements from requirements.txt 16 | install_reqs = parse_requirements('requirements.txt', session=PipSession()) 17 | install_requires = [str(ir.req) for ir in install_reqs] 18 | 19 | 20 | setup( 21 | name='cloudtrail_anomaly', 22 | versioning='build-id', 23 | author='Will Bengtson', 24 | keywords='cloudtrail', 25 | url='https://github.com/netflix-skunkworks/cloudtrail-anomaly', 26 | setup_requires=['setupmeta'], 27 | python_requires='>=3.6', 28 | install_requires=install_requires, 29 | entry_points={ 30 | 'console_scripts': [ 31 | 'ct_anomaly = cloudtrail_anomaly.cli:cli' 32 | ] 33 | } 34 | ) 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.egg-info 3 | *.py[cod] 4 | .cache 5 | .discovery.lock 6 | .eggs 7 | .python-version 8 | .setupmeta.version 9 | .tox 10 | .venv 11 | 12 | .idea/ 13 | __pycache__/ 14 | build/ 15 | dist/ 16 | test-reports/ 17 | # Byte-compiled / optimized / DLL files 18 | __pycache__/ 19 | *.py[cod] 20 | *$py.class 21 | 22 | # C extensions 23 | *.so 24 | 25 | # Distribution / packaging 26 | .Python 27 | build/ 28 | develop-eggs/ 29 | dist/ 30 | downloads/ 31 | eggs/ 32 | .eggs/ 33 | lib/ 34 | lib64/ 35 | parts/ 36 | sdist/ 37 | var/ 38 | wheels/ 39 | *.egg-info/ 40 | .installed.cfg 41 | *.egg 42 | MANIFEST 43 | 44 | # PyInstaller 45 | # Usually these files are written by a python script from a template 46 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 47 | *.manifest 48 | *.spec 49 | 50 | # Installer logs 51 | pip-log.txt 52 | pip-delete-this-directory.txt 53 | 54 | # Unit test / coverage reports 55 | htmlcov/ 56 | .tox/ 57 | .coverage 58 | .coverage.* 59 | .cache 60 | nosetests.xml 61 | coverage.xml 62 | *.cover 63 | .hypothesis/ 64 | .pytest_cache/ 65 | 66 | # Translations 67 | *.mo 68 | *.pot 69 | 70 | # Django stuff: 71 | *.log 72 | local_settings.py 73 | db.sqlite3 74 | 75 | # Flask stuff: 76 | instance/ 77 | .webassets-cache 78 | 79 | # Scrapy stuff: 80 | .scrapy 81 | 82 | # Sphinx documentation 83 | docs/_build/ 84 | 85 | # PyBuilder 86 | target/ 87 | 88 | # Jupyter Notebook 89 | .ipynb_checkpoints 90 | 91 | # pyenv 92 | .python-version 93 | 94 | # celery beat schedule file 95 | celerybeat-schedule 96 | 97 | # SageMath parsed files 98 | *.sage.py 99 | 100 | # Environments 101 | .env 102 | .venv 103 | env/ 104 | venv/ 105 | ENV/ 106 | env.bak/ 107 | venv.bak/ 108 | 109 | # Spyder project settings 110 | .spyderproject 111 | .spyproject 112 | 113 | # Rope project settings 114 | .ropeproject 115 | 116 | # mkdocs documentation 117 | /site 118 | 119 | # mypy 120 | .mypy_cache/ 121 | 122 | config.yml 123 | -------------------------------------------------------------------------------- /cloudtrail_anomaly/aws/athena.py: -------------------------------------------------------------------------------- 1 | import re 2 | import time 3 | 4 | from cloudtrail_anomaly import log 5 | 6 | 7 | def query_athena(config, query, cloudaux=None, max_execution=20): 8 | """Run Athena Query""" 9 | response = cloudaux.call( 10 | 'athena.client.start_query_execution', 11 | QueryString=query, 12 | QueryExecutionContext={ 13 | 'Database': config.get('athena', {}).get('database', 'default') 14 | }, 15 | ResultConfiguration={ 16 | 'OutputLocation': 's3://' + config['aws']['athena']['bucket'] + '/' + config.get('aws', {}).get('athena', {}).get('prefix', 'cloudtrail_anomaly') 17 | } 18 | ) 19 | 20 | execution_id = response['QueryExecutionId'] 21 | 22 | state = 'RUNNING' 23 | 24 | # wait 2 mins to run query before moving on 25 | max_execution_timeout = 120 26 | 27 | while (max_execution > 0 and state in ['RUNNING']): 28 | max_execution = max_execution - 1 29 | response = cloudaux.call( 30 | 'athena.client.get_query_execution', 31 | QueryExecutionId = execution_id 32 | ) 33 | 34 | if 'QueryExecution' in response and \ 35 | 'Status' in response['QueryExecution'] and \ 36 | 'State' in response['QueryExecution']['Status']: 37 | state = response['QueryExecution']['Status']['State'] 38 | if state == 'FAILED': 39 | return False 40 | elif state == 'SUCCEEDED': 41 | s3_path = response['QueryExecution']['ResultConfiguration']['OutputLocation'] 42 | filename = re.findall('.*\/(.*)', s3_path)[0] 43 | return filename 44 | time.sleep(1) 45 | return False 46 | 47 | 48 | def create_table(config, account_number, cloudaux=None, max_execution=20): 49 | 50 | query_string = """CREATE EXTERNAL TABLE IF NOT EXISTS cloudtrail_{account_number} ( 51 | eventVersion STRING, 52 | userIdentity STRUCT< 53 | type: STRING, 54 | principalId: STRING, 55 | arn: STRING, 56 | accountId: STRING, 57 | invokedBy: STRING, 58 | accessKeyId: STRING, 59 | userName: STRING, 60 | sessionContext: STRUCT< 61 | attributes: STRUCT< 62 | mfaAuthenticated: STRING, 63 | creationDate: STRING>, 64 | sessionIssuer: STRUCT< 65 | type: STRING, 66 | principalId: STRING, 67 | arn: STRING, 68 | accountId: STRING, 69 | userName: STRING>>>, 70 | eventTime STRING, 71 | eventSource STRING, 72 | eventName STRING, 73 | awsRegion STRING, 74 | sourceIpAddress STRING, 75 | userAgent STRING, 76 | errorCode STRING, 77 | errorMessage STRING, 78 | requestParameters STRING, 79 | responseElements STRING, 80 | additionalEventData STRING, 81 | requestId STRING, 82 | eventId STRING, 83 | resources ARRAY>, 87 | eventType STRING, 88 | apiVersion STRING, 89 | readOnly STRING, 90 | recipientAccountId STRING, 91 | serviceEventDetails STRING, 92 | sharedEventID STRING, 93 | vpcEndpointId STRING 94 | ) 95 | COMMENT 'CloudTrail table for {ct_bucket}' 96 | ROW FORMAT SERDE 'com.amazon.emr.hive.serde.CloudTrailSerde' 97 | STORED AS INPUTFORMAT 'com.amazon.emr.cloudtrail.CloudTrailInputFormat' 98 | OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' 99 | LOCATION 's3://{ct_bucket}/AWSLogs/{account_number}/CloudTrail/' 100 | TBLPROPERTIES ('classification'='cloudtrail');""" 101 | 102 | athena_query = query_string.format( 103 | account_number=account_number, 104 | ct_bucket=config.get('aws', {}).get('athena', {}).get('cloudtrailBucket', 'cloudtrailbucket')) 105 | 106 | file_name = query_athena(config, athena_query, cloudaux) 107 | 108 | return file_name 109 | -------------------------------------------------------------------------------- /cloudtrail_anomaly/cli.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | import time 4 | import yaml 5 | 6 | import boto3 7 | from cloudaux import CloudAux 8 | import click 9 | import click_log 10 | import iso8601 11 | import pytz 12 | 13 | from cloudtrail_anomaly import log 14 | from cloudtrail_anomaly.__about__ import __version__ 15 | from cloudtrail_anomaly.aws.orgs import get_accounts_from_orgs 16 | from cloudtrail_anomaly.aws.iam import get_roles_in_account 17 | from cloudtrail_anomaly.aws.athena import query_athena, create_table 18 | from cloudtrail_anomaly.aws.s3 import read_data_from_s3 19 | 20 | click_log.basic_config(log) 21 | 22 | 23 | class YAML(click.ParamType): 24 | name = 'yaml' 25 | def convert(self, value, param, ctx): 26 | try: 27 | with open(value, 'rb') as f: 28 | return yaml.load(f.read(), Loader=yaml.SafeLoader) 29 | except (IOError, OSError) as e: 30 | self.fail('Could not open file: {0}'.format(value)) 31 | 32 | 33 | class CommaList(click.ParamType): 34 | name = 'commalist' 35 | def convert(self, value, param, ctx): 36 | return value.split(',') 37 | 38 | class AppContext(object): 39 | def __init__(self): 40 | self.config = None 41 | 42 | 43 | pass_context = click.make_pass_decorator(AppContext, ensure=True) 44 | 45 | 46 | @click.group() 47 | @click_log.simple_verbosity_option(log) 48 | @click.version_option(version=__version__) 49 | @click.option('--config', type=YAML(), help='Configuration file to use.') 50 | @pass_context 51 | def cli(ctx, config): 52 | if not ctx.config: 53 | ctx.config = config 54 | log.debug('Current context. Config: {}'.format(json.dumps(ctx.config, indent=2))) 55 | 56 | 57 | @cli.group() 58 | def detect(): 59 | pass 60 | 61 | @cli.group() 62 | def setup(): 63 | pass 64 | 65 | 66 | @detect.command() 67 | @click.option('--accounts', type=CommaList(), help='Comma separated list of AWS accounts') 68 | @pass_context 69 | def anomaly(ctx, accounts): 70 | """Detect anomalies in CloudTrail""" 71 | 72 | conn_details = { 73 | 'account_number': ctx.config.get('aws', {}).get('organizations', {}).get('accountId', '123'), 74 | 'assume_role': ctx.config.get('aws', {}).get('organizations', {}).get('roleName', '123'), 75 | 'session_name': 'cloudtrail-anomaly' 76 | } 77 | 78 | ca = CloudAux(**conn_details) 79 | 80 | if not accounts: 81 | accounts = get_accounts_from_orgs(ca) 82 | log.info('Received {} accounts from Organizations'.format(len(accounts))) 83 | else: 84 | log.info('Received {} accounts from command line'.format(len(accounts))) 85 | 86 | conn_details['assume_role'] = ctx.config.get('aws', {}).get('iam', {}).get('roleName', '123') 87 | 88 | conn_details_athena = { 89 | 'account_number': ctx.config['aws']['athena']['accountId'], 90 | 'assume_role': ctx.config['aws']['athena']['roleName'], 91 | 'session_name': 'cloudtrail-anomaly', 92 | 'region': ctx.config.get('aws', {}).get('region', 'us-east-1') 93 | } 94 | ca_athena = CloudAux(**conn_details_athena) 95 | 96 | dynamodb = boto3.resource('dynamodb', region_name=ctx.config.get('aws', {}).get('region', 'us-east-1')) 97 | dynamo_table = dynamodb.Table(ctx.config.get('aws', {}).get('dynamoTableName', 'cloudtrail_anomaly')) 98 | 99 | sns_topic = boto3.client('sns', region_name=ctx.config.get('aws', {}).get('region', 'us-east-1')) 100 | 101 | for account in accounts: 102 | conn_details['account_number'] = account 103 | 104 | ca = CloudAux(**conn_details) 105 | 106 | roles = get_roles_in_account(ca) 107 | 108 | for role in roles: 109 | role_name = roles[role]['RoleName'] 110 | new_ttl = int(time.mktime((datetime.datetime.now() + datetime.timedelta(days=ctx.config['roleAction']['dayThreshold'])).timetuple())) 111 | principal_id = roles[role]['RoleId'] 112 | athena_query = "SELECT DISTINCT eventsource, eventname FROM cloudtrail_{} WHERE useridentity.type = 'AssumedRole' AND useridentity.sessioncontext.sessionissuer.principalid= '{}' AND eventTime > to_iso8601(current_timestamp - interval '1' hour);".format(account, principal_id) 113 | 114 | log.info('Running Athena Query for {} in {}'.format(roles[role]['RoleName'], account)) 115 | file_name = query_athena(ctx.config, athena_query, ca_athena) 116 | 117 | if not file_name: 118 | log.error("Execution failed or timed out") 119 | continue 120 | 121 | s3_key = ctx.config.get('aws', {}).get('athena', {}).get('prefix', 'cloudtrail_anomaly') + '/' + file_name 122 | 123 | data = read_data_from_s3(ctx.config['aws']['athena']['bucket'], s3_key, ca_athena) 124 | 125 | role_actions = [] 126 | 127 | # Remove the header and loop through calls in last hour: 128 | for call in data[1:]: 129 | service_pair = call.split(',') 130 | service_action = ':'.join(service_pair) 131 | if len(service_action) == 0: 132 | continue 133 | 134 | log.debug('Checking DynamoDB for never seen before actions on {} in {}'.format(role_name, account)) 135 | key = {'RoleId': principal_id, 'Action': service_action} 136 | response = dynamo_table.get_item(Key=key) 137 | 138 | if response and 'Item' in response: 139 | dynamo_table.update_item(Key=key, 140 | UpdateExpression='SET #ttl = :ttl', 141 | ExpressionAttributeNames={'#ttl': 'TTL'}, 142 | ExpressionAttributeValues={':ttl': new_ttl}) 143 | else: 144 | # keep track of which actions are new 145 | if service_action not in ctx.config['roleAction'].get('IgnoredActionsNotify', []): 146 | role_actions.append(service_action) 147 | log.info('Newly seen action: {} - {} in {}'.format(role_name, service_action, account)) 148 | dynamo_table.put_item(Item={'RoleId': principal_id, 149 | 'Action': service_action, 150 | 'TTL': new_ttl}) 151 | 152 | if len(role_actions) > 0: 153 | arn = role 154 | create_date = roles[role]['CreateDate'] 155 | role_name = roles[role]['RoleName'] 156 | skip_alert = False 157 | 158 | # if the role is too new, don't alert 159 | if create_date > datetime.datetime.now(pytz.utc) - datetime.timedelta(days=ctx.config['roleAction']['dayThreshold']): 160 | skip_alert = True 161 | log.debug('{} in {} is too new, skipping alert'.format(role_name, account)) 162 | # if the role is a service role, don't alert 163 | if 'aws-service-role' in arn.split('/'): 164 | skip_alert = True 165 | log.debug('{} in {} is an AWS service role, skipping alert'.format(role_name, account)) 166 | 167 | if not skip_alert: 168 | log.info('Sending alert for new actions for {} in {}'.format(role_name, account)) 169 | alert = { 170 | 'actions': ', '.join(action for action in role_actions), 171 | 'role': role_name, 172 | 'account': account 173 | } 174 | sns_topic.publish( 175 | TopicArn=ctx.config['aws']['snsTopicArn'], 176 | Message=json.dumps(alert) 177 | ) 178 | 179 | 180 | @setup.command() 181 | @click.option('--accounts', type=CommaList(), help='Comma separated list of AWS accounts') 182 | @pass_context 183 | def athena(ctx, accounts): 184 | """Setup athena tables.""" 185 | 186 | conn_details = { 187 | 'account_number': ctx.config.get('aws', {}).get('organizations', {}).get('accountId', '123'), 188 | 'assume_role': ctx.config.get('aws', {}).get('organizations', {}).get('roleName', '123'), 189 | 'session_name': 'cloudtrail-anomaly' 190 | } 191 | 192 | ca = CloudAux(**conn_details) 193 | 194 | conn_details_athena = { 195 | 'account_number': ctx.config['aws']['athena']['accountId'], 196 | 'assume_role': ctx.config['aws']['athena']['roleName'], 197 | 'session_name': 'cloudtrail-anomaly', 198 | 'region': ctx.config.get('aws', {}).get('region', 'us-east-1') 199 | } 200 | ca_athena = CloudAux(**conn_details_athena) 201 | 202 | if not accounts: 203 | accounts = get_accounts_from_orgs(ca) 204 | log.info('Received {} accounts from Organizations'.format(len(accounts))) 205 | else: 206 | log.info('Received {} accounts from command line'.format(len(accounts))) 207 | 208 | for account in accounts: 209 | file_name = create_table(ctx.config, accounts[0], ca_athena) 210 | 211 | if not file_name: 212 | log.error("Execution failed or timed out for account {}".format(account)) 213 | continue 214 | else: 215 | log.info("Successfully created Athena table for account {}".format(account)) 216 | 217 | log.info('Successfully created Athena tables') 218 | 219 | 220 | if __name__ == '__main__': 221 | cli() 222 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CloudTrail Application Anomaly Detection 2 | 3 | This project is a simple CloudTrail based anomaly detection for use in AWS. It keeps track of all API actions a principal calls (that are tracked by CloudTrail) for a N day period and alerts on new API calls after the N day period. 4 | 5 | ## Disclaimer 6 | 7 | We are releasing this as a proof-of-concept with no intent to provide support. Please use it as a starting point for your own work in this space. 8 | 9 | ## Getting Started 10 | 11 | To get started you will need the following resources created in your AWS environment: 12 | 13 | - DynamoDB Table to keep API actions 14 | - Athena Tables (1 per account you want to track CloudTrail anomalies for roles) 15 | - SNS Topic to alert to when an anomaly is detected 16 | 17 | ### Prerequisites 18 | 19 | You must centralize CloudTrail to a single S3 bucket for Athena to query. The code is configured to query for any API calls from the last `1` hour. This means you should run updates at least this frequently otherwise you may miss events that happened since last update but outside of the `1` hour window. You can adjust these parameters as it makes sense in your environment, but generally more frequent updates are better because you get quicker notifications. 20 | 21 | ### Create DynamoDB Table 22 | 23 | The DynamoDB table should be created in the account and region you wish to run this tool in. 24 | 25 | Create a table with a `Partition Key` called `RoleId` and a `Sort Key` called `Action`. You must also enable TTL for a field called `TTL`. 26 | 27 | We recommend allowing AWS to autoscale your DynamoDB and you most likely will need to play around with read and write capacity as your roll this out. 28 | 29 | ### Create SNS Topic 30 | 31 | Create a SNS topic in the account and region you wish to run this tool in. It can also be in another account, but you will need to setup a cross account SNS resource policy if you do. Note this ARN of the SNS topic for use in creating your IAM role later. 32 | 33 | ### IAM Roles Creation 34 | 35 | Create the following roles in your accounts: 36 | 37 | - `ctanomalyInstanceProfile` in the account you wish to run this tool from 38 | - `CT_OrgsRole` in your AWS Organizations parent account 39 | - `CT_IamReadOnly` in every account you wish to run anomaly detection against with CloudTrail 40 | - `CT_AthenaRole` in the account where you have CloudTrail centralized to a S3 bucket 41 | 42 | #### CloudTrail Anomaly Application Role 43 | 44 | This is the role that the code will run as from any AWS account you choose. This can be a completely new AWS account (recommended), the account you centralize CloudTrail to, or any other AWS account in your enterprise. 45 | 46 | Role Name: `ctanomalyInstanceProfile` 47 | 48 | Inline Policy: 49 | 50 | ```json 51 | { 52 | "Version": "2012-10-17", 53 | "Statement": [ 54 | { 55 | "Action": [ 56 | "sts:AssumeRole" 57 | ], 58 | "Effect": "Allow", 59 | "Resource": [ 60 | "arn:aws:iam::ORGS_ACCOUNT_NUMBER:role/CT_OrgsRole", 61 | "arn:aws:iam::*:role/CT_IamReadOnly", 62 | "arn:aws:iam::ACCOUNT_WHERE_CLOUDTRAILBUCKET_IS:role/CT_AthenaRole" 63 | ] 64 | }, 65 | { 66 | "Action": [ 67 | "dynamodb:PutItem", 68 | "dynamodb:Query", 69 | "dynamodb:UpdateItem" 70 | ], 71 | "Effect": "Allow", 72 | "Resource": "*" 73 | }, 74 | { 75 | "Action": [ 76 | "sns:Publish" 77 | ], 78 | "Effect": "Allow", 79 | "Resource": "arn:aws:sns:REGION:ACCOUNT_NUMBER:cloudtrail_anomaly" 80 | } 81 | ] 82 | } 83 | ``` 84 | 85 | Trust Policy: 86 | 87 | ```json 88 | { 89 | "Version": "2012-10-17", 90 | "Statement": [ 91 | { 92 | "Effect": "Allow", 93 | "Principal": { 94 | "Service": "ec2.amazonaws.com" 95 | }, 96 | "Action": "sts:AssumeRole" 97 | } 98 | ] 99 | } 100 | ``` 101 | 102 | #### Organization Role 103 | 104 | If you want to be automatically look at roles in all of your AWS Organization accounts, you must provide a role with a trust relationship to the CloudTrail Anomaly Application Role above. 105 | 106 | Role Name: `CT_OrgsRole` 107 | 108 | Inline Policy: 109 | 110 | ```json 111 | { 112 | "Version": "2012-10-17", 113 | "Statement": [ 114 | { 115 | "Action": [ 116 | "organizations:ListAccounts" 117 | ], 118 | "Effect": "Allow", 119 | "Resource": "*" 120 | } 121 | ] 122 | } 123 | ``` 124 | 125 | Trust Policy: 126 | 127 | ```json 128 | { 129 | "Version": "2012-10-17", 130 | "Statement": [ 131 | { 132 | "Effect": "Allow", 133 | "Principal": { 134 | "AWS": "arn:aws:iam::ACCOUNT_NUMBER:role/ctanomalyInstanceProfile" 135 | }, 136 | "Action": "sts:AssumeRole", 137 | "Condition": {} 138 | } 139 | ] 140 | } 141 | ``` 142 | 143 | #### IAM Read Only Role 144 | 145 | The IAM read only role lists all the roles in each AWS account to know which roles to track in CloudTrail. 146 | 147 | Role Name: `CT_IamReadOnly` 148 | 149 | Inline Policy: 150 | 151 | ```json 152 | { 153 | "Version": "2012-10-17", 154 | "Statement": [ 155 | { 156 | "Effect": "Allow", 157 | "Action": "iam:ListRoles", 158 | "Resource": "*" 159 | } 160 | ] 161 | } 162 | ``` 163 | 164 | Trust Policy: 165 | 166 | ```json 167 | { 168 | "Version": "2012-10-17", 169 | "Statement": [ 170 | { 171 | "Effect": "Allow", 172 | "Principal": { 173 | "AWS": "arn:aws:iam::ACCOUNT_NUMBER:role/ctanomalyInstanceProfile" 174 | }, 175 | "Action": "sts:AssumeRole", 176 | "Condition": {} 177 | } 178 | ] 179 | } 180 | ``` 181 | 182 | #### Athena Role 183 | 184 | This role is used to make the Athena queries. It needs access to the Athena service, Glue service, and access to the S3 bucket where CloudTrail is stored. 185 | 186 | Role Name: `CT_AthenaRole` 187 | 188 | Easiest way to get started is to attach the `AmazonAthenaFullAccess` managed policy. 189 | 190 | `arn:aws:iam::aws:policy/AmazonAthenaFullAccess` 191 | 192 | In addition to this, you will need access to the CloudTrail bucket. 193 | 194 | Inline Policy: 195 | 196 | ```json 197 | { 198 | "Version": "2012-10-17", 199 | "Statement": [ 200 | { 201 | "Effect": "Allow", 202 | "Action": [ 203 | "s3:GetObject", 204 | "s3:ListBucket" 205 | ], 206 | "Resource": [ 207 | "arn:aws:s3:::CLOUDTRAIL_BUCKET_HERE", 208 | "arn:aws:s3:::CLOUDTRAIL_BUCKET_HERE/*" 209 | ] 210 | } 211 | ] 212 | } 213 | ``` 214 | 215 | Trust Policy: 216 | 217 | ```json 218 | { 219 | "Version": "2012-10-17", 220 | "Statement": [ 221 | { 222 | "Effect": "Allow", 223 | "Principal": { 224 | "AWS": "arn:aws:iam::ACCOUNT_NUMBER:role/ctanomalyInstanceProfile" 225 | }, 226 | "Action": "sts:AssumeRole", 227 | "Condition": {} 228 | } 229 | ] 230 | } 231 | ``` 232 | 233 | ### Athena Table Creation 234 | 235 | To create the necessary Athena tables run the following command: 236 | 237 | `ct_anomaly --verbosity INFO --config config.yml setup athena` 238 | 239 | alternatively you can choose to create the Athena tables for only a subset of your accounts with the following command: 240 | 241 | `ct_anomaly --verbosity INFO --config config.yml setup athena --accounts 1234567890,0987654321` 242 | 243 | where your account numbers are comma delimited. 244 | 245 | ### Config File Example 246 | 247 | ```yaml 248 | roleAction: 249 | dayThreshold: 90 250 | # IgnoredActionsNotify: 251 | # - sts.amazonaws.com:GetCallerIdentity 252 | aws: 253 | region: us-west-2 254 | dynamoTableName: cloudtrail_anomaly 255 | snsTopicArn: arn:aws:sns:us-west-2:1234567890:cloudtrail_anomaly 256 | organizations: 257 | accountId: 1234567890 258 | roleName: CT_OrgsRole 259 | iam: 260 | roleName: CT_IamReadOnly 261 | athena: 262 | database: default 263 | accountId: 0987654321 264 | roleName: CT_AthenaRole 265 | bucket: aws-athena-query-results-0987654321-us-west-2 266 | prefix: cloudtrailbucket 267 | cloudtrailBucket: cloudtrailbucket 268 | ``` 269 | 270 | ## Detecting Anomalies 271 | 272 | To detect anomalies for roles in all accounts, run the following command: 273 | 274 | `ct_anomaly --verbosity INFO --config config.yml detect anomaly` 275 | 276 | To detect anomalies for roles in a subset of accounts, run the following command: 277 | 278 | `ct_anomaly --verbosity INFO --config config.yml detect anomaly --accounts 1234567890,0987654321` 279 | 280 | where your account numbers are comma delimited. 281 | 282 | ### Example Output 283 | 284 | ``` 285 | ct_anomaly --verbosity INFO --config config.yml detect anomaly 286 | Getting accounts from organizations 287 | Received 25 accounts from Organizations 288 | Getting roles from 1234567890 289 | Running Athena Query for AWSServiceRoleForOrganizations in 1234567890 290 | Running Athena Query for AWSServiceRoleForSupport in 1234567890 291 | Running Athena Query for AWSServiceRoleForTrustedAdvisor in 1234567890 292 | Running Athena Query for CT_AthenaRole in 1234567890 293 | Newly seen action: CT_AthenaRole - athena.amazonaws.com:GetQueryExecution in 1234567890 294 | Running Athena Query for CT_IamReadOnly in 1234567890 295 | Newly seen action: CT_IamReadOnly - iam.amazonaws.com:ListRoles in 1234567890 296 | Running Athena Query for demoLambda in 1234567890 297 | Running Athena Query for demoRole in 1234567890 298 | Running Athena Query for trailblazer in 1234567890 299 | ... 300 | ``` --------------------------------------------------------------------------------