├── .github ├── FUNDING.yml └── workflows │ └── publish.yml ├── CloudWatch2S3-additional-account.template ├── CloudWatch2S3.template ├── LICENSE ├── README.md └── architecture.svg /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: CloudSnorkel 2 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish to AWS Serverless Application Repository 2 | 3 | on: push 4 | 5 | jobs: 6 | publish: 7 | name: Publish to AWS 8 | 9 | runs-on: ubuntu-18.04 10 | 11 | steps: 12 | - uses: actions/checkout@master 13 | - name: Set up Python 3.8 14 | uses: actions/setup-python@v1 15 | with: 16 | python-version: 3.8 17 | - name: Set up AWS CLI 18 | run: sudo pip install awscli 19 | - name: Publish 20 | if: startsWith(github.event.ref, 'refs/tags') 21 | env: 22 | AWS_DEFAULT_REGION: us-east-1 23 | AWS_ACCESS_KEY_ID: ${{ secrets.aws_access_key_id }} 24 | AWS_SECRET_ACCESS_KEY: ${{ secrets.aws_secret_access_key }} 25 | run: | 26 | aws serverlessrepo create-application-version --application-id arn:aws:serverlessrepo:us-east-1:${{ secrets.aws_account }}:applications/CloudWatch2S3-additional-account --semantic-version ${GITHUB_REF##*/} --source-code-url https://github.com/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA} --template-body file://./CloudWatch2S3-additional-account.template 27 | aws serverlessrepo create-application-version --application-id arn:aws:serverlessrepo:us-east-1:${{ secrets.aws_account }}:applications/CloudWatch2S3 --semantic-version ${GITHUB_REF##*/} --source-code-url https://github.com/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA} --template-body file://./CloudWatch2S3.template 28 | -------------------------------------------------------------------------------- /CloudWatch2S3-additional-account.template: -------------------------------------------------------------------------------- 1 | Description: Continuously dump all matching CloudWatch Log groups to a bucket in a 2 | central account for long-term storage (by CloudSnorkel) 3 | Metadata: 4 | AWS::CloudFormation::Interface: 5 | ParameterGroups: 6 | - Label: 7 | default: Target 8 | Parameters: 9 | - LogDestination 10 | - Label: 11 | default: CloudWatch Logs 12 | Parameters: 13 | - SubscribeSchedule 14 | - LogGroupNamePrefix 15 | ParameterLabels: 16 | LogDestination: 17 | default: Log Destination 18 | LogGroupNamePrefix: 19 | default: Required Log Group Name Prefix 20 | SubscribeSchedule: 21 | default: Look for New Logs Schedule 22 | AWS::ServerlessRepo::Application: 23 | Author: CloudSnorkel 24 | Description: Logging source for CloudWatch2S3 from a separate AWS account. Deploy 25 | CloudWatch2S3 to your main account first. 26 | HomePageUrl: https://github.com/CloudSnorkel/CloudWatch2S3 27 | Labels: 28 | - cloudwatch 29 | - s3 30 | - export 31 | LicenseUrl: LICENSE 32 | Name: CloudWatch2S3-additional-account 33 | ReadmeUrl: README.md 34 | SemanticVersion: 1.0.0 35 | SourceCodeUrl: https://github.com/CloudSnorkel/CloudWatch2S3 36 | SpdxLicenseId: MIT 37 | Parameters: 38 | LogDestination: 39 | AllowedPattern: arn:[a-z\-]+:logs:[a-z1-9\-]+:[0-9]+:destination:.* 40 | Description: Log destination ARN from the outputs of the main template 41 | Type: String 42 | LogGroupNamePrefix: 43 | Default: '' 44 | Description: Prefix to match against log group that should be exported (leave 45 | empty to export all log groups) 46 | Type: String 47 | SubscribeSchedule: 48 | Default: rate(1 hour) 49 | Description: Schedule to look for new log groups for export (in case CloudTrail 50 | missed something) 51 | Type: String 52 | Resources: 53 | LogSubscriberFunction: 54 | Properties: 55 | Code: 56 | ZipFile: 57 | Fn::Sub: | 58 | import traceback 59 | 60 | import boto3 61 | import botocore.exceptions 62 | import cfnresponse 63 | 64 | logs_client = boto3.client("logs") 65 | 66 | 67 | def subscribe(log_group_name): 68 | print("Subscribe ", log_group_name) 69 | 70 | if log_group_name.startswith("/aws/lambda/${AWS::StackName}") \ 71 | or log_group_name.startswith("/aws/kinesisfirehose/${AWS::StackName}"): 72 | print("Skipping our log groups to avoid endless recursion") 73 | return 74 | 75 | try: 76 | logs_client.put_subscription_filter( 77 | logGroupName=log_group_name, 78 | filterName="BucketBackupFilter", 79 | filterPattern="", 80 | destinationArn="${LogDestination}", 81 | ) 82 | except logs_client.exceptions.LimitExceededException: 83 | print(f"ERROR: Unable to subscribe to {log_group_name} as it already has an active subscription") 84 | 85 | 86 | def matched_log_groups(prefix): 87 | print(f"Finding all log groups with prefix '{prefix}'") 88 | 89 | log_group_paginator = logs_client.get_paginator("describe_log_groups") 90 | 91 | paginate_params = {} 92 | if prefix: 93 | paginate_params["logGroupNamePrefix"] = prefix 94 | 95 | for log_group_page in log_group_paginator.paginate(**paginate_params): 96 | for log_group in log_group_page["logGroups"]: 97 | yield log_group["logGroupName"] 98 | 99 | 100 | def subscribe_all(): 101 | for log_group_name in matched_log_groups("${LogGroupNamePrefix}"): 102 | subscribe(log_group_name) 103 | 104 | 105 | def unsubscribe_all(): 106 | for log_group_name in matched_log_groups(""): 107 | print("Unsubscribe ", log_group_name) 108 | 109 | try: 110 | logs_client.delete_subscription_filter( 111 | logGroupName=log_group_name, 112 | filterName="BucketBackupFilter", 113 | ) 114 | except botocore.exceptions.ClientError: 115 | pass 116 | 117 | 118 | def handler(event, context): 119 | print('event:', event) 120 | 121 | if "ResponseURL" in event and "RequestType" in event: 122 | # custom resource callback 123 | try: 124 | if event["RequestType"] in ["Create", "Update"]: 125 | print("Subscribe to all new log groups on resource", event["RequestType"]) 126 | subscribe_all() 127 | 128 | elif event["RequestType"] == "Delete": 129 | print("Unsubscribe all on resource", event["RequestType"]) 130 | unsubscribe_all() 131 | 132 | cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, "ok") 133 | 134 | except Exception as e: 135 | try: 136 | traceback.print_last() 137 | except ValueError: 138 | print("Caught exception but unable to print stack trace") 139 | print(e) 140 | cfnresponse.send(event, context, cfnresponse.FAILED, {}, "fail") 141 | 142 | else: 143 | # other call 144 | detail_type = event.get("detail-type") 145 | 146 | if detail_type == "AWS API Call via CloudTrail": 147 | print("Subscribe to specific new log group from CloudTrail") 148 | 149 | request_parameters = event['detail']['requestParameters'] 150 | 151 | if request_parameters: 152 | log_group_name = request_parameters['logGroupName'] 153 | 154 | if log_group_name.startswith("${LogGroupNamePrefix}"): 155 | subscribe(log_group_name) 156 | else: 157 | print(log_group_name, "doesn't match required prefix '${LogGroupNamePrefix}'") 158 | 159 | else: 160 | print("Bad parameters") 161 | 162 | elif detail_type == "Scheduled Event": 163 | print("Subscribe to all new log groups on schedule") 164 | 165 | subscribe_all() 166 | 167 | else: 168 | print("Subscribe to all new log groups") 169 | 170 | subscribe_all() 171 | Handler: index.handler 172 | Role: 173 | Fn::GetAtt: 174 | - LogSubscriberRole 175 | - Arn 176 | Runtime: python3.9 177 | Timeout: 300 178 | Type: AWS::Lambda::Function 179 | LogSubscriberPermission: 180 | Properties: 181 | Action: lambda:InvokeFunction 182 | FunctionName: 183 | Fn::GetAtt: 184 | - LogSubscriberFunction 185 | - Arn 186 | Principal: 187 | Fn::Sub: events.${AWS::URLSuffix} 188 | SourceArn: 189 | Fn::GetAtt: 190 | - LogSubscriberRule 191 | - Arn 192 | Type: AWS::Lambda::Permission 193 | LogSubscriberRole: 194 | Properties: 195 | AssumeRolePolicyDocument: 196 | Statement: 197 | - Action: 198 | - sts:AssumeRole 199 | Effect: Allow 200 | Principal: 201 | Service: 202 | - Fn::Sub: lambda.${AWS::URLSuffix} 203 | Version: '2012-10-17' 204 | ManagedPolicyArns: 205 | - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 206 | Policies: 207 | - PolicyDocument: 208 | Statement: 209 | - Action: 210 | - logs:DeleteSubscriptionFilter 211 | - logs:DescribeLogGroups 212 | - logs:PutSubscriptionFilter 213 | Effect: Allow 214 | Resource: '*' 215 | Sid: Logs 216 | Version: '2012-10-17' 217 | PolicyName: Logs 218 | Type: AWS::IAM::Role 219 | LogSubscriberRule: 220 | Properties: 221 | EventPattern: 222 | detail: 223 | eventName: 224 | - CreateLogGroup 225 | eventSource: 226 | - Fn::Sub: logs.${AWS::URLSuffix} 227 | detail-type: 228 | - AWS API Call via CloudTrail 229 | source: 230 | - aws.logs 231 | ScheduleExpression: 232 | Ref: SubscribeSchedule 233 | Targets: 234 | - Arn: 235 | Fn::GetAtt: 236 | - LogSubscriberFunction 237 | - Arn 238 | Id: LogSubscriberLambda 239 | Type: AWS::Events::Rule 240 | Subscriber: 241 | DependsOn: 242 | - LogSubscriberFunction 243 | Properties: 244 | ServiceToken: 245 | Fn::GetAtt: 246 | - LogSubscriberFunction 247 | - Arn 248 | Type: Custom::Subscriber 249 | Transform: AWS::Serverless-2016-10-31 250 | -------------------------------------------------------------------------------- /CloudWatch2S3.template: -------------------------------------------------------------------------------- 1 | Conditions: 2 | AllowedAccountsSpecified: 3 | Fn::Not: 4 | - Fn::Equals: 5 | - Fn::Join: 6 | - ',' 7 | - Ref: AllowedAccounts 8 | - '0' 9 | CreateBucket: 10 | Fn::Equals: 11 | - Ref: BucketName 12 | - '' 13 | Encrypt: 14 | Fn::Not: 15 | - Fn::Equals: 16 | - Ref: KeyArn 17 | - '' 18 | ProcessingRequired: 19 | Fn::Not: 20 | - Fn::Equals: 21 | - Ref: LogFormat 22 | - CloudWatch JSON (GZIP) 23 | Description: Continuously dump all matching CloudWatch Log groups to a bucket for 24 | long-term storage (by CloudSnorkel) 25 | Mappings: 26 | Partitions: 27 | aws: 28 | LogEndpoints: 29 | - logs.af-south-1.amazonaws.com 30 | - logs.ap-east-1.amazonaws.com 31 | - logs.ap-northeast-1.amazonaws.com 32 | - logs.ap-northeast-2.amazonaws.com 33 | - logs.ap-northeast-3.amazonaws.com 34 | - logs.ap-south-1.amazonaws.com 35 | - logs.ap-southeast-1.amazonaws.com 36 | - logs.ap-southeast-2.amazonaws.com 37 | - logs.ca-central-1.amazonaws.com 38 | - logs.eu-central-1.amazonaws.com 39 | - logs.eu-north-1.amazonaws.com 40 | - logs.eu-south-1.amazonaws.com 41 | - logs.eu-west-1.amazonaws.com 42 | - logs.eu-west-2.amazonaws.com 43 | - logs.eu-west-3.amazonaws.com 44 | - logs.me-south-1.amazonaws.com 45 | - logs.sa-east-1.amazonaws.com 46 | - logs.us-east-1.amazonaws.com 47 | - logs.us-east-2.amazonaws.com 48 | - logs.us-west-1.amazonaws.com 49 | - logs.us-west-2.amazonaws.com 50 | aws-cn: 51 | LogEndpoints: 52 | - logs.cn-north-1.amazonaws.com.cn 53 | - logs.cn-northwest-1.amazonaws.com.cn 54 | aws-iso: 55 | LogEndpoints: 56 | - logs.us-iso-east-1.c2s.ic.gov 57 | aws-iso-b: 58 | LogEndpoints: 59 | - logs.us-isob-east-1.sc2s.sgov.gov 60 | aws-us-gov: 61 | LogEndpoints: 62 | - logs.us-gov-east-1.amazonaws.com 63 | - logs.us-gov-west-1.amazonaws.com 64 | Metadata: 65 | AWS::CloudFormation::Interface: 66 | ParameterGroups: 67 | - Label: 68 | default: Storage 69 | Parameters: 70 | - BucketName 71 | - BucketPrefix 72 | - LogFormat 73 | - Label: 74 | default: Other 75 | Parameters: 76 | - DestinationName 77 | - Label: 78 | default: Security 79 | Parameters: 80 | - AllowedAccounts 81 | - KeyArn 82 | - Label: 83 | default: Tweaks 84 | Parameters: 85 | - ShardCount 86 | - Retention 87 | - BufferIntervalHint 88 | - BufferSizeHint 89 | - ProcessorBufferIntervalHint 90 | - ProcessorBufferSizeHint 91 | - Label: 92 | default: CloudWatch Logs 93 | Parameters: 94 | - SubscribeSchedule 95 | - LogGroupNamePrefix 96 | ParameterLabels: 97 | AllowedAccounts: 98 | default: Allowed Accounts 99 | BucketName: 100 | default: Bucket ARN 101 | BucketPrefix: 102 | default: Key Prefix 103 | BufferIntervalHint: 104 | default: Delivery Buffer Timeout 105 | BufferSizeHint: 106 | default: Delivery Buffer Size 107 | DestinationName: 108 | default: Log Destination Name 109 | KeyArn: 110 | default: KMS Key ARN 111 | LogFormat: 112 | default: Export Format 113 | LogGroupNamePrefix: 114 | default: Required Log Group Name Prefix 115 | ProcessorBufferIntervalHint: 116 | default: Processing Lambda Buffer Timeout 117 | ProcessorBufferSizeHint: 118 | default: Processing Lambda Buffer Size 119 | Retention: 120 | default: Kinesis Retention 121 | ShardCount: 122 | default: Kinesis Shard Count 123 | SubscribeSchedule: 124 | default: Look for New Logs Schedule 125 | AWS::ServerlessRepo::Application: 126 | Author: CloudSnorkel 127 | Description: Logging infrastructure for exporting all CloudWatch logs from multiple 128 | accounts to a single S3 bucket. 129 | HomePageUrl: https://github.com/CloudSnorkel/CloudWatch2S3 130 | Labels: 131 | - cloudwatch 132 | - s3 133 | - export 134 | LicenseUrl: LICENSE 135 | Name: CloudWatch2S3 136 | ReadmeUrl: README.md 137 | SemanticVersion: 1.0.0 138 | SourceCodeUrl: https://github.com/CloudSnorkel/CloudWatch2S3 139 | SpdxLicenseId: MIT 140 | Outputs: 141 | Bucket: 142 | Description: Bucket where all logs will be written 143 | Value: 144 | Fn::If: 145 | - CreateBucket 146 | - Fn::GetAtt: 147 | - LogBucket 148 | - Arn 149 | - Ref: BucketName 150 | Export: 151 | Name: !Sub '${AWS::StackName}-BucketArn' 152 | LogDestination: 153 | Description: Log destination ARN to be used when setting up other accounts to 154 | exports logs 155 | Value: 156 | Fn::GetAtt: 157 | - LogDestination 158 | - Arn 159 | Export: 160 | Name: !Sub '${AWS::StackName}-LogDestinationArn' 161 | Parameters: 162 | AllowedAccounts: 163 | Default: '0' 164 | Description: Comma separated list of external account numbers allowed to export 165 | logs to this bucket (leave as '0' to disallow external accounts) 166 | Type: List 167 | BucketName: 168 | Default: '' 169 | Description: ARN of bucket where all logs will be exported (leave empty to automatically 170 | create) 171 | Type: String 172 | BucketPrefix: 173 | Default: logs/ 174 | Description: Prefix to prepend to all exported file names 175 | Type: String 176 | BufferIntervalHint: 177 | Default: '300' 178 | Description: Firehose buffering interval hint (in seconds) 179 | Type: Number 180 | BufferSizeHint: 181 | Default: '50' 182 | Description: Firehose buffering size hint (in megabytes) 183 | Type: Number 184 | DestinationName: 185 | AllowedPattern: '[a-zA-Z0-9]+' 186 | Default: BucketBackupLogDestination 187 | Description: Name of log destination (must be unique across this account) 188 | Type: String 189 | KeyArn: 190 | Default: '' 191 | Description: KMS Key id to encrypt Kinesis stream and S3 bucket at rest (leave 192 | empty to disable encryption) 193 | Type: String 194 | LogFormat: 195 | AllowedValues: 196 | - Raw 197 | - CloudWatch JSON (GZIP) 198 | Default: Raw 199 | Description: Format in which logs will be saved in the bucket 200 | Type: String 201 | LogGroupNamePrefix: 202 | Default: '' 203 | Description: Prefix to match against log group that should be exported (leave 204 | empty to export all log groups) 205 | Type: String 206 | ProcessorBufferIntervalHint: 207 | Default: '60' 208 | Description: Processing Lambda buffer timeout (in seconds, only in raw format 209 | mode) 210 | MaxValue: 900 211 | MinValue: 60 212 | Type: Number 213 | ProcessorBufferSizeHint: 214 | Default: '1' 215 | Description: Processing Lambda buffer size (in megabytes, only in raw format mode) 216 | -- keep this low as uncompressed buffer data must not exceed Lambda's limit 217 | of 6MB response 218 | Type: Number 219 | Retention: 220 | Default: '24' 221 | Description: Number of hours records remain in Kinesis in case delivery is slow 222 | or fails 223 | Type: Number 224 | ShardCount: 225 | Default: '1' 226 | Description: Number of Kinesis stream shards each capable of 1MB/s or 1000 log 227 | records per second 228 | Type: Number 229 | SubscribeSchedule: 230 | Default: rate(1 hour) 231 | Description: Schedule to look for new log groups for export (in case CloudTrail 232 | missed something) 233 | Type: String 234 | Resources: 235 | DeliveryRole: 236 | Properties: 237 | AssumeRolePolicyDocument: 238 | Statement: 239 | - Action: 240 | - sts:AssumeRole 241 | Condition: 242 | StringEquals: 243 | sts:ExternalId: 244 | Ref: AWS::AccountId 245 | Effect: Allow 246 | Principal: 247 | Service: 248 | - Fn::Sub: firehose.${AWS::URLSuffix} 249 | Version: '2012-10-17' 250 | Policies: 251 | - PolicyDocument: 252 | Statement: 253 | - Action: 254 | - s3:AbortMultipartUpload 255 | - s3:GetBucketLocation 256 | - s3:GetObject 257 | - s3:ListBucket 258 | - s3:ListBucketMultipartUploads 259 | - s3:PutObject 260 | Effect: Allow 261 | Resource: 262 | - Fn::If: 263 | - CreateBucket 264 | - Fn::GetAtt: 265 | - LogBucket 266 | - Arn 267 | - Ref: BucketName 268 | - Fn::Sub: 269 | - ${Param1}/* 270 | - Param1: 271 | Fn::If: 272 | - CreateBucket 273 | - Fn::GetAtt: 274 | - LogBucket 275 | - Arn 276 | - Ref: BucketName 277 | - Action: 278 | - kinesis:DescribeStream 279 | - kinesis:GetShardIterator 280 | - kinesis:GetRecords 281 | Effect: Allow 282 | Resource: 283 | Fn::GetAtt: 284 | - LogStream 285 | - Arn 286 | - Action: 287 | - logs:PutLogEvents 288 | Effect: Allow 289 | Resource: 290 | - Fn::Sub: arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/kinesisfirehose/${AWS::StackName}-DeliveryStream*:log-stream:* 291 | - Fn::If: 292 | - Encrypt 293 | - Action: 294 | - kms:Decrypt 295 | Condition: 296 | StringEquals: 297 | kms:ViaService: 298 | Fn::Sub: kinesis.${AWS::Region}.${AWS::URLSuffix} 299 | Effect: Allow 300 | Resource: 301 | - Ref: KeyArn 302 | - Ref: AWS::NoValue 303 | - Fn::If: 304 | - Encrypt 305 | - Action: 306 | - kms:GenerateDataKey 307 | - kms:Decrypt 308 | Condition: 309 | StringEquals: 310 | kms:ViaService: 311 | Fn::Sub: s3.${AWS::Region}.${AWS::URLSuffix} 312 | Effect: Allow 313 | Resource: 314 | - Ref: KeyArn 315 | - Ref: AWS::NoValue 316 | Version: '2012-10-17' 317 | PolicyName: DeliveryPolicy 318 | Type: AWS::IAM::Role 319 | DeliveryStream: 320 | DependsOn: 321 | - KinesisRoleLambdaPolicy 322 | Properties: 323 | DeliveryStreamType: KinesisStreamAsSource 324 | ExtendedS3DestinationConfiguration: 325 | BucketARN: 326 | Fn::If: 327 | - CreateBucket 328 | - Fn::GetAtt: 329 | - LogBucket 330 | - Arn 331 | - Ref: BucketName 332 | BufferingHints: 333 | IntervalInSeconds: 334 | Ref: BufferIntervalHint 335 | SizeInMBs: 336 | Ref: BufferSizeHint 337 | CloudWatchLoggingOptions: 338 | Enabled: true 339 | LogGroupName: 340 | Ref: DeliveryStreamLog 341 | LogStreamName: 342 | Ref: DeliveryStreamLogStream 343 | CompressionFormat: UNCOMPRESSED 344 | EncryptionConfiguration: 345 | Fn::If: 346 | - Encrypt 347 | - KMSEncryptionConfig: 348 | AWSKMSKeyARN: 349 | Ref: KeyArn 350 | - Ref: AWS::NoValue 351 | Prefix: 352 | Ref: BucketPrefix 353 | ProcessingConfiguration: 354 | Fn::If: 355 | - ProcessingRequired 356 | - Enabled: true 357 | Processors: 358 | - Parameters: 359 | - ParameterName: LambdaArn 360 | ParameterValue: 361 | Fn::GetAtt: 362 | - LogProcessorFunction 363 | - Arn 364 | - ParameterName: BufferSizeInMBs 365 | ParameterValue: 366 | Ref: ProcessorBufferSizeHint 367 | - ParameterName: BufferIntervalInSeconds 368 | ParameterValue: 369 | Ref: ProcessorBufferIntervalHint 370 | - ParameterName: NumberOfRetries 371 | ParameterValue: '3' 372 | Type: Lambda 373 | - Ref: AWS::NoValue 374 | RoleARN: 375 | Fn::GetAtt: 376 | - DeliveryRole 377 | - Arn 378 | KinesisStreamSourceConfiguration: 379 | KinesisStreamARN: 380 | Fn::GetAtt: 381 | - LogStream 382 | - Arn 383 | RoleARN: 384 | Fn::GetAtt: 385 | - DeliveryRole 386 | - Arn 387 | Type: AWS::KinesisFirehose::DeliveryStream 388 | DeliveryStreamLog: 389 | Properties: 390 | LogGroupName: 391 | Fn::Sub: /aws/kinesisfirehose/${AWS::StackName}-DeliveryStream 392 | Type: AWS::Logs::LogGroup 393 | DeliveryStreamLogStream: 394 | Properties: 395 | LogGroupName: 396 | Ref: DeliveryStreamLog 397 | LogStreamName: S3Delivery 398 | Type: AWS::Logs::LogStream 399 | KinesisRole: 400 | Properties: 401 | AssumeRolePolicyDocument: 402 | Statement: 403 | - Action: 404 | - sts:AssumeRole 405 | Effect: Allow 406 | Principal: 407 | Service: 408 | Fn::FindInMap: 409 | - Partitions 410 | - Ref: AWS::Partition 411 | - LogEndpoints 412 | Version: '2012-10-17' 413 | Type: AWS::IAM::Role 414 | KinesisRoleLambdaPolicy: 415 | Properties: 416 | PolicyDocument: 417 | Statement: 418 | - Action: 419 | - lambda:InvokeFunction 420 | - lambda:GetFunctionConfiguration 421 | Effect: Allow 422 | Resource: 423 | Fn::GetAtt: 424 | - LogProcessorFunction 425 | - Arn 426 | Version: '2012-10-17' 427 | PolicyName: KinesisCallProcessor 428 | Roles: 429 | - Ref: DeliveryRole 430 | Type: AWS::IAM::Policy 431 | KinesisRolePolicy: 432 | Properties: 433 | PolicyDocument: 434 | Statement: 435 | - Action: 436 | - kinesis:PutRecord 437 | - kinesis:PutRecords 438 | Effect: Allow 439 | Resource: 440 | - Fn::GetAtt: 441 | - LogStream 442 | - Arn 443 | - Action: 444 | - iam:PassRole 445 | Effect: Allow 446 | Resource: 447 | Fn::GetAtt: 448 | - KinesisRole 449 | - Arn 450 | - Fn::If: 451 | - Encrypt 452 | - Action: 453 | - kms:GenerateDataKey 454 | Effect: Allow 455 | Resource: 456 | - Ref: KeyArn 457 | - Ref: AWS::NoValue 458 | Version: '2012-10-17' 459 | PolicyName: KinesisWrite 460 | Roles: 461 | - Ref: KinesisRole 462 | Type: AWS::IAM::Policy 463 | LogBucket: 464 | Condition: CreateBucket 465 | Type: AWS::S3::Bucket 466 | LogDestination: 467 | DependsOn: 468 | - LogStream 469 | - KinesisRole 470 | - KinesisRolePolicy 471 | Properties: 472 | DestinationName: 473 | Ref: DestinationName 474 | DestinationPolicy: 475 | Fn::Sub: 476 | - |- 477 | { 478 | "Version": "2012-10-17", 479 | "Statement": [ 480 | { 481 | "Effect": "Allow", 482 | "Principal": { 483 | "AWS": ["${AWS::AccountId}"] 484 | }, 485 | "Action": "logs:PutSubscriptionFilter", 486 | "Resource": "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:destination:${DestinationName}" 487 | } 488 | ${Extra} 489 | ] 490 | } 491 | - DestinationName: 492 | Ref: DestinationName 493 | Extra: 494 | Fn::If: 495 | - AllowedAccountsSpecified 496 | - Fn::Sub: 497 | - |- 498 | ,{ 499 | "Effect": "Allow", 500 | "Principal": {"AWS": ["${Param1}${Param2} 501 | - Param1: 502 | Fn::Join: 503 | - '","' 504 | - Ref: AllowedAccounts 505 | Param2: 506 | Fn::Sub: |- 507 | "]}, 508 | "Action": "logs:PutSubscriptionFilter", 509 | "Resource": "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:destination:${DestinationName}" 510 | } 511 | - '' 512 | RoleArn: 513 | Fn::GetAtt: 514 | - KinesisRole 515 | - Arn 516 | TargetArn: 517 | Fn::GetAtt: 518 | - LogStream 519 | - Arn 520 | Type: AWS::Logs::Destination 521 | LogProcessorFunction: 522 | Properties: 523 | Code: 524 | ZipFile: 525 | Fn::Sub: | 526 | import base64 527 | import gzip 528 | import json 529 | 530 | 531 | def handle_records(records): 532 | for record in records: 533 | record_id = record["recordId"] 534 | data = json.loads(gzip.decompress(base64.b64decode(record["data"]))) 535 | 536 | if data["messageType"] == "CONTROL_MESSAGE": 537 | yield { 538 | "result": "Dropped", 539 | "recordId": record_id 540 | } 541 | elif data["messageType"] == "DATA_MESSAGE": 542 | yield { 543 | "recordId": record_id, 544 | "result": "Ok", 545 | "data": base64.b64encode("".join( 546 | f"{data['logGroup']}:{data['logStream']}\t{e['timestamp']}\t{e['message']}" 547 | for e in data["logEvents"] 548 | ).encode("utf-8")).decode("utf-8") 549 | } 550 | else: 551 | yield { 552 | "result": "ProcessingFailed", 553 | "recordId": record_id 554 | } 555 | 556 | 557 | def handler(event, context): 558 | result = [] 559 | size = 0 560 | 561 | for i, record in enumerate(handle_records(event["records"])): 562 | record_size = len(str(record)) 563 | if record_size >= 6000000: 564 | print("Dropping single record that's over 6MB") 565 | result.append({ 566 | "result": "Dropped", 567 | "recordId": record["recordId"] 568 | }) 569 | else: 570 | size += record_size 571 | if size < 6000000: 572 | # lambda limits output to 6mb 573 | # kinesis will treat records not here as failed and retry 574 | # TODO or do we need to reingest? 575 | result.append(record) 576 | else: 577 | print("Failing record as output is over 6MB") 578 | result.append({ 579 | "result": "ProcessingFailed", 580 | "recordId": record["recordId"] 581 | }) 582 | 583 | return {"records": result} 584 | Handler: index.handler 585 | Role: 586 | Fn::GetAtt: 587 | - LogProcessorRole 588 | - Arn 589 | Runtime: python3.9 590 | Timeout: 300 591 | Type: AWS::Lambda::Function 592 | LogProcessorRole: 593 | Properties: 594 | AssumeRolePolicyDocument: 595 | Statement: 596 | - Action: 597 | - sts:AssumeRole 598 | Effect: Allow 599 | Principal: 600 | Service: 601 | - Fn::Sub: lambda.${AWS::URLSuffix} 602 | Version: '2012-10-17' 603 | ManagedPolicyArns: 604 | - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 605 | Policies: [] 606 | Type: AWS::IAM::Role 607 | LogStream: 608 | Properties: 609 | RetentionPeriodHours: 610 | Ref: Retention 611 | ShardCount: 612 | Ref: ShardCount 613 | StreamEncryption: 614 | Fn::If: 615 | - Encrypt 616 | - EncryptionType: KMS 617 | KeyId: 618 | Ref: KeyArn 619 | - Ref: AWS::NoValue 620 | Type: AWS::Kinesis::Stream 621 | LogSubscriberFunction: 622 | Properties: 623 | Code: 624 | ZipFile: 625 | Fn::Sub: | 626 | import traceback 627 | 628 | import boto3 629 | import botocore.exceptions 630 | import cfnresponse 631 | 632 | logs_client = boto3.client("logs") 633 | 634 | 635 | def subscribe(log_group_name): 636 | print("Subscribe ", log_group_name) 637 | 638 | if log_group_name.startswith("/aws/lambda/${AWS::StackName}") \ 639 | or log_group_name.startswith("/aws/kinesisfirehose/${AWS::StackName}"): 640 | print("Skipping our log groups to avoid endless recursion") 641 | return 642 | 643 | try: 644 | logs_client.put_subscription_filter( 645 | logGroupName=log_group_name, 646 | filterName="BucketBackupFilter", 647 | filterPattern="", 648 | destinationArn="${LogDestination.Arn}", 649 | ) 650 | except logs_client.exceptions.LimitExceededException: 651 | print(f"ERROR: Unable to subscribe to {log_group_name} as it already has an active subscription") 652 | 653 | 654 | def matched_log_groups(prefix): 655 | print(f"Finding all log groups with prefix '{prefix}'") 656 | 657 | log_group_paginator = logs_client.get_paginator("describe_log_groups") 658 | 659 | paginate_params = {} 660 | if prefix: 661 | paginate_params["logGroupNamePrefix"] = prefix 662 | 663 | for log_group_page in log_group_paginator.paginate(**paginate_params): 664 | for log_group in log_group_page["logGroups"]: 665 | yield log_group["logGroupName"] 666 | 667 | 668 | def subscribe_all(): 669 | for log_group_name in matched_log_groups("${LogGroupNamePrefix}"): 670 | subscribe(log_group_name) 671 | 672 | 673 | def unsubscribe_all(): 674 | for log_group_name in matched_log_groups(""): 675 | print("Unsubscribe ", log_group_name) 676 | 677 | try: 678 | logs_client.delete_subscription_filter( 679 | logGroupName=log_group_name, 680 | filterName="BucketBackupFilter", 681 | ) 682 | except botocore.exceptions.ClientError: 683 | pass 684 | 685 | 686 | def handler(event, context): 687 | print('event:', event) 688 | 689 | if "ResponseURL" in event and "RequestType" in event: 690 | # custom resource callback 691 | try: 692 | if event["RequestType"] in ["Create", "Update"]: 693 | print("Subscribe to all new log groups on resource", event["RequestType"]) 694 | subscribe_all() 695 | 696 | elif event["RequestType"] == "Delete": 697 | print("Unsubscribe all on resource", event["RequestType"]) 698 | unsubscribe_all() 699 | 700 | cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, "ok") 701 | 702 | except Exception as e: 703 | try: 704 | traceback.print_last() 705 | except ValueError: 706 | print("Caught exception but unable to print stack trace") 707 | print(e) 708 | cfnresponse.send(event, context, cfnresponse.FAILED, {}, "fail") 709 | 710 | else: 711 | # other call 712 | detail_type = event.get("detail-type") 713 | 714 | if detail_type == "AWS API Call via CloudTrail": 715 | print("Subscribe to specific new log group from CloudTrail") 716 | 717 | request_parameters = event['detail']['requestParameters'] 718 | 719 | if request_parameters: 720 | log_group_name = request_parameters['logGroupName'] 721 | 722 | if log_group_name.startswith("${LogGroupNamePrefix}"): 723 | subscribe(log_group_name) 724 | else: 725 | print(log_group_name, "doesn't match required prefix '${LogGroupNamePrefix}'") 726 | 727 | else: 728 | print("Bad parameters") 729 | 730 | elif detail_type == "Scheduled Event": 731 | print("Subscribe to all new log groups on schedule") 732 | 733 | subscribe_all() 734 | 735 | else: 736 | print("Subscribe to all new log groups") 737 | 738 | subscribe_all() 739 | Handler: index.handler 740 | Role: 741 | Fn::GetAtt: 742 | - LogSubscriberRole 743 | - Arn 744 | Runtime: python3.9 745 | Timeout: 300 746 | Type: AWS::Lambda::Function 747 | LogSubscriberPermission: 748 | Properties: 749 | Action: lambda:InvokeFunction 750 | FunctionName: 751 | Fn::GetAtt: 752 | - LogSubscriberFunction 753 | - Arn 754 | Principal: 755 | Fn::Sub: events.${AWS::URLSuffix} 756 | SourceArn: 757 | Fn::GetAtt: 758 | - LogSubscriberRule 759 | - Arn 760 | Type: AWS::Lambda::Permission 761 | LogSubscriberRole: 762 | Properties: 763 | AssumeRolePolicyDocument: 764 | Statement: 765 | - Action: 766 | - sts:AssumeRole 767 | Effect: Allow 768 | Principal: 769 | Service: 770 | - Fn::Sub: lambda.${AWS::URLSuffix} 771 | Version: '2012-10-17' 772 | ManagedPolicyArns: 773 | - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 774 | Policies: 775 | - PolicyDocument: 776 | Statement: 777 | - Action: 778 | - logs:DeleteSubscriptionFilter 779 | - logs:DescribeLogGroups 780 | - logs:PutSubscriptionFilter 781 | Effect: Allow 782 | Resource: '*' 783 | Sid: Logs 784 | Version: '2012-10-17' 785 | PolicyName: Logs 786 | Type: AWS::IAM::Role 787 | LogSubscriberRule: 788 | Properties: 789 | EventPattern: 790 | detail: 791 | eventName: 792 | - CreateLogGroup 793 | eventSource: 794 | - Fn::Sub: logs.${AWS::URLSuffix} 795 | detail-type: 796 | - AWS API Call via CloudTrail 797 | source: 798 | - aws.logs 799 | ScheduleExpression: 800 | Ref: SubscribeSchedule 801 | Targets: 802 | - Arn: 803 | Fn::GetAtt: 804 | - LogSubscriberFunction 805 | - Arn 806 | Id: LogSubscriberLambda 807 | Type: AWS::Events::Rule 808 | Subscriber: 809 | DependsOn: 810 | - LogSubscriberFunction 811 | Properties: 812 | ServiceToken: 813 | Fn::GetAtt: 814 | - LogSubscriberFunction 815 | - Arn 816 | Type: Custom::Subscriber 817 | Transform: AWS::Serverless-2016-10-31 818 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019-2020 CloudSnorkel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## AWS CloudWatch to S3 2 | 3 | Logging infrastructure for exporting all CloudWatch logs from multiple accounts to a single S3 bucket. 4 | 5 | Available on AWS Serverless Application Repository for easy deployment: 6 | 7 | * [CloudWatch2S3](https://serverlessrepo.aws.amazon.com/applications/arn:aws:serverlessrepo:us-east-1:859319237877:applications~CloudWatch2S3) 8 | * [CloudWatch2S3-additional-account](https://serverlessrepo.aws.amazon.com/applications/arn:aws:serverlessrepo:us-east-1:859319237877:applications~CloudWatch2S3-additional-account) 9 | 10 | ### Overview 11 | 12 | ![Architecture diagram](https://github.com/CloudSnorkel/CloudWatch2S3/raw/master/architecture.svg?sanitize=true) 13 | 14 | This project supplies a CloudFormation template that setups Kinesis stream that takes log records from CloudWatch and 15 | writes them to a specific S3 bucket as they are arrive. Log records can be retrieved from multiple AWS accounts using 16 | the second CloudFormation template. 17 | 18 | Log records are batched together across log groups and partitioned into folders based on time of ingestion. Log format 19 | can be configured to either be raw log lines or compressed CloudWatch JSON. The raw log format is: 20 | 21 | LOG_GROUP:LOG_STREAM\tTIMESTAMP\tRAW_LOG_LINE 22 | 23 | For example: 24 | 25 | /aws/lambda/SomeLambdaFunction:2019/02/05/[$LATEST]b346f603d7bb4b6aa77b53bc4050bc37 1549428326 INFO hello world 26 | 27 | Subscription of CloudWatch log groups is done in two ways. If CloudTrail is enabled, every new log group will immediacy 28 | be subscribed. In addition, every hour a subscription Lambda is executed to look for new log groups and subscribe them. 29 | Finally, the same subscription Lambda is executed during deployment of the CloudFormation stack so all log groups 30 | matching the configured prefix will be subscribed immediately on deployment. 31 | 32 | If CloudTrail is not enabled, it may take up to an hour for new log groups to be subscribed. This time can be configured 33 | in the CloudFormation stack using the `SubscribeSchedule` parameter. In CloudFormation UI it may be named _Look for New 34 | Logs Schedule_. 35 | 36 | ### Deploy 37 | 38 | If you have just one AWS account, simply deploy `CloudWatch2S3.template` in CloudFormation. 39 | 40 | If you have multiple AWS accounts, choose a central account where all logs will be stored in S3 and deploy 41 | `CloudWatch2S3.template` in CloudFormation. Once done, go to the outputs tab and copy the value of `LogDestination`. 42 | Then go to the other accounts and deploy `CloudWatch2S3-additional-account.template` in CloudFormation. You will need to 43 | supply the value you copied as the `LogDestination` parameter. 44 | 45 | #### Parameters 46 | 47 | There are a lot of parameters to play with, but the defaults should be good enough for most. If you have a lot of log 48 | records coming in (more than 1000/s or 1MB/s), you might want to increase Kinesis shard count. 49 | 50 | ### Known Limitations 51 | 52 | Cross region export is not supported by CloudWatch Logs. If you need to gather logs from multiple regions, create the CloudFormation stack in each required region. You can use CloudFormation Stack Sets to deploy to all regions at once. 53 | 54 | Single CloudWatch records can't be over 6MB when using anything else but raw log format. Kinesis uses Lambda to convert data and Lambda output is limited to 6MB. Note that data comes in compressed from CloudWatch but has to come out decompressed from Lambda. So the decompressed record can't be over 6MB. You will see record failures in CloudWatch metrics for the Kinesis stream for this and errors in the log for the processor Lambda function. 55 | 56 | ### Troubleshooting 57 | 58 | * Make sure the right CloudWatch log groups are subscribed 59 | * Look for errors in CloudWatch log group `/aws/kinesisfirehose/-DeliveryStream` 60 | * Look for errors in CloudWatch log group `/aws/lambda/-LogProcessor-` 61 | * Make sure Kinesis, Firehose and S3 have access to your KMS key when using encryption 62 | * Increase Kinesis shard count with `ShardCount` (_Kinesis Shard Count_) CloudFormation parameter 63 | 64 | ### Origin 65 | 66 | This project is based on Amazon's [Stream Amazon CloudWatch Logs to a Centralized Account for Audit and Analysis](https://aws.amazon.com/blogs/architecture/stream-amazon-cloudwatch-logs-to-a-centralized-account-for-audit-and-analysis/) but adds: 67 | 68 | * One step installation 69 | * Zero scripts 70 | * Works out of the box 71 | * Easier configuration without editing files 72 | * No hard dependency on CloudTrail 73 | * Optional unpacking of CloudWatch JSON format 74 | -------------------------------------------------------------------------------- /architecture.svg: -------------------------------------------------------------------------------- 1 | 2 |
Main Account
Main Account
Kinesis
[Not supported by viewer]
Target Bucket
<b>Target Bucket</b>
Subscribe
Subscribe
Subscriber Function
<b>Subscriber Function</b>
CloudWatch Timer
<b>CloudWatch Timer</b>
CloudTrail
<b>CloudTrail</b><br>
CloudWatch Logs
<b>CloudWatch Logs</b>
Processing Function
<b>Processing Function</b>
Export
Export
Subscribe
Subscribe
Subscriber Function
<b>Subscriber Function</b>
CloudWatch Timer
<b>CloudWatch Timer</b>
CloudTrail
<b>CloudTrail</b><br>
CloudWatch Logs
<b>CloudWatch Logs</b>
Export
Export
Additional Account
Additional Account<br> --------------------------------------------------------------------------------