├── .gitignore ├── CloudWatchEvents-Firehose-Resources ├── sourceToSplunk.yml └── splunkDashboard.json ├── CloudWatchLogs-Firehose-Resources ├── cwlToSplunk.yml ├── lambda.py ├── splunkDashboard.json └── tests.py ├── CloudWatchMetrics-Firehose-Resources ├── cwMetricsToSplunk.yml ├── lambda.py ├── splunkDashboard-event.json ├── splunkDashboard-metric.json └── tests.py ├── LICENSE ├── S3-SQS-Lambda-Firehose-Resources ├── billingCURToS3.yml ├── cloudTrailToS3.yml ├── eventsInS3ToSplunk.yml ├── lambda.py ├── route53QueryLogsToS3.yml ├── sampleElbToS3.yml ├── splunkDashboard.json ├── ta_route53 │ ├── README │ ├── default │ │ ├── app.conf │ │ ├── props.conf │ │ └── transforms.conf │ ├── metadata │ │ └── default.meta │ └── static │ │ ├── appIcon.png │ │ ├── appIconAlt.png │ │ ├── appIconAlt_2x.png │ │ └── appIcon_2x.png ├── test-fixtures │ ├── sample-cloudtrail.json.gz │ ├── sample-route53Resolver.log.gz │ ├── sample-s3ServerAccess │ ├── sample-vpcflow.log.gz │ ├── testFile1.log │ ├── testFile1.log.gz │ └── testdir │ │ └── 123 │ │ ├── testFile2.log │ │ └── testFile2.log.gz ├── tests.py └── vpcFlowLogToS3.yml ├── Single Account CloudFormation ├── cloudTrail.yml ├── guardDuty.yml ├── iamAccessAnalyzer.yml ├── scdm-s3-pull.yml └── securityHub.yml ├── VPCFlowLogs-Firehose-Resources ├── kdfToSplunk.yml ├── splunkDashboard.json └── vpcFlowLogsToKDF.yml └── readme.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.DS_Store 2 | *__pycache__* 3 | -------------------------------------------------------------------------------- /CloudWatchEvents-Firehose-Resources/sourceToSplunk.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | AWSTemplateFormatVersion: "2010-09-09" 4 | Description: This is a CloudFormation template to configure events coming from a source to be sent to Splunk via CloudWach Events and Kinesis Data Firehose. 5 | 6 | Parameters: 7 | 8 | service: 9 | Type: String 10 | Description: Service name used in tagging AWS resources. 11 | Default: splunk-aws-gdi-toolkit 12 | 13 | stage: 14 | Type: String 15 | Description: Used to distinguish between stages of an environment (dev, test, prod, stage, etc). Only used in AWS resource tagging. 16 | Default: dev 17 | 18 | contact: 19 | Description: Used to identify a contact for the resources created in this stack. Only used in AWS resource tagging. As an example, this could be an email address or username. 20 | Type: String 21 | Default: "" 22 | 23 | logType: 24 | Type: String 25 | Description: Log type, used in AWS resource names and to determine log source. 26 | AllowedValues: 27 | - guardduty 28 | - securityhub 29 | 30 | splunkHECEndpoint: 31 | Type: String 32 | Description: Destination URL that Firehose will send data to. 33 | 34 | splunkHECToken: 35 | Type: String 36 | Description: HEC token Firehose will use to authenticate data being sent to Splunk. 37 | 38 | cloudWatchAlertEmail: 39 | Type: String 40 | Description: Email address for receiving alerts related to CloudTrail ingestion. Leave empty for no alerting. 41 | Default: "" 42 | 43 | Conditions: 44 | enableAlerting: !Not 45 | - !Equals 46 | - !Ref cloudWatchAlertEmail 47 | - "" 48 | 49 | Resources: 50 | # Firehose > Splunk resources 51 | firehose: 52 | Type: AWS::KinesisFirehose::DeliveryStream 53 | Properties: 54 | DeliveryStreamName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose" 55 | DeliveryStreamType: DirectPut 56 | SplunkDestinationConfiguration: 57 | BufferingHints: 58 | IntervalInSeconds: 5 59 | SizeInMBs: 1 60 | CloudWatchLoggingOptions: 61 | Enabled: true 62 | LogGroupName: !Ref firehoseLogGroup 63 | LogStreamName: "SplunkDelivery" 64 | HECAcknowledgmentTimeoutInSeconds: 300 65 | HECEndpoint: !Ref splunkHECEndpoint 66 | HECEndpointType: "Raw" 67 | HECToken: !Ref splunkHECToken 68 | RetryOptions: 69 | DurationInSeconds: 3600 70 | S3BackupMode: "FailedEventsOnly" 71 | S3Configuration: 72 | BucketARN: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-${logType}-firehose-backsplash" 73 | BufferingHints: 74 | IntervalInSeconds: 300 75 | SizeInMBs: 16 76 | CompressionFormat: "GZIP" 77 | Prefix: !Sub "$${AWS::AccountId}-${AWS::Region}-${logType}-firehose" 78 | RoleARN: !GetAtt firehoseIAMRole.Arn 79 | 80 | firehoseIAMPolicy: 81 | Type: AWS::IAM::ManagedPolicy 82 | Properties: 83 | PolicyDocument: 84 | Version: '2012-10-17' 85 | Statement: 86 | - Effect: Allow 87 | Action: 88 | - logs:Describe* 89 | - logs:PutLogEvents 90 | Resource: !GetAtt firehoseLogGroup.Arn 91 | - Effect: Allow 92 | Action: 93 | - s3:PutObject 94 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-${logType}-firehose-backsplash/*" 95 | ManagedPolicyName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose-iam-policy" 96 | 97 | firehoseIAMRole: 98 | Type: AWS::IAM::Role 99 | Properties: 100 | AssumeRolePolicyDocument: 101 | Version: '2012-10-17' 102 | Statement: 103 | - Effect: "Allow" 104 | Principal: 105 | Service: "firehose.amazonaws.com" 106 | Action: 107 | - sts:AssumeRole 108 | ManagedPolicyArns: 109 | - !Ref firehoseIAMPolicy 110 | RoleName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose-iam-role" 111 | Tags: 112 | - Key: service 113 | Value: !Ref service 114 | - Key: stage 115 | Value: !Ref stage 116 | - Key: contact 117 | Value: !Ref contact 118 | 119 | firehoseBacksplashBucket: 120 | Type: AWS::S3::Bucket 121 | Properties: 122 | BucketName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose-backsplash" 123 | AccessControl: Private 124 | PublicAccessBlockConfiguration: 125 | BlockPublicAcls: true 126 | BlockPublicPolicy: true 127 | IgnorePublicAcls: true 128 | RestrictPublicBuckets: true 129 | BucketEncryption: 130 | ServerSideEncryptionConfiguration: 131 | - ServerSideEncryptionByDefault: 132 | SSEAlgorithm: AES256 133 | LifecycleConfiguration: 134 | Rules: 135 | - Id: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose-backsplash-cleanup" 136 | AbortIncompleteMultipartUpload: 137 | DaysAfterInitiation: 1 138 | Status: Enabled 139 | Tags: 140 | - Key: service 141 | Value: !Ref service 142 | - Key: stage 143 | Value: !Ref stage 144 | - Key: contact 145 | Value: !Ref contact 146 | 147 | firehoseLogGroup: 148 | Type: AWS::Logs::LogGroup 149 | Properties: 150 | LogGroupName: !Sub "/aws/kinesisfirehose/${AWS::AccountId}-${AWS::Region}-${logType}-firehose" 151 | RetentionInDays: 7 152 | 153 | firehoseLogStream: 154 | Type: AWS::Logs::LogStream 155 | Properties: 156 | LogGroupName: !Ref firehoseLogGroup 157 | LogStreamName: "SplunkDelivery" 158 | 159 | # Monitoring resoruces 160 | monitoringSNSTopic: 161 | Condition: enableAlerting 162 | Type: AWS::SNS::Topic 163 | Properties: 164 | DisplayName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-alerting-topic" 165 | TopicName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-alerting-topic" 166 | Subscription: 167 | - Endpoint: !Ref cloudWatchAlertEmail 168 | Protocol: email 169 | 170 | firehoseDeliveryAlarm: 171 | Condition: enableAlerting 172 | Type: AWS::CloudWatch::Alarm 173 | Properties: 174 | ActionsEnabled: True 175 | AlarmActions: 176 | - !Ref monitoringSNSTopic 177 | AlarmDescription: !Sub "Alarm if Firehose ${AWS::AccountId}-${AWS::Region}-${logType}-firehose cannot deliver to Splunk." 178 | AlarmName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose-delivery-alarm" 179 | ComparisonOperator: LessThanThreshold 180 | Dimensions: 181 | - Name: DeliveryStreamName 182 | Value: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose" 183 | EvaluationPeriods: 1 184 | MetricName: DeliveryToSplunk.Success 185 | Namespace: AWS/Firehose 186 | Period: 60 187 | Statistic: Maximum 188 | Threshold: 1 189 | Unit: Count 190 | 191 | firehoseThrottlingAlarm: 192 | Condition: enableAlerting 193 | Type: AWS::CloudWatch::Alarm 194 | Properties: 195 | ActionsEnabled: True 196 | AlarmActions: 197 | - !Ref monitoringSNSTopic 198 | AlarmDescription: !Sub "Alarm if Firehose ${AWS::AccountId}-${AWS::Region}-${logType}-firehose is throttling events because of quota limits." 199 | AlarmName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose-throttling-alarm" 200 | ComparisonOperator: GreaterThanThreshold 201 | Dimensions: 202 | - Name: DeliveryStreamName 203 | Value: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose" 204 | EvaluationPeriods: 1 205 | MetricName: ThrottledRecords 206 | Namespace: AWS/Firehose 207 | Period: 60 208 | Statistic: Maximum 209 | Threshold: 0 210 | Unit: Count 211 | 212 | # CloudWatch Event Rule 213 | cloudWatchEventsPolicy: 214 | Type: AWS::IAM::ManagedPolicy 215 | Properties: 216 | PolicyDocument: 217 | Version: 2012-10-17 218 | Statement: 219 | - Effect: Allow 220 | Action: 221 | - firehose:PutRecordBatch 222 | - firehose:PutRecord 223 | Resource: !GetAtt firehose.Arn 224 | ManagedPolicyName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-cwe-iam-policy" 225 | 226 | cloudWatchEventsRole: 227 | Type: AWS::IAM::Role 228 | Properties: 229 | AssumeRolePolicyDocument: 230 | Version: 2012-10-17 231 | Statement: 232 | - Effect: Allow 233 | Principal: 234 | Service: 235 | - events.amazonaws.com 236 | Action: sts:AssumeRole 237 | ManagedPolicyArns: 238 | - !Ref cloudWatchEventsPolicy 239 | RoleName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-cwe-iam-role" 240 | Tags: 241 | - Key: service 242 | Value: !Ref service 243 | - Key: stage 244 | Value: !Ref stage 245 | - Key: contact 246 | Value: !Ref contact 247 | 248 | cloudWatchEventsRule: 249 | Type: AWS::Events::Rule 250 | Properties: 251 | EventPattern: 252 | source: 253 | - !Sub "aws.${logType}" 254 | Targets: 255 | - Arn: !GetAtt firehose.Arn 256 | RoleArn: !GetAtt cloudWatchEventsRole.Arn 257 | Id: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-cwe" 258 | 259 | Outputs: 260 | firehoseArn: 261 | Value: !GetAtt firehose.Arn 262 | firehoseIAMRoleArn: 263 | Value: !GetAtt firehoseIAMRole.Arn 264 | firehoseBacksplashBucketArn: 265 | Value: !GetAtt firehoseBacksplashBucket.Arn 266 | firehoseLogGroupArn: 267 | Value: !GetAtt firehoseLogGroup.Arn 268 | monitoringSNSTopicArn: 269 | Condition: enableAlerting 270 | Value: !Ref monitoringSNSTopic 271 | cloudWatchEventsRole: 272 | Value: !GetAtt cloudWatchEventsRole.Arn 273 | cloudWatchEventsRule: 274 | Value: !GetAtt cloudWatchEventsRule.Arn -------------------------------------------------------------------------------- /CloudWatchEvents-Firehose-Resources/splunkDashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "visualizations": { 3 | "viz_AcyVeBot": { 4 | "type": "splunk.column", 5 | "showProgressBar": false, 6 | "showLastUpdated": false, 7 | "title": "$sourcetype$ License Usage Over Time (GB)", 8 | "dataSources": { 9 | "primary": "ds_6WBt9iY9" 10 | }, 11 | "options": { 12 | "legendDisplay": "off" 13 | } 14 | }, 15 | "viz_KMF849o7": { 16 | "type": "splunk.singlevalue", 17 | "title": "$sourcetype$ Event License Usage", 18 | "dataSources": { 19 | "primary": "ds_HUEXsOZi" 20 | }, 21 | "options": { 22 | "numberPrecision": 2, 23 | "unit": "GB", 24 | "trendDisplay": "off", 25 | "sparklineDisplay": "off" 26 | } 27 | }, 28 | "viz_OF3a0hX5": { 29 | "type": "splunk.singlevalue", 30 | "title": "$sourcetype$ Event Count", 31 | "dataSources": { 32 | "primary": "ds_LN5KxPiz" 33 | }, 34 | "options": { 35 | "unit": "events", 36 | "trendDisplay": "off", 37 | "sparklineDisplay": "off" 38 | } 39 | }, 40 | "viz_kqPNrs3R": { 41 | "type": "splunk.line", 42 | "showProgressBar": false, 43 | "showLastUpdated": false, 44 | "title": "$sourcetype$ Event Count Over Time", 45 | "dataSources": { 46 | "primary": "ds_7YVGDkt0" 47 | }, 48 | "options": { 49 | "legendDisplay": "off" 50 | } 51 | }, 52 | "viz_eU3T7rCG": { 53 | "type": "splunk.singlevalue", 54 | "title": "AWS Accounts Sending Events", 55 | "dataSources": { 56 | "primary": "ds_noJxIRWy" 57 | } 58 | }, 59 | "viz_nO6QvmkM": { 60 | "type": "splunk.line", 61 | "title": "AWS Accounts Sending $sourcetype$ Events", 62 | "dataSources": { 63 | "primary": "ds_FFly4UzK" 64 | }, 65 | "options": { 66 | "legendDisplay": "off" 67 | } 68 | }, 69 | "viz_NY0AedMk": { 70 | "type": "splunk.line", 71 | "title": "$sourcetype$ Event Latency Over Time", 72 | "showProgressBar": false, 73 | "showLastUpdated": false, 74 | "dataSources": { 75 | "primary": "ds_srz6OcK5" 76 | }, 77 | "options": { 78 | "legendDisplay": "off" 79 | } 80 | }, 81 | "viz_XqNxe7aK": { 82 | "type": "splunk.singlevalue", 83 | "title": "$sourcetype$ Latest Event Latency", 84 | "dataSources": { 85 | "primary": "ds_tCds3VFx" 86 | }, 87 | "options": { 88 | "sparklineDisplay": "off", 89 | "unit": "seconds" 90 | }, 91 | "showProgressBar": false, 92 | "showLastUpdated": false 93 | } 94 | }, 95 | "dataSources": { 96 | "ds_6WBt9iY9": { 97 | "type": "ds.search", 98 | "options": { 99 | "query": "index=_internal source=*license_usage* idx=$eventIndex$ st=$sourcetype$ NOT source=*license_usage_summary.log*\n| timechart sum(b) as ingestedDataGB\n| eval ingestedDataGB = ingestedDataGB/1024/1024/1024", 100 | "queryParameters": { 101 | "earliest": "$global_time.earliest$", 102 | "latest": "$global_time.latest$" 103 | } 104 | }, 105 | "name": "licenseUsageBase" 106 | }, 107 | "ds_HUEXsOZi": { 108 | "type": "ds.chain", 109 | "options": { 110 | "extend": "ds_6WBt9iY9", 111 | "query": "| stats sum(ingestedDataGB) as ingestedDataGB" 112 | }, 113 | "name": "licenseUsageTotal" 114 | }, 115 | "ds_7YVGDkt0": { 116 | "type": "ds.search", 117 | "options": { 118 | "query": "index=_internal series=$sourcetype$ sourcetype=splunkd source=*metrics.log*\n| timechart sum(ev) as eventCount" 119 | }, 120 | "name": "eventCountBase" 121 | }, 122 | "ds_LN5KxPiz": { 123 | "type": "ds.chain", 124 | "options": { 125 | "extend": "ds_7YVGDkt0", 126 | "query": "| stats sum(eventCount) as eventCount" 127 | }, 128 | "name": "eventCountTotal" 129 | }, 130 | "ds_FFly4UzK": { 131 | "type": "ds.search", 132 | "options": { 133 | "query": "index=$eventIndex$ sourcetype=$sourcetype$\n| timechart dc(vendor_account) as accountCount" 134 | }, 135 | "name": "accountCountTimechart" 136 | }, 137 | "ds_noJxIRWy": { 138 | "type": "ds.search", 139 | "options": { 140 | "query": "index=$eventIndex$ sourcetype=$sourcetype$\n| stats dc(vendor_account) as accountCount" 141 | }, 142 | "name": "accountCountTotal" 143 | }, 144 | "ds_srz6OcK5": { 145 | "type": "ds.search", 146 | "options": { 147 | "query": "index=$eventIndex$ sourcetype=$sourcetype$ \n| eval eventLatency = abs(_indextime - _time)\n| timechart avg(eventLatency) as eventLatency" 148 | }, 149 | "name": "eventLatencyBase" 150 | }, 151 | "ds_tCds3VFx": { 152 | "type": "ds.search", 153 | "options": { 154 | "query": "index=$eventIndex$ sourcetype=$sourcetype$ | head 1\n| eval eventLatency = abs(_indextime - _time)\n| table eventLatency" 155 | }, 156 | "name": "eventLatencyLatest" 157 | }, 158 | "ds_N5KOizZ9": { 159 | "type": "ds.search", 160 | "options": { 161 | "query": "| tstats count where index=* by index", 162 | "queryParameters": { 163 | "earliest": "$global_time.earliest$", 164 | "latest": "$global_time.latest$" 165 | } 166 | }, 167 | "name": "indexList" 168 | }, 169 | "ds_f9m7vKr7": { 170 | "type": "ds.search", 171 | "options": { 172 | "query": "| tstats count where index=$eventIndex$ by sourcetype" 173 | }, 174 | "name": "sourcetypeList" 175 | } 176 | }, 177 | "defaults": { 178 | "dataSources": { 179 | "ds.search": { 180 | "options": { 181 | "queryParameters": { 182 | "latest": "$global_time.latest$", 183 | "earliest": "$global_time.earliest$" 184 | } 185 | } 186 | } 187 | } 188 | }, 189 | "inputs": { 190 | "input_global_trp": { 191 | "type": "input.timerange", 192 | "options": { 193 | "token": "global_time", 194 | "defaultValue": "-24h@h,now" 195 | }, 196 | "title": "Global Time Range" 197 | }, 198 | "input_9rNFa8NM": { 199 | "options": { 200 | "items": [ 201 | { 202 | "label": "main", 203 | "value": "main" 204 | } 205 | ], 206 | "defaultValue": "main", 207 | "token": "eventIndex" 208 | }, 209 | "title": "Event Index", 210 | "type": "input.dropdown", 211 | "dataSources": { 212 | "primary": "ds_N5KOizZ9" 213 | } 214 | }, 215 | "input_C3SY1WEW": { 216 | "options": { 217 | "items": [ 218 | { 219 | "label": "All", 220 | "value": "*" 221 | } 222 | ], 223 | "defaultValue": "*", 224 | "token": "sourcetype" 225 | }, 226 | "title": "Event Sourcetype", 227 | "type": "input.dropdown", 228 | "dataSources": { 229 | "primary": "ds_f9m7vKr7" 230 | } 231 | } 232 | }, 233 | "layout": { 234 | "type": "grid", 235 | "options": {}, 236 | "structure": [ 237 | { 238 | "item": "viz_eU3T7rCG", 239 | "type": "block", 240 | "position": { 241 | "x": 600, 242 | "y": 0, 243 | "w": 300, 244 | "h": 100 245 | } 246 | }, 247 | { 248 | "item": "viz_XqNxe7aK", 249 | "type": "block", 250 | "position": { 251 | "x": 900, 252 | "y": 0, 253 | "w": 300, 254 | "h": 100 255 | } 256 | }, 257 | { 258 | "item": "viz_KMF849o7", 259 | "type": "block", 260 | "position": { 261 | "x": 0, 262 | "y": 0, 263 | "w": 300, 264 | "h": 100 265 | } 266 | }, 267 | { 268 | "item": "viz_AcyVeBot", 269 | "type": "block", 270 | "position": { 271 | "x": 0, 272 | "y": 100, 273 | "w": 1200, 274 | "h": 300 275 | } 276 | }, 277 | { 278 | "item": "viz_kqPNrs3R", 279 | "type": "block", 280 | "position": { 281 | "x": 0, 282 | "y": 400, 283 | "w": 1200, 284 | "h": 300 285 | } 286 | }, 287 | { 288 | "item": "viz_nO6QvmkM", 289 | "type": "block", 290 | "position": { 291 | "x": 0, 292 | "y": 700, 293 | "w": 1200, 294 | "h": 300 295 | } 296 | }, 297 | { 298 | "item": "viz_NY0AedMk", 299 | "type": "block", 300 | "position": { 301 | "x": 0, 302 | "y": 1000, 303 | "w": 1200, 304 | "h": 300 305 | } 306 | }, 307 | { 308 | "item": "viz_OF3a0hX5", 309 | "type": "block", 310 | "position": { 311 | "x": 300, 312 | "y": 0, 313 | "w": 300, 314 | "h": 100 315 | } 316 | } 317 | ], 318 | "globalInputs": [ 319 | "input_global_trp", 320 | "input_9rNFa8NM", 321 | "input_C3SY1WEW" 322 | ] 323 | }, 324 | "description": "", 325 | "title": "Splunk AWS GDI Toolkit - CloudWatch-Events-Firehose" 326 | } -------------------------------------------------------------------------------- /CloudWatchLogs-Firehose-Resources/cwlToSplunk.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: This is a CloudFormation template to create infrastructure to send logs that are put into a CloudWatch Logs group. 4 | 5 | Parameters: 6 | 7 | service: 8 | Type: String 9 | Description: Service name used in tagging AWS resources. 10 | Default: splunk-aws-gdi-toolkit 11 | 12 | stage: 13 | Type: String 14 | Description: Used to distinguish between stages of an environment (dev, test, prod, stage, etc). Only used in AWS resource tagging. 15 | Default: dev 16 | 17 | contact: 18 | Description: Used to identify a contact for the resources created in this stack. Only used in AWS resource tagging. As an example, this could be an email address or username. 19 | Type: String 20 | Default: "" 21 | 22 | logType: 23 | Type: String 24 | Description: Log type, used in AWS resource names. 25 | AllowedPattern: "(?!(xn--|-s3alias$))^[a-z0-9][a-z0-9-]{1,20}[a-z0-9]$" 26 | 27 | cwlFilterPattern: 28 | Type: String 29 | Description: Filter pattern used control which data is sent to Splunk from CloudWatch Logs. Leave blank to capture all logs. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html 30 | Default: "" 31 | 32 | cwlLogGroupName: 33 | Type: String 34 | Description: CloudWatch log group name which has events that will be forwarded to Splunk. 35 | 36 | splunkHECEndpoint: 37 | Type: String 38 | Description: Destination URL that Firehose will send data to. 39 | 40 | splunkHECToken: 41 | Type: String 42 | Description: HEC token Firehose will use to authenticate data being sent to Splunk. 43 | 44 | splunkIndex: 45 | Type: String 46 | Description: Name of the index in Splunk events will be sent to. 47 | 48 | splunkSourcetype: 49 | Type: String 50 | Description: Name of the sourcetype for the events that will be sent to Splunk. 51 | 52 | splunkSource: 53 | Type: String 54 | Description: Name of the source for the events that will be sent to Splunk. 55 | 56 | splunkHost: 57 | Type: String 58 | Description: Name of the host for the events that will be sent to Splunk. 59 | 60 | cloudWatchAlertEmail: 61 | Type: String 62 | Description: Email address for receiving alerts related to CloudTrail ingestion. Leave empty for no alerting. 63 | Default: "" 64 | 65 | Conditions: 66 | enableAlerting: !Not 67 | - !Equals 68 | - !Ref cloudWatchAlertEmail 69 | - "" 70 | 71 | Resources: 72 | # CloudWatch Logs > Firehose resources 73 | cwlSubscription: 74 | Type: AWS::Logs::SubscriptionFilter 75 | Properties: 76 | DestinationArn: !GetAtt firehose.Arn 77 | FilterPattern: !Ref cwlFilterPattern 78 | LogGroupName: !Ref cwlLogGroupName 79 | RoleArn: !GetAtt cwlIAMRole.Arn 80 | 81 | cwlIAMPolicy: 82 | Type: AWS::IAM::ManagedPolicy 83 | Properties: 84 | PolicyDocument: 85 | Version: '2012-10-17' 86 | Statement: 87 | - Effect: Allow 88 | Action: 89 | - firehose:PutRecord 90 | - firehose:PutRecordBatch 91 | Resource: !GetAtt firehose.Arn 92 | ManagedPolicyName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-cwl-iam-policy" 93 | 94 | cwlIAMRole: 95 | Type: AWS::IAM::Role 96 | Properties: 97 | AssumeRolePolicyDocument: 98 | Version: '2012-10-17' 99 | Statement: 100 | - Effect: "Allow" 101 | Principal: 102 | Service: "logs.amazonaws.com" 103 | Action: 104 | - sts:AssumeRole 105 | ManagedPolicyArns: 106 | - !Ref cwlIAMPolicy 107 | RoleName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-cwl-iam-role" 108 | Tags: 109 | - Key: service 110 | Value: !Ref service 111 | - Key: stage 112 | Value: !Ref stage 113 | - Key: contact 114 | Value: !Ref contact 115 | 116 | # Firehose > Splunk resources 117 | firehose: 118 | Type: AWS::KinesisFirehose::DeliveryStream 119 | Properties: 120 | DeliveryStreamName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose" 121 | DeliveryStreamType: DirectPut 122 | SplunkDestinationConfiguration: 123 | BufferingHints: 124 | IntervalInSeconds: 5 125 | SizeInMBs: 1 126 | CloudWatchLoggingOptions: 127 | Enabled: true 128 | LogGroupName: !Ref firehoseLogGroup 129 | LogStreamName: "SplunkDelivery" 130 | HECAcknowledgmentTimeoutInSeconds: 300 131 | HECEndpoint: !Ref splunkHECEndpoint 132 | HECEndpointType: "Event" 133 | HECToken: !Ref splunkHECToken 134 | ProcessingConfiguration: 135 | Enabled: true 136 | Processors: 137 | - Parameters: 138 | - ParameterName: LambdaArn 139 | ParameterValue: !GetAtt lambdaFunction.Arn 140 | - ParameterName: RoleArn 141 | ParameterValue: !GetAtt firehoseIAMRole.Arn 142 | - ParameterName: BufferSizeInMBs 143 | ParameterValue: 1 144 | - ParameterName: BufferIntervalInSeconds 145 | ParameterValue: 5 146 | Type: Lambda 147 | RetryOptions: 148 | DurationInSeconds: 3600 149 | S3BackupMode: "FailedEventsOnly" 150 | S3Configuration: 151 | BucketARN: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-${logType}-firehose-backsplash" 152 | BufferingHints: 153 | IntervalInSeconds: 300 154 | SizeInMBs: 16 155 | CompressionFormat: "GZIP" 156 | Prefix: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose" 157 | RoleARN: !GetAtt firehoseIAMRole.Arn 158 | 159 | firehoseIAMPolicy: 160 | Type: AWS::IAM::ManagedPolicy 161 | Properties: 162 | PolicyDocument: 163 | Version: '2012-10-17' 164 | Statement: 165 | - Effect: Allow 166 | Action: 167 | - logs:Describe* 168 | - logs:PutLogEvents 169 | Resource: !GetAtt firehoseLogGroup.Arn 170 | - Effect: Allow 171 | Action: 172 | - s3:PutObject 173 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-${logType}-firehose-backsplash/*" 174 | - Effect: Allow 175 | Action: 176 | - lambda:InvokeFunction 177 | - lambda:GetFunctionConfiguration 178 | Resource: !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${AWS::AccountId}-${AWS::Region}-${logType}-lambda-function" 179 | ManagedPolicyName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose-iam-policy" 180 | 181 | firehoseIAMRole: 182 | Type: AWS::IAM::Role 183 | Properties: 184 | AssumeRolePolicyDocument: 185 | Version: '2012-10-17' 186 | Statement: 187 | - Effect: "Allow" 188 | Principal: 189 | Service: "firehose.amazonaws.com" 190 | Action: 191 | - sts:AssumeRole 192 | ManagedPolicyArns: 193 | - !Ref firehoseIAMPolicy 194 | RoleName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose-iam-role" 195 | Tags: 196 | - Key: service 197 | Value: !Ref service 198 | - Key: stage 199 | Value: !Ref stage 200 | - Key: contact 201 | Value: !Ref contact 202 | 203 | firehoseBacksplashBucket: 204 | Type: AWS::S3::Bucket 205 | Properties: 206 | BucketName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose-backsplash" 207 | AccessControl: Private 208 | PublicAccessBlockConfiguration: 209 | BlockPublicAcls: true 210 | BlockPublicPolicy: true 211 | IgnorePublicAcls: true 212 | RestrictPublicBuckets: true 213 | BucketEncryption: 214 | ServerSideEncryptionConfiguration: 215 | - ServerSideEncryptionByDefault: 216 | SSEAlgorithm: AES256 217 | LifecycleConfiguration: 218 | Rules: 219 | - Id: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose-backsplash-cleanup" 220 | AbortIncompleteMultipartUpload: 221 | DaysAfterInitiation: 1 222 | Status: Enabled 223 | Tags: 224 | - Key: service 225 | Value: !Ref service 226 | - Key: stage 227 | Value: !Ref stage 228 | - Key: contact 229 | Value: !Ref contact 230 | 231 | firehoseLogGroup: 232 | Type: AWS::Logs::LogGroup 233 | Properties: 234 | LogGroupName: !Sub "/aws/kinesisfirehose/${AWS::AccountId}-${AWS::Region}-${logType}-firehose" 235 | RetentionInDays: 7 236 | 237 | firehoseLogStream: 238 | Type: AWS::Logs::LogStream 239 | Properties: 240 | LogGroupName: !Ref firehoseLogGroup 241 | LogStreamName: "SplunkDelivery" 242 | 243 | # Lambda resources 244 | lambdaFunction: 245 | Type: AWS::Lambda::Function 246 | DependsOn: lambdaLogGroup 247 | Properties: 248 | Architectures: 249 | - arm64 250 | Code: 251 | ZipFile: | 252 | import os, gzip, json, base64 253 | 254 | SPLUNK_SOURCE = os.environ['SPLUNK_SOURCE'] 255 | SPLUNK_SOURCETYPE = os.environ['SPLUNK_SOURCETYPE'] 256 | SPLUNK_HOST = os.environ['SPLUNK_HOST'] 257 | SPLUNK_INDEX = os.environ['SPLUNK_INDEX'] 258 | 259 | # Default Lambda handler 260 | def handler(event, context): 261 | 262 | returnRecords = [] 263 | 264 | # Decode events, and split into separate items in a list 265 | for record in event['records']: 266 | 267 | # Decode and uncompress raw log event 268 | decodedData = json.loads(gzip.decompress(base64.b64decode(record['data']))) 269 | 270 | formattedEvents = "" 271 | returnEvent = {} 272 | 273 | # Loop through each log event, construct event, add to return array 274 | for logEvent in decodedData['logEvents']: 275 | 276 | # Format Splunk event 277 | formattedEvents += '{ "time": ' + str(logEvent['timestamp']) + ', "host": "' + SPLUNK_HOST + '", "source": "' + SPLUNK_SOURCE + '", "sourcetype": "' + SPLUNK_SOURCETYPE + '", "index": "' + SPLUNK_INDEX + '", "event": ' + json.dumps(logEvent['message']) + '}' 278 | 279 | # Construct return event 280 | returnEvent['recordId'] = dict(record)['recordId'] 281 | returnEvent['result'] = "Ok" 282 | returnEvent['data'] = base64.b64encode(formattedEvents.encode('utf-8')).decode() 283 | 284 | # Print for debugging 285 | print("Processed record " + record['recordId']) 286 | 287 | # Add return events to array to return all the records to Firehose 288 | returnRecords.append(returnEvent) 289 | 290 | return {'records': returnRecords} 291 | Description: Lambda function for processing Firehose messages into standard event format for Splunk. 292 | Environment: 293 | Variables: 294 | SPLUNK_INDEX: !Ref splunkIndex 295 | SPLUNK_SOURCE: !Ref splunkSource 296 | SPLUNK_SOURCETYPE: !Ref splunkSourcetype 297 | SPLUNK_HOST: !Ref splunkHost 298 | FunctionName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-lambda-function" 299 | Handler: index.handler 300 | MemorySize: 512 301 | Role: !GetAtt lambdaIAMRole.Arn 302 | Runtime: python3.12 303 | Tags: 304 | - Key: service 305 | Value: !Ref service 306 | - Key: stage 307 | Value: !Ref stage 308 | - Key: contact 309 | Value: !Ref contact 310 | Timeout: 300 311 | 312 | lambdaIAMPolicy: 313 | Type: AWS::IAM::ManagedPolicy 314 | Properties: 315 | PolicyDocument: 316 | Version: '2012-10-17' 317 | Statement: 318 | - Effect: Allow 319 | Action: 320 | - firehose:PutRecord 321 | - firehose:PutRecordBatch 322 | Resource: !Sub "arn:aws:firehose:${AWS::Region}:${AWS::AccountId}:deliverystream/${AWS::AccountId}-${AWS::Region}-${logType}-firehose" 323 | - Effect: Allow 324 | Action: 325 | - logs:CreateLogStream 326 | - logs:PutLogEvents 327 | Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${AWS::AccountId}-${AWS::Region}-${logType}-lambda-function*" 328 | ManagedPolicyName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-lambda-iam-policy" 329 | 330 | lambdaIAMRole: 331 | Type: AWS::IAM::Role 332 | Properties: 333 | AssumeRolePolicyDocument: 334 | Version: '2012-10-17' 335 | Statement: 336 | - Effect: "Allow" 337 | Principal: 338 | Service: "lambda.amazonaws.com" 339 | Action: 340 | - sts:AssumeRole 341 | ManagedPolicyArns: 342 | - !Ref lambdaIAMPolicy 343 | RoleName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-lambda-iam-role" 344 | Tags: 345 | - Key: service 346 | Value: !Ref service 347 | - Key: stage 348 | Value: !Ref stage 349 | - Key: contact 350 | Value: !Ref contact 351 | 352 | lambdaLogGroup: 353 | Type: AWS::Logs::LogGroup 354 | Properties: 355 | LogGroupName: !Sub "/aws/lambda/${AWS::AccountId}-${AWS::Region}-${logType}-lambda-function" 356 | RetentionInDays: 7 357 | 358 | # Monitoring resoruces 359 | monitoringSNSTopic: 360 | Condition: enableAlerting 361 | Type: AWS::SNS::Topic 362 | Properties: 363 | DisplayName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-alerting-topic" 364 | TopicName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-alerting-topic" 365 | Subscription: 366 | - Endpoint: !Ref cloudWatchAlertEmail 367 | Protocol: email 368 | 369 | lambdaInvocationAlarm: 370 | Condition: enableAlerting 371 | Type: AWS::CloudWatch::Alarm 372 | Properties: 373 | ActionsEnabled: True 374 | AlarmActions: 375 | - !Ref monitoringSNSTopic 376 | AlarmDescription: !Sub "Alarm if lambda function ${AWS::AccountId}-${AWS::Region}-${logType}-lambda-function errors out. Check CloudWatch Logs to verify the function is running correctly." 377 | AlarmName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-lambda-error-invocations" 378 | ComparisonOperator: GreaterThanOrEqualToThreshold 379 | Dimensions: 380 | - Name: FunctionName 381 | Value: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-lambda-function" 382 | EvaluationPeriods: 1 383 | MetricName: Errors 384 | Namespace: AWS/Lambda 385 | Period: 60 386 | Statistic: Sum 387 | Threshold: 1 388 | Unit: Count 389 | 390 | lambdaDurationAlarm: 391 | Condition: enableAlerting 392 | Type: AWS::CloudWatch::Alarm 393 | Properties: 394 | ActionsEnabled: True 395 | AlarmActions: 396 | - !Ref monitoringSNSTopic 397 | AlarmDescription: !Sub "Alarm if lambda function ${AWS::AccountId}-${AWS::Region}-${logType}-lambda-function runs over 5 minutes." 398 | AlarmName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-lambda-error-duration" 399 | ComparisonOperator: GreaterThanOrEqualToThreshold 400 | Dimensions: 401 | - Name: FunctionName 402 | Value: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-lambda-function" 403 | EvaluationPeriods: 1 404 | MetricName: Duration 405 | Namespace: AWS/Lambda 406 | Period: 60 407 | Statistic: Maximum 408 | Threshold: 300000 409 | Unit: Milliseconds 410 | 411 | firehoseDeliveryAlarm: 412 | Condition: enableAlerting 413 | Type: AWS::CloudWatch::Alarm 414 | Properties: 415 | ActionsEnabled: True 416 | AlarmActions: 417 | - !Ref monitoringSNSTopic 418 | AlarmDescription: !Sub "Alarm if Firehose ${AWS::AccountId}-${AWS::Region}-${logType}-firehose cannot deliver to Splunk." 419 | AlarmName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose-delivery-alarm" 420 | ComparisonOperator: LessThanThreshold 421 | Dimensions: 422 | - Name: DeliveryStreamName 423 | Value: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose" 424 | EvaluationPeriods: 1 425 | MetricName: DeliveryToSplunk.Success 426 | Namespace: AWS/Firehose 427 | Period: 60 428 | Statistic: Maximum 429 | Threshold: 1 430 | Unit: Count 431 | 432 | firehoseThrottlingAlarm: 433 | Condition: enableAlerting 434 | Type: AWS::CloudWatch::Alarm 435 | Properties: 436 | ActionsEnabled: True 437 | AlarmActions: 438 | - !Ref monitoringSNSTopic 439 | AlarmDescription: !Sub "Alarm if Firehose ${AWS::AccountId}-${AWS::Region}-${logType}-firehose is throttling events because of quota limits." 440 | AlarmName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose-throttling-alarm" 441 | ComparisonOperator: GreaterThanThreshold 442 | Dimensions: 443 | - Name: DeliveryStreamName 444 | Value: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-firehose" 445 | EvaluationPeriods: 1 446 | MetricName: ThrottledRecords 447 | Namespace: AWS/Firehose 448 | Period: 60 449 | Statistic: Maximum 450 | Threshold: 0 451 | Unit: Count 452 | 453 | Outputs: 454 | cwlSubscriptionID: 455 | Value: !Ref cwlSubscription 456 | cwlIAMRoleArn: 457 | Value: !GetAtt cwlIAMRole.Arn 458 | firehoseArn: 459 | Value: !GetAtt firehose.Arn 460 | firehoseIAMRoleArn: 461 | Value: !GetAtt firehoseIAMRole.Arn 462 | firehoseBacksplashBucketArn: 463 | Value: !GetAtt firehoseBacksplashBucket.Arn 464 | firehoseLogGroupArn: 465 | Value: !GetAtt firehoseLogGroup.Arn 466 | lambdaFunctionArn: 467 | Value: !GetAtt lambdaFunction.Arn 468 | lambdaIAMRoleArn: 469 | Value: !GetAtt lambdaIAMRole.Arn 470 | lambdaLogGroupArn: 471 | Value: !GetAtt lambdaLogGroup.Arn -------------------------------------------------------------------------------- /CloudWatchLogs-Firehose-Resources/lambda.py: -------------------------------------------------------------------------------- 1 | import os, gzip, json, base64 2 | 3 | SPLUNK_SOURCE = os.environ['SPLUNK_SOURCE'] 4 | SPLUNK_SOURCETYPE = os.environ['SPLUNK_SOURCETYPE'] 5 | SPLUNK_HOST = os.environ['SPLUNK_HOST'] 6 | SPLUNK_INDEX = os.environ['SPLUNK_INDEX'] 7 | 8 | # Default Lambda handler 9 | def handler(event, context): 10 | 11 | returnRecords = [] 12 | 13 | # Decode events, and split into separate items in a list 14 | for record in event['records']: 15 | 16 | # Decode and uncompress raw log event 17 | decodedData = json.loads(gzip.decompress(base64.b64decode(record['data']))) 18 | 19 | formattedEvents = "" 20 | returnEvent = {} 21 | 22 | # Loop through each log event, construct event, add to return array 23 | for logEvent in decodedData['logEvents']: 24 | 25 | # Format Splunk event 26 | formattedEvents += '{ "time": ' + str(logEvent['timestamp']) + ', "host": "' + SPLUNK_HOST + '", "source": "' + SPLUNK_SOURCE + '", "sourcetype": "' + SPLUNK_SOURCETYPE + '", "index": "' + SPLUNK_INDEX + '", "event": ' + json.dumps(logEvent['message']) + '}' 27 | 28 | # Construct return event 29 | returnEvent['recordId'] = dict(record)['recordId'] 30 | returnEvent['result'] = "Ok" 31 | returnEvent['data'] = base64.b64encode(formattedEvents.encode('utf-8')).decode() 32 | 33 | # Print for debugging 34 | print("Processed record " + record['recordId']) 35 | 36 | # Add return events to array to return all the records to Firehose 37 | returnRecords.append(returnEvent) 38 | 39 | return {'records': returnRecords} 40 | -------------------------------------------------------------------------------- /CloudWatchLogs-Firehose-Resources/splunkDashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "visualizations": { 3 | "viz_AcyVeBot": { 4 | "type": "splunk.column", 5 | "showProgressBar": false, 6 | "showLastUpdated": false, 7 | "title": "$sourcetype$ License Usage Over Time (GB)", 8 | "dataSources": { 9 | "primary": "ds_6WBt9iY9" 10 | }, 11 | "options": { 12 | "legendDisplay": "off" 13 | } 14 | }, 15 | "viz_KMF849o7": { 16 | "type": "splunk.singlevalue", 17 | "title": "$sourcetype$ Event License Usage", 18 | "dataSources": { 19 | "primary": "ds_HUEXsOZi" 20 | }, 21 | "options": { 22 | "numberPrecision": 2, 23 | "unit": "GB", 24 | "trendDisplay": "off", 25 | "sparklineDisplay": "off" 26 | } 27 | }, 28 | "viz_OF3a0hX5": { 29 | "type": "splunk.singlevalue", 30 | "title": "$sourcetype$ Event Count", 31 | "dataSources": { 32 | "primary": "ds_LN5KxPiz" 33 | }, 34 | "options": { 35 | "unit": "events", 36 | "trendDisplay": "off", 37 | "sparklineDisplay": "off" 38 | } 39 | }, 40 | "viz_kqPNrs3R": { 41 | "type": "splunk.line", 42 | "showProgressBar": false, 43 | "showLastUpdated": false, 44 | "title": "$sourcetype$ Event Count Over Time", 45 | "dataSources": { 46 | "primary": "ds_7YVGDkt0" 47 | }, 48 | "options": { 49 | "legendDisplay": "off" 50 | } 51 | }, 52 | "viz_NY0AedMk": { 53 | "type": "splunk.line", 54 | "title": "$sourcetype$ Event Latency Over Time", 55 | "showProgressBar": false, 56 | "showLastUpdated": false, 57 | "dataSources": { 58 | "primary": "ds_srz6OcK5" 59 | }, 60 | "options": { 61 | "legendDisplay": "off" 62 | } 63 | }, 64 | "viz_XqNxe7aK": { 65 | "type": "splunk.singlevalue", 66 | "title": "$sourcetype$ Latest Event Latency", 67 | "dataSources": { 68 | "primary": "ds_tCds3VFx" 69 | }, 70 | "options": { 71 | "sparklineDisplay": "off", 72 | "unit": "seconds" 73 | }, 74 | "showProgressBar": false, 75 | "showLastUpdated": false 76 | } 77 | }, 78 | "dataSources": { 79 | "ds_6WBt9iY9": { 80 | "type": "ds.search", 81 | "options": { 82 | "query": "index=_internal source=*license_usage* idx=$eventIndex$ st=$sourcetype$ NOT source=*license_usage_summary.log*\n| timechart sum(b) as ingestedDataGB\n| eval ingestedDataGB = ingestedDataGB/1024/1024/1024", 83 | "queryParameters": { 84 | "earliest": "$global_time.earliest$", 85 | "latest": "$global_time.latest$" 86 | } 87 | }, 88 | "name": "licenseUsageBase" 89 | }, 90 | "ds_HUEXsOZi": { 91 | "type": "ds.chain", 92 | "options": { 93 | "extend": "ds_6WBt9iY9", 94 | "query": "| stats sum(ingestedDataGB) as ingestedDataGB" 95 | }, 96 | "name": "licenseUsageTotal" 97 | }, 98 | "ds_7YVGDkt0": { 99 | "type": "ds.search", 100 | "options": { 101 | "query": "index=_internal series=$sourcetype$ sourcetype=splunkd source=*metrics.log*\n| timechart sum(ev) as eventCount" 102 | }, 103 | "name": "eventCountBase" 104 | }, 105 | "ds_LN5KxPiz": { 106 | "type": "ds.chain", 107 | "options": { 108 | "extend": "ds_7YVGDkt0", 109 | "query": "| stats sum(eventCount) as eventCount" 110 | }, 111 | "name": "eventCountTotal" 112 | }, 113 | "ds_srz6OcK5": { 114 | "type": "ds.search", 115 | "options": { 116 | "query": "index=$eventIndex$ sourcetype=$sourcetype$ \n| eval eventLatency = abs(_indextime - _time)\n| timechart avg(eventLatency) as eventLatency" 117 | }, 118 | "name": "eventLatencyBase" 119 | }, 120 | "ds_tCds3VFx": { 121 | "type": "ds.search", 122 | "options": { 123 | "query": "index=$eventIndex$ sourcetype=$sourcetype$ | head 1\n| eval eventLatency = abs(_indextime - _time)\n| table eventLatency" 124 | }, 125 | "name": "eventLatencyLatest" 126 | }, 127 | "ds_N5KOizZ9": { 128 | "type": "ds.search", 129 | "options": { 130 | "query": "| tstats count where index=* by index", 131 | "queryParameters": { 132 | "earliest": "$global_time.earliest$", 133 | "latest": "$global_time.latest$" 134 | } 135 | }, 136 | "name": "indexList" 137 | }, 138 | "ds_f9m7vKr7": { 139 | "type": "ds.search", 140 | "options": { 141 | "query": "| tstats count where index=$eventIndex$ by sourcetype" 142 | }, 143 | "name": "sourcetypeList" 144 | } 145 | }, 146 | "defaults": { 147 | "dataSources": { 148 | "ds.search": { 149 | "options": { 150 | "queryParameters": { 151 | "latest": "$global_time.latest$", 152 | "earliest": "$global_time.earliest$" 153 | } 154 | } 155 | } 156 | } 157 | }, 158 | "inputs": { 159 | "input_global_trp": { 160 | "type": "input.timerange", 161 | "options": { 162 | "token": "global_time", 163 | "defaultValue": "-24h@h,now" 164 | }, 165 | "title": "Global Time Range" 166 | }, 167 | "input_9rNFa8NM": { 168 | "options": { 169 | "items": [ 170 | { 171 | "label": "main", 172 | "value": "main" 173 | } 174 | ], 175 | "defaultValue": "main", 176 | "token": "eventIndex" 177 | }, 178 | "title": "Event Index", 179 | "type": "input.dropdown", 180 | "dataSources": { 181 | "primary": "ds_N5KOizZ9" 182 | } 183 | }, 184 | "input_C3SY1WEW": { 185 | "options": { 186 | "items": [ 187 | { 188 | "label": "All", 189 | "value": "*" 190 | } 191 | ], 192 | "defaultValue": "*", 193 | "token": "sourcetype" 194 | }, 195 | "title": "Event Sourcetype", 196 | "type": "input.dropdown", 197 | "dataSources": { 198 | "primary": "ds_f9m7vKr7" 199 | } 200 | } 201 | }, 202 | "layout": { 203 | "type": "grid", 204 | "options": {}, 205 | "structure": [ 206 | { 207 | "item": "viz_OF3a0hX5", 208 | "type": "block", 209 | "position": { 210 | "x": 400, 211 | "y": 0, 212 | "w": 400, 213 | "h": 100 214 | } 215 | }, 216 | { 217 | "item": "viz_XqNxe7aK", 218 | "type": "block", 219 | "position": { 220 | "x": 800, 221 | "y": 0, 222 | "w": 400, 223 | "h": 100 224 | } 225 | }, 226 | { 227 | "item": "viz_KMF849o7", 228 | "type": "block", 229 | "position": { 230 | "x": 0, 231 | "y": 0, 232 | "w": 400, 233 | "h": 100 234 | } 235 | }, 236 | { 237 | "item": "viz_AcyVeBot", 238 | "type": "block", 239 | "position": { 240 | "x": 0, 241 | "y": 100, 242 | "w": 1200, 243 | "h": 300 244 | } 245 | }, 246 | { 247 | "item": "viz_kqPNrs3R", 248 | "type": "block", 249 | "position": { 250 | "x": 0, 251 | "y": 400, 252 | "w": 1200, 253 | "h": 300 254 | } 255 | }, 256 | { 257 | "item": "viz_NY0AedMk", 258 | "type": "block", 259 | "position": { 260 | "x": 0, 261 | "y": 700, 262 | "w": 1200, 263 | "h": 300 264 | } 265 | } 266 | ], 267 | "globalInputs": [ 268 | "input_global_trp", 269 | "input_9rNFa8NM", 270 | "input_C3SY1WEW" 271 | ] 272 | }, 273 | "description": "", 274 | "title": "Splunk AWS GDI Toolkit - CloudWatchLogs-Firehose" 275 | } -------------------------------------------------------------------------------- /CloudWatchLogs-Firehose-Resources/tests.py: -------------------------------------------------------------------------------- 1 | import unittest, os, importlib 2 | 3 | 4 | class CloudWatchLogs_Firehose_Resources_Tests(unittest.TestCase): 5 | 6 | def setUp(self): 7 | # Set environment variables 8 | os.environ['SPLUNK_SOURCE'] = "841154226728" 9 | os.environ['SPLUNK_SOURCETYPE'] = "aws:cloudwatchlogs" 10 | os.environ['SPLUNK_HOST'] = "841154226728" 11 | os.environ['SPLUNK_INDEX'] = "aws" 12 | os.environ['PRESERVE_DOUBLE_BACKSLASHES_IN_EVENT'] = "false" 13 | 14 | # import lambda module to test 15 | self.lambda_module = importlib.import_module('lambda') 16 | 17 | 18 | def test_one(self): 19 | 20 | testInput = {'invocationId': 'becd65b6-252a-4e66-bbb1-1001c091abc3', 'deliveryStreamArn': 'arn:aws:firehose:us-west-2:841154226728:deliverystream/841154226728-us-west-2-lambdacloudtrail-firehose', 'region': 'us-west-2', 'records': [{'recordId': '49636824083378936888143884586489700312348099680449593346000000', 'approximateArrivalTimestamp': 1673385843089, 'data': 'H4sIAAAAAAAAANVU207bQBD9FcvqI8Z7mb3lzSGBVgoNxYZUXBQ5ziYyOHHwhQQQ/95xoOCqRRSpL/WL5ZmzZ86cmfWDu7BlGc9tdLeybsftBVEwPuyHYXDQd3fcfL20BYY1UCqAMamYxnCWzw+KvF5hxo/XpZ/Fi8k09tsory69tS0rj3lJltfTqojTzHsCerN6mVRpvnyiCqvCxgvkYoRxn1CfEv/80yCI+mF0CWY2kYzKCTcKuJwY4LGlM8K0igk1CinKelImRbpqGPfTrLJF6XbO3adae03xqCm+NxpE+V6RTjIvWWdh65BXsdG30+Fm3rvd7LuXW1H9W7usGp4HN52iNq44VUpQSrA/tu2SGkIpI6ANYAjFKTBSQ2OCAcq10JwA6qtS9LiKF2gXlYpvE9gN3/npPdKHUXAcOcf2pkbol2nH0WZKZiCZp6bEesIkxDNWcy/WQsSghcLHOcVWUX/HeXbrYuk+7vwmWBgtlZSKEiKE4iAEEMkU0ZJJQYiiSnBiKBO6IX1DMFAgbcFHRZ7gh506szSzTsk7/vvz94NROMjn5S9I/3VE/sshv7ULbfD4FTx+AY8bMM6CRAzncTbev2FH93qzGiRwfRWOhrtXZb7cnd//2R5lCEiOL0WZ4VJTAMZBaYIzMqABrcMhM2kIR+fesgfl/Rf2iLPx59Udue5+73aH3RMRhKP37MHWeVOBM6WNVtiykpxKSiVjeAHAaGo40xr9om/bo9r29L/2Prrs/0Ad+0t1x/2j4cdv40XVq4u42t5HJviuIs6ivKi6aZbhGrRzsE0c2kVe3Dlhem87Dv5WwDnsYjTeOM+ZE1wfLM238ab9y8cfXFONoq8FAAA='}, {'recordId': '49636824083378936888143884602226287706271729640464711682000000', 'approximateArrivalTimestamp': 1673385872981, 'data': 'H4sIAAAAAAAAANVUwW7bSAz9FUHYYxRxhpzhjG9u4mYXSNokdpNFm8CQrLGhQrayklwnLfrvSzvd1tvWqbvoZaGTSA7n8fG9+RDPQ9tmszB6uAtxLz7uj/rjs8Fw2D8ZxAdxvVqERsKOlDKktWXtJFzVs5OmXt5JJs1WbVpl87zI0u2qZNkmq9B2iU4mVb0suiYrq+SxMJkuF5OurBePrYZdE7K59NKgMQWVKkjf/HbaHw2Go1vy09xqZXP0TGhzT5gFNQXtOAPlWVq0y7ydNOXduuPzsupC08a9N/HjXUfry0fry4+uT0f1UVPmVTJZVcOtQ0mnry+uXt7Pjt/dP49vN6AG78KiW/f5EJeFYENGxWwUW288ACnjtITIMSnttbVOkWYiZzUKUIcKyCmvBV9XCsddNhe6lGVEZ5xF5/ngH+6l/XDUvxxFl+GvpZT+UfSiwAiBmJKscCExGHziJtM8yR1Jf022MCG6klEFfy/6xNbNIv548DVgRuGVBZYA8igI0SpnDFlmx1YGMg6B2XvHxjizAzCB8duAz5t6Ij+hiKZlFaIWe+mP95/2r4en9az9V2X6ZUXp50Pplha2i8dfisefi8frYlAKRlpp83o8PTcXD1eDV6uHRl3U+cXh27ZeHM7e76JHG7AIBsCLoJx35D0DoAHjPCBYa72knbEAu/ZJYGGbnsGL45/d5i9Ah3uiuxycv/x5ud10x8sm6zaCUxYPjY/m7U33rKwqkcF2jjaJszCvm4doWL4PEgRN0dkziWb30afMK5FPL3K4iX9vfIfiM7BedKuJ2ZIiYJGBTOtEDtYIMfJZIjTaKLtjfMteP2m2jHWGptBJNilk+gkWSc4+T6wPho0kxdl7mc0JDucEkUe24jpA5WRn4IFJgLDMYpmAyKORM7v2ZYX1/4vZXnSzP+vF+Vt18vui41P7lNmEHnkXUbhx2oHSMq00kt0Srh9WQiSj5FU3Bjeb300PPWG2fbb5C9CpPdF9a7a9AG6ZzahDq7/vNaP/o9VuP/4N69+DI/sHAAA='}]} 21 | 22 | expectedOutput = {'records': [{'recordId': '49636824083378936888143884586489700312348099680449593346000000', 'result': 'Ok', 'data': 'eyAidGltZSI6IDE2NzMzODU4MzM5NzMsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJTVEFSVCBSZXF1ZXN0SWQ6IDg5ZDBmNDYyLTdkMGUtNTljMC05ZTgzLWE4NTVhNDg1Nzc3NyBWZXJzaW9uOiAkTEFURVNUXG4ifXsgInRpbWUiOiAxNjczMzg1ODM0MTQwLCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUHJvY2Vzc2VkIGZpbGUgczM6Ly84NDExNTQyMjY3MjgtdXMtd2VzdC0yLWNsb3VkdHJhaWwvQVdTTG9ncy84NDExNTQyMjY3MjgvQ2xvdWRUcmFpbC91cy13ZXN0LTIvMjAyMy8wMS8xMC84NDExNTQyMjY3MjhfQ2xvdWRUcmFpbF91cy13ZXN0LTJfMjAyMzAxMTBUMjEyMFpfRnEyUHo4eHBMYzRralNXTy5qc29uLmd6XG4ifXsgInRpbWUiOiAxNjczMzg1ODM0MjI2LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUHJvY2Vzc2VkIGZpbGUgczM6Ly84NDExNTQyMjY3MjgtdXMtd2VzdC0yLWNsb3VkdHJhaWwvQVdTTG9ncy84NDExNTQyMjY3MjgvQ2xvdWRUcmFpbC91cy13ZXN0LTIvMjAyMy8wMS8xMC84NDExNTQyMjY3MjhfQ2xvdWRUcmFpbF91cy13ZXN0LTJfMjAyMzAxMTBUMjEyNVpfSHB5MGtCWEJCT0JVNUFTVy5qc29uLmd6XG4ifXsgInRpbWUiOiAxNjczMzg1ODM0MjI3LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiRU5EIFJlcXVlc3RJZDogODlkMGY0NjItN2QwZS01OWMwLTllODMtYTg1NWE0ODU3Nzc3XG4ifXsgInRpbWUiOiAxNjczMzg1ODM0MjI3LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUkVQT1JUIFJlcXVlc3RJZDogODlkMGY0NjItN2QwZS01OWMwLTllODMtYTg1NWE0ODU3Nzc3XHREdXJhdGlvbjogMjUzLjcwIG1zXHRCaWxsZWQgRHVyYXRpb246IDI1NCBtc1x0TWVtb3J5IFNpemU6IDEwMjQgTUJcdE1heCBNZW1vcnkgVXNlZDogODMgTUJcdFxuIn0='}, {'recordId': '49636824083378936888143884602226287706271729640464711682000000', 'result': 'Ok', 'data': 'eyAidGltZSI6IDE2NzMzODU4NjM4OTcsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJTVEFSVCBSZXF1ZXN0SWQ6IGU3MzBlNDc0LWFkOGUtNTNlOS04Y2ZiLWI4NDM3NDI0NmQ1ZSBWZXJzaW9uOiAkTEFURVNUXG4ifXsgInRpbWUiOiAxNjczMzg1ODY0MDU5LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUHJvY2Vzc2VkIGZpbGUgczM6Ly84NDExNTQyMjY3MjgtdXMtd2VzdC0yLWNsb3VkdHJhaWwvQVdTTG9ncy84NDExNTQyMjY3MjgvQ2xvdWRUcmFpbC91cy13ZXN0LTIvMjAyMy8wMS8xMC84NDExNTQyMjY3MjhfQ2xvdWRUcmFpbF91cy13ZXN0LTJfMjAyMzAxMTBUMjEyNVpfZlA1UXlWRVV3eXIxUW9iUS5qc29uLmd6XG4ifXsgInRpbWUiOiAxNjczMzg1ODY0MDYwLCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiRU5EIFJlcXVlc3RJZDogZTczMGU0NzQtYWQ4ZS01M2U5LThjZmItYjg0Mzc0MjQ2ZDVlXG4ifXsgInRpbWUiOiAxNjczMzg1ODY0MDYwLCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUkVQT1JUIFJlcXVlc3RJZDogZTczMGU0NzQtYWQ4ZS01M2U5LThjZmItYjg0Mzc0MjQ2ZDVlXHREdXJhdGlvbjogMTYzLjU5IG1zXHRCaWxsZWQgRHVyYXRpb246IDE2NCBtc1x0TWVtb3J5IFNpemU6IDEwMjQgTUJcdE1heCBNZW1vcnkgVXNlZDogODMgTUJcdFxuIn17ICJ0aW1lIjogMTY3MzM4NTg2Njc5MiwgImhvc3QiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZSI6ICI4NDExNTQyMjY3MjgiLCAic291cmNldHlwZSI6ICJhd3M6Y2xvdWR3YXRjaGxvZ3MiLCAiaW5kZXgiOiAiYXdzIiwgImV2ZW50IjogIlNUQVJUIFJlcXVlc3RJZDogYTcyYTM1ZDItYWNkZS01YzNkLWI3OWItNjllNTc1NzJhMjc0IFZlcnNpb246ICRMQVRFU1RcbiJ9eyAidGltZSI6IDE2NzMzODU4NjY4NDMsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJQcm9jZXNzZWQgZmlsZSBzMzovLzg0MTE1NDIyNjcyOC11cy13ZXN0LTItY2xvdWR0cmFpbC9BV1NMb2dzLzg0MTE1NDIyNjcyOC9DbG91ZFRyYWlsL3VzLXdlc3QtMi8yMDIzLzAxLzEwLzg0MTE1NDIyNjcyOF9DbG91ZFRyYWlsX3VzLXdlc3QtMl8yMDIzMDExMFQyMTI1Wl9OdGdYb25QajFHSG50N0w2Lmpzb24uZ3pcbiJ9eyAidGltZSI6IDE2NzMzODU4NjY4NDQsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJFTkQgUmVxdWVzdElkOiBhNzJhMzVkMi1hY2RlLTVjM2QtYjc5Yi02OWU1NzU3MmEyNzRcbiJ9eyAidGltZSI6IDE2NzMzODU4NjY4NDQsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJSRVBPUlQgUmVxdWVzdElkOiBhNzJhMzVkMi1hY2RlLTVjM2QtYjc5Yi02OWU1NzU3MmEyNzRcdER1cmF0aW9uOiA1MS42MiBtc1x0QmlsbGVkIER1cmF0aW9uOiA1MiBtc1x0TWVtb3J5IFNpemU6IDEwMjQgTUJcdE1heCBNZW1vcnkgVXNlZDogODMgTUJcdFxuIn0='}]} 23 | 24 | self.assertEqual(self.lambda_module.handler(testInput, "none"), expectedOutput) 25 | 26 | 27 | def test_two(self): 28 | 29 | testInput = {'invocationId': '16808bd2-e8f8-47d4-aaa3-b2cc01c644f5', 'deliveryStreamArn': 'arn:aws:firehose:us-west-2:841154226728:deliverystream/841154226728-us-west-2-lambdacloudtrail-firehose', 'region': 'us-west-2', 'records': [{'recordId': '49636824083378936888143884676531704283065306992770809858000000', 'approximateArrivalTimestamp': 1673386013835, 'data': 'H4sIAAAAAAAAAK1SXU/bMBT9K1G0R0Lsazu2+xZoYNNgsCaANECVkzhdpjTp8kELiP++2wKjaEIb0p4s33t87vE5996d264zM5vcLqw7csdhEk6PozgODyN3x22WtW2xrDilggMEEhSWq2Z22DbDAju+WXZ+ZeZpbvxtlDd03tJ2vQdeVjVD3remrLxHoFcMddaXTf1IFfetNXPkAgLMJ9SnxL/8cBQmUZxcc12kAdAgZVpyFqSaM2NpQUBJQ6iWSNENaZe15WLNeFBWvW07d3TpPs7aXw9P1sP3L46SZr8t08rLllW89cjr4eLr+clqNr5ZHbjXG1HRja37Nc+9W+aojUlGpRRcU8KYlBpYAFoTBopxzrUGIZWUilCGDa4kB0GJ0ID6+hI97s0c7aKBZEwFhHA8d569R/o4CSeJM7E/B4R+ykdOQCXJmGGeEqrwRJ5LLy0E83ShuSiywmpBnHP8KuofOU9uXdXuw84fgplSLKBEcS1EEKCNwAlhnEmiGBWASjEvqhnTqE6+JVhp2BZ82jYZXmzuFGVlnY6N/L/n74cX8VEz614h/ZeI/OdH1N/ahW3w9AU8/Q2ersGEUpIABfFtehZ9Dg35+L2GoTKLSbv7o2vq3dndG/ZozAknaC6FAFCYDcVEcRih6A2RnCgdaI2po4y37XmVZ/Rl/N40/4M68Y/qJtHpyfvX7aofD63pNwtHA7IL4My7q36vrCpcg+0e3TSO7bxpb524vLNYJMCd4z2smpXz1DnD9Rk5im3q6+9fP/wCFaeti5AEAAA='}, {'recordId': '49636824083378936888143884689912095254560024072808300546000000', 'approximateArrivalTimestamp': 1673386034806, 'data': 'H4sIAAAAAAAAANVVW0/bWBD+K5a1jxifmTlzLnkLEKqVoFCSEtGCIjs5ibzrxKztFNqq/33H0G3TS2i6qlbaR8/tfPPNzOf38TI0TbYIo7e3Ie7FR/1Rf3I6GA77zwbxXlzdrUItZqcBWCMai07MZbV4VlfrW/Gk2V2Tltkyn2XpZlSybpK70LQJJtOyWs/aOivK5DEwma9X07aoVo+lhm0dsqXUQoWUKkhBpa9/O+mPBsPRjfbz3CCYnLzVZHKvKQswV+hspsBbKdGs82ZaF7ddxeOibEPdxL3X8eNbh93jo+7xw/HJqDqsi7xMpnflcCMpaXH84vLsfnH05v44vnkANXgTVm1X531czAQbWQJrmck6pYwF7xHIGoUGLShFqLw3ZJUC7ZwxTnmUOK0EX1sIx222FLrAWCInWWyV3vuHeyk/HPUvRtFF+Gstob/PehHoLJ/OYZroGc0TnqFNcmBKPGHurc1Zh3l0Ka0K/l70ka3rVfxh72vADrXgMc4bUA6BlfOeQBGTAYfoxQaCVzE6kgluA+y/BHxeV1P5CLNoXpQhaqiX/nj+aX88PKkWzReR6ecRpZ+S0o1d2AyefA6efAqedMEKQI1kJurVhFfP3YuXHK4Gh1djBcv9P5pqtb94t40e58F4qww6ZRwwomZwzhMZVkKeZ/QMgEYju+308CY9g+dHPzvNX4DO74juYnB+9vPrdt0ereusfVg4VLBPPlo21+1BUZayBps+fHCchmVVv42GxbsgDwjW6PRArNl99NHzUtanFzl6sH+vfWZLWpZVWreePXrLlpFIsxGj9IZW7s56JYdnZIu/3z6R1ebJY5tmncBkJmEHOmFGnfi5dkmYezkRtpkh3unY2IAmJZjBClqlFcjRAWnt0bJBQo+ILH4H2A1yG2BJ/r8c28m4afDitLw/wUF9WQyfOjahwEjjIpBWCwPCsyEj3DrHnUo5QscaiDqR0uj1Vnoc/if0hEySYEd6HoM36UF+Ncmu/jw4H684DI7PDvKrxY/okd12Xsu/Q0rIIrIcn9XYvemZ5ejlSwRb/iwWtkh1Rw89oUW7LPsvQIc7ovtWi3YCuKlFZPfVNi0i9y+16ObD34ZIaDgcCQAA'}]} 30 | 31 | expectedOutput = {'records': [{'recordId': '49636824083378936888143884676531704283065306992770809858000000', 'result': 'Ok', 'data': 'eyAidGltZSI6IDE2NzMzODYwMDQ3MzMsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJTVEFSVCBSZXF1ZXN0SWQ6IDYxNzBjM2EzLTg1OGYtNWRkNy1iZjUzLTlmOTQ1ZmNmZTk1MCBWZXJzaW9uOiAkTEFURVNUXG4ifXsgInRpbWUiOiAxNjczMzg2MDA0ODkyLCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUHJvY2Vzc2VkIGZpbGUgczM6Ly84NDExNTQyMjY3MjgtdXMtd2VzdC0yLWNsb3VkdHJhaWwvQVdTTG9ncy84NDExNTQyMjY3MjgvQ2xvdWRUcmFpbC91cy13ZXN0LTEvMjAyMy8wMS8xMC84NDExNTQyMjY3MjhfQ2xvdWRUcmFpbF91cy13ZXN0LTFfMjAyMzAxMTBUMjEyNVpfVUVLQWEwSGhuMnVsYXBSci5qc29uLmd6XG4ifXsgInRpbWUiOiAxNjczMzg2MDA0ODkzLCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiRU5EIFJlcXVlc3RJZDogNjE3MGMzYTMtODU4Zi01ZGQ3LWJmNTMtOWY5NDVmY2ZlOTUwXG4ifXsgInRpbWUiOiAxNjczMzg2MDA0ODkzLCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUkVQT1JUIFJlcXVlc3RJZDogNjE3MGMzYTMtODU4Zi01ZGQ3LWJmNTMtOWY5NDVmY2ZlOTUwXHREdXJhdGlvbjogMTYwLjIyIG1zXHRCaWxsZWQgRHVyYXRpb246IDE2MSBtc1x0TWVtb3J5IFNpemU6IDEwMjQgTUJcdE1heCBNZW1vcnkgVXNlZDogODMgTUJcdFxuIn0='}, {'recordId': '49636824083378936888143884689912095254560024072808300546000000', 'result': 'Ok', 'data': 'eyAidGltZSI6IDE2NzMzODYwMjU3MDQsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJTVEFSVCBSZXF1ZXN0SWQ6IDE0YWJjZjFjLTRkM2YtNWQyNy1iMTUzLTkzMmI5NzdiNTRlZiBWZXJzaW9uOiAkTEFURVNUXG4ifXsgInRpbWUiOiAxNjczMzg2MDI1OTA0LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUHJvY2Vzc2VkIGZpbGUgczM6Ly84NDExNTQyMjY3MjgtdXMtd2VzdC0yLWNsb3VkdHJhaWwvQVdTTG9ncy84NDExNTQyMjY3MjgvQ2xvdWRUcmFpbC91cy13ZXN0LTIvMjAyMy8wMS8xMC84NDExNTQyMjY3MjhfQ2xvdWRUcmFpbF91cy13ZXN0LTJfMjAyMzAxMTBUMjEzMFpfNW5OOFFVNWVZRUNZVzAxbS5qc29uLmd6XG4ifXsgInRpbWUiOiAxNjczMzg2MDI1OTA1LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiRU5EIFJlcXVlc3RJZDogMTRhYmNmMWMtNGQzZi01ZDI3LWIxNTMtOTMyYjk3N2I1NGVmXG4ifXsgInRpbWUiOiAxNjczMzg2MDI1OTA1LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUkVQT1JUIFJlcXVlc3RJZDogMTRhYmNmMWMtNGQzZi01ZDI3LWIxNTMtOTMyYjk3N2I1NGVmXHREdXJhdGlvbjogMjAxLjM5IG1zXHRCaWxsZWQgRHVyYXRpb246IDIwMiBtc1x0TWVtb3J5IFNpemU6IDEwMjQgTUJcdE1heCBNZW1vcnkgVXNlZDogODMgTUJcdFxuIn17ICJ0aW1lIjogMTY3MzM4NjAzMzc0NiwgImhvc3QiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZSI6ICI4NDExNTQyMjY3MjgiLCAic291cmNldHlwZSI6ICJhd3M6Y2xvdWR3YXRjaGxvZ3MiLCAiaW5kZXgiOiAiYXdzIiwgImV2ZW50IjogIlNUQVJUIFJlcXVlc3RJZDogY2ExNmIzYTYtNTgxNC01NTI0LTlmNDgtZWY5MTgyNTdhNjM1IFZlcnNpb246ICRMQVRFU1RcbiJ9eyAidGltZSI6IDE2NzMzODYwMzM5MjksICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJQcm9jZXNzZWQgZmlsZSBzMzovLzg0MTE1NDIyNjcyOC11cy13ZXN0LTItY2xvdWR0cmFpbC9BV1NMb2dzLzg0MTE1NDIyNjcyOC9DbG91ZFRyYWlsL3VzLXdlc3QtMi8yMDIzLzAxLzEwLzg0MTE1NDIyNjcyOF9DbG91ZFRyYWlsX3VzLXdlc3QtMl8yMDIzMDExMFQyMTMwWl9MV3NzMlJNbHhMMkVyVmlTLmpzb24uZ3pcbiJ9eyAidGltZSI6IDE2NzMzODYwMzM5ODIsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJQcm9jZXNzZWQgZmlsZSBzMzovLzg0MTE1NDIyNjcyOC11cy13ZXN0LTItY2xvdWR0cmFpbC9BV1NMb2dzLzg0MTE1NDIyNjcyOC9DbG91ZFRyYWlsL3VzLWVhc3QtMS8yMDIzLzAxLzEwLzg0MTE1NDIyNjcyOF9DbG91ZFRyYWlsX3VzLWVhc3QtMV8yMDIzMDExMFQyMTI1Wl9hWWtCUFduNWVFRk9CYllnLmpzb24uZ3pcbiJ9eyAidGltZSI6IDE2NzMzODYwMzM5ODMsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJFTkQgUmVxdWVzdElkOiBjYTE2YjNhNi01ODE0LTU1MjQtOWY0OC1lZjkxODI1N2E2MzVcbiJ9eyAidGltZSI6IDE2NzMzODYwMzM5ODMsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJSRVBPUlQgUmVxdWVzdElkOiBjYTE2YjNhNi01ODE0LTU1MjQtOWY0OC1lZjkxODI1N2E2MzVcdER1cmF0aW9uOiAyMzcuMDkgbXNcdEJpbGxlZCBEdXJhdGlvbjogMjM4IG1zXHRNZW1vcnkgU2l6ZTogMTAyNCBNQlx0TWF4IE1lbW9yeSBVc2VkOiA4MyBNQlx0XG4ifQ=='}]} 32 | 33 | self.assertEqual(self.lambda_module.handler(testInput, "none"), expectedOutput) 34 | 35 | 36 | def test_three(self): 37 | 38 | testInput = {'invocationId': 'b2d8ec22-3a77-48bb-94a1-d679d7b5e6f3', 'deliveryStreamArn': 'arn:aws:firehose:us-west-2:841154226728:deliverystream/841154226728-us-west-2-lambdacloudtrail-firehose', 'region': 'us-west-2', 'records': [{'recordId': '49636824083378936888143884739585648256705528554502225922000000', 'approximateArrivalTimestamp': 1673386126384, 'data': 'H4sIAAAAAAAAAK1S207bQBD9FcvqI8Y7e9+8BRJQJe52QeKiaO2sgyvHpr6QAOLfOw4UXFWoRarsl5k5e+bMmXnyl65p7MLFD3fOH/mTcTyeHU6jaLw/9bf8alW6GtOaAwhOqVRUY7qoFvt11d1hJbSrJizsMpnbcIgKuiZYuaYNaJAWVTdva5sXwQswyLoybfOqfKGK2trZJXJRQllIIAQSXn05GMfTKL7hJkskBZkwoziTieHMOsgI1coSMAopmi5p0jq/6xn38qJ1deOPrvyXXrt987hvvntxEFe7dZ4UQboqosGjoKUXp+fH68Xkfr3n32xETe9d2fY8T34+R21MMVBKKE4JKAGSYCzBMGIM59IIKpimErgykmkFVAPvP4762hw9bu0S7QKpGNMSQFGlt355j/RRPD6LvTP3o0Po1/nIk9xYkAKCNJuTQNAsCRJmWcCtlCCUcFKl3jmOivpH3qtb16X/vPWHYE5xLUZIoEIwJihKYMAoaM0YNz2MERSFP44CHwnmwgwFn9RVioGbe1leOK9ho/Dv+w/HF9FBtWh+Q4bvKwrfHoWDWxiCZ+/g2Rt41oMJAIkpMHI5W9VVeUtYs9DdyWV2dLv9vanK7cXjR/bghLg1geHGEyG54EpwIUBJY6SQUjBpNDda6A/tkWRoz/Ro8tlt/gd15h/VnU1Pjj9/btftpKttuzk40LAtpLdsrtudvCjwDIY1uikcumVVP3hR/ugwSSj3Dncwa9fea+Ubns/I02yT78e/ef4J1YJoi5AEAAA='}, {'recordId': '49636824083378936888143884758832956230790041569144602626000000', 'approximateArrivalTimestamp': 1673386154334, 'data': 'H4sIAAAAAAAAANVUXVPTUBD9K5mMj4Ts3r2ffStQHR0QJNH6AdO5TZMaJ20gSQFl+O9uwY+qFKvjiw+ZSXZP9p49u+deh7O8bf00Tz+e5WEv3Oun/dHBIEn6TwbhVlhfzvOGw1YiKimENsJyuKqnT5p6ccaZ2F+2ceVn44mPV1HRoo0u87aLRJRV9WLSNb6sojtgVCzmWVfW87tSSdfkfsa1BAiKAWOE+N2j/X46SNJT6YqxFqjH5IwkPXaSfI4FCGs8oDNcol2M26wpz5YVH5dVlzdt2HsX3p21uzw8XR6+O9xP692mHFdRdlklKz9FnRi+eHV4Nd27uHocnt6SGlzk825Z5zosJ8yNDKExyoKUYAnBoeIHlCINTmh0KDSgRTAgNQM0vwvnBPPrSta48zOWC7UhshqlEkptfdWeyydp/zgNjvPzBUOfTnqB95JUDpPIeS0iZZ2NfKGyiCXyErwxGRbBK26V+feCL2qdzMObrZ8JK0EkyUq71E9bHpJBTYDCkhbagXKWeyIjldYOaR1hLX4gfNTUGX/kk6AoqzxoqRf/fv5xf5js19P2B2T8fUTxt5/ilV1YBY++g0ffwKMlGBAhFUjwdjQcvE61HNpiPnzWnSfPtz+09Xx7+mmdPIbV1aSl0eSkYn1YBnTO8PiIlVcsF2ughDFSr5fHrMozeL73p9P8B+zMhuyOB0eHf75uJ93eovHd7cKREdtIwaw96XbKquI1WM3dJQ7yWd18DJLyU94LEIQMDnY46q+CL5mXvD69wNJt/L72BSFyw7yhhv3EO8B6sL1IIfcqjOXewTlFbCYpJMj722eLSnrQbBoybaS3Eds3i9SycU/aRsK6IvPjXHlpNzIbW81K4KsB2OFg2DOOOVtH5MgSIzQKQ0ZJh8xf4jrCRsv/wWxCvR1Nizm8vzgskx3egjc75w+ZbSmPFlKBcSCBG9VKcg6sUbwfIC1fpDxQvlVZA1R2vTzqAbNtMs1/wM5tyO5Xs21EcMVsKMQ2wf1mQ/G3Zju9+QwtDoYU/QcAAA=='}]} 39 | 40 | expectedOutput = {'records': [{'recordId': '49636824083378936888143884739585648256705528554502225922000000', 'result': 'Ok', 'data': 'eyAidGltZSI6IDE2NzMzODYxMTcyNzgsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJTVEFSVCBSZXF1ZXN0SWQ6IDY0OWExNjUxLWNmZDAtNTJmYi1iM2EzLTRhNjYxNTc1ZTY3YyBWZXJzaW9uOiAkTEFURVNUXG4ifXsgInRpbWUiOiAxNjczMzg2MTE3NDU5LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUHJvY2Vzc2VkIGZpbGUgczM6Ly84NDExNTQyMjY3MjgtdXMtd2VzdC0yLWNsb3VkdHJhaWwvQVdTTG9ncy84NDExNTQyMjY3MjgvQ2xvdWRUcmFpbC91cy13ZXN0LTIvMjAyMy8wMS8xMC84NDExNTQyMjY3MjhfQ2xvdWRUcmFpbF91cy13ZXN0LTJfMjAyMzAxMTBUMjEzMFpfd3JvbmgwM3NnOHVQWmZOaC5qc29uLmd6XG4ifXsgInRpbWUiOiAxNjczMzg2MTE3NDYwLCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiRU5EIFJlcXVlc3RJZDogNjQ5YTE2NTEtY2ZkMC01MmZiLWIzYTMtNGE2NjE1NzVlNjdjXG4ifXsgInRpbWUiOiAxNjczMzg2MTE3NDYwLCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUkVQT1JUIFJlcXVlc3RJZDogNjQ5YTE2NTEtY2ZkMC01MmZiLWIzYTMtNGE2NjE1NzVlNjdjXHREdXJhdGlvbjogMTgxLjU2IG1zXHRCaWxsZWQgRHVyYXRpb246IDE4MiBtc1x0TWVtb3J5IFNpemU6IDEwMjQgTUJcdE1heCBNZW1vcnkgVXNlZDogODMgTUJcdFxuIn0='}, {'recordId': '49636824083378936888143884758832956230790041569144602626000000', 'result': 'Ok', 'data': 'eyAidGltZSI6IDE2NzMzODYxNDUyNTUsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJTVEFSVCBSZXF1ZXN0SWQ6IGFhNDM1ZTBkLTlhNjItNTg5OC1hZjVjLTAyM2E0MGE3N2MxZiBWZXJzaW9uOiAkTEFURVNUXG4ifXsgInRpbWUiOiAxNjczMzg2MTQ1NjI1LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUHJvY2Vzc2VkIGZpbGUgczM6Ly84NDExNTQyMjY3MjgtdXMtd2VzdC0yLWNsb3VkdHJhaWwvQVdTTG9ncy84NDExNTQyMjY3MjgvQ2xvdWRUcmFpbC91cy13ZXN0LTIvMjAyMy8wMS8xMC84NDExNTQyMjY3MjhfQ2xvdWRUcmFpbF91cy13ZXN0LTJfMjAyMzAxMTBUMjEzMFpfV0VYVDY0VzhmbldKdHFTTi5qc29uLmd6XG4ifXsgInRpbWUiOiAxNjczMzg2MTQ1NjI3LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiRU5EIFJlcXVlc3RJZDogYWE0MzVlMGQtOWE2Mi01ODk4LWFmNWMtMDIzYTQwYTc3YzFmXG4ifXsgInRpbWUiOiAxNjczMzg2MTQ1NjI3LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUkVQT1JUIFJlcXVlc3RJZDogYWE0MzVlMGQtOWE2Mi01ODk4LWFmNWMtMDIzYTQwYTc3YzFmXHREdXJhdGlvbjogMzcyLjEzIG1zXHRCaWxsZWQgRHVyYXRpb246IDM3MyBtc1x0TWVtb3J5IFNpemU6IDEwMjQgTUJcdE1heCBNZW1vcnkgVXNlZDogODMgTUJcdFxuIn17ICJ0aW1lIjogMTY3MzM4NjE1MzY0MywgImhvc3QiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZSI6ICI4NDExNTQyMjY3MjgiLCAic291cmNldHlwZSI6ICJhd3M6Y2xvdWR3YXRjaGxvZ3MiLCAiaW5kZXgiOiAiYXdzIiwgImV2ZW50IjogIlNUQVJUIFJlcXVlc3RJZDogNjBjNjc0YTgtMTA3Yy01ZjVjLWEzNjgtMjg5ZmNhYmU1YTQ4IFZlcnNpb246ICRMQVRFU1RcbiJ9eyAidGltZSI6IDE2NzMzODYxNTM3NjQsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJQcm9jZXNzZWQgZmlsZSBzMzovLzg0MTE1NDIyNjcyOC11cy13ZXN0LTItY2xvdWR0cmFpbC9BV1NMb2dzLzg0MTE1NDIyNjcyOC9DbG91ZFRyYWlsL3VzLXdlc3QtMi8yMDIzLzAxLzEwLzg0MTE1NDIyNjcyOF9DbG91ZFRyYWlsX3VzLXdlc3QtMl8yMDIzMDExMFQyMTI1Wl9nZm4waHZPaVNCbGVkWUJxLmpzb24uZ3pcbiJ9eyAidGltZSI6IDE2NzMzODYxNTM3NjUsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJFTkQgUmVxdWVzdElkOiA2MGM2NzRhOC0xMDdjLTVmNWMtYTM2OC0yODlmY2FiZTVhNDhcbiJ9eyAidGltZSI6IDE2NzMzODYxNTM3NjUsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJSRVBPUlQgUmVxdWVzdElkOiA2MGM2NzRhOC0xMDdjLTVmNWMtYTM2OC0yODlmY2FiZTVhNDhcdER1cmF0aW9uOiAxMjIuMzAgbXNcdEJpbGxlZCBEdXJhdGlvbjogMTIzIG1zXHRNZW1vcnkgU2l6ZTogMTAyNCBNQlx0TWF4IE1lbW9yeSBVc2VkOiA4MyBNQlx0XG4ifQ=='}]} 41 | 42 | self.assertEqual(self.lambda_module.handler(testInput, "none"), expectedOutput) 43 | 44 | 45 | def test_four(self): 46 | 47 | testInput = {'invocationId': 'b8029edf-6911-4fdc-926c-458e7d02e5d6', 'deliveryStreamArn': 'arn:aws:firehose:us-west-2:841154226728:deliverystream/841154226728-us-west-2-lambdacloudtrail-firehose', 'region': 'us-west-2', 'records': [{'recordId': '49636824083378936888143884785464383111080711026001313794000000', 'approximateArrivalTimestamp': 1673386203792, 'data': 'H4sIAAAAAAAAAK1S207bQBD9FcvqI8Y7e9+8hSSgVqFc7ILERdHa3kRunRh8IQHEv3ccKLiqUIvUB6/kM2dnzp4zj/7S1bVduPj+xvkDfzyMh7PDSRQNDyb+jl+uV65CWHMAwSmVimqEi3JxUJXtDVZCu67Dwi6TzIZ9VtDWwdrVTUCDtCjbrKlsXgTPxGDertImL1fPraKmcnaJvSihLCQQAgkvP02H8SSKr7mZJ5KCTJhRnMnEcGYdzAnVyhIwClvUbVKnVX7TddzPi8ZVtT+49J9njbrhcTd8dD6Ny1GVJ0WQrouodylo6PnJ2dFmMb7b7PvXW1GTO7dquj6Pfp6hNqYYKCUMcKCAw4nUjKAuSpikWgqlNDX4Eck1V1wqxpHKOeprcvS4sUu0CxBnWoLhXNCdX95j+ygensbeqbttkfo5G3hzlokUjAkkkWkgMqBBAjYJTGZdYg1YTaR3hk9F/QPvxa2rlf+084dgZoTQkjJOOScapRNDO4UguWBCMiMNM4YYQgQD9p5goURf8HFVpvjjMm+eF86r2SD8e/7h8Dyalov6N2b4FlH4eins7UKfPHsjz17Js45MAEiM0YiL2Yk7GsGPDdXiSzEVF7e73+tytbt4eM8epUEyohVawQFPyrtcFZMYrSFoEumiNFooRt63R/btmXwdfzTN/6AO/lHd6eT46OPrdtWM28o224UDyna18pb1VbOXFwWuQb/Gt4VDtyyrey/KHxyCBNHDPUTtxnupfMP1GXiabfHu+ddPPwH/bPr+kAQAAA=='}]} 48 | 49 | expectedOutput = {'records': [{'recordId': '49636824083378936888143884785464383111080711026001313794000000', 'result': 'Ok', 'data': 'eyAidGltZSI6IDE2NzMzODYxOTQ0NTIsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJTVEFSVCBSZXF1ZXN0SWQ6IGYzZDVjMTk5LTYwNmMtNWQxMi1iMWFiLTlkYWViYTkxYTgwNiBWZXJzaW9uOiAkTEFURVNUXG4ifXsgInRpbWUiOiAxNjczMzg2MTk0NTc1LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUHJvY2Vzc2VkIGZpbGUgczM6Ly84NDExNTQyMjY3MjgtdXMtd2VzdC0yLWNsb3VkdHJhaWwvQVdTTG9ncy84NDExNTQyMjY3MjgvQ2xvdWRUcmFpbC91cy13ZXN0LTIvMjAyMy8wMS8xMC84NDExNTQyMjY3MjhfQ2xvdWRUcmFpbF91cy13ZXN0LTJfMjAyMzAxMTBUMjEyNVpfUWVPQzFreDI4NUpsTDVacS5qc29uLmd6XG4ifXsgInRpbWUiOiAxNjczMzg2MTk0NTc2LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiRU5EIFJlcXVlc3RJZDogZjNkNWMxOTktNjA2Yy01ZDEyLWIxYWItOWRhZWJhOTFhODA2XG4ifXsgInRpbWUiOiAxNjczMzg2MTk0NTc2LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUkVQT1JUIFJlcXVlc3RJZDogZjNkNWMxOTktNjA2Yy01ZDEyLWIxYWItOWRhZWJhOTFhODA2XHREdXJhdGlvbjogMTIzLjg3IG1zXHRCaWxsZWQgRHVyYXRpb246IDEyNCBtc1x0TWVtb3J5IFNpemU6IDEwMjQgTUJcdE1heCBNZW1vcnkgVXNlZDogODMgTUJcdFxuIn0='}]} 50 | 51 | self.assertEqual(self.lambda_module.handler(testInput, "none"), expectedOutput) 52 | 53 | 54 | def test_five(self): 55 | 56 | testInput = {'invocationId': '09cb53c3-0beb-40f2-8bdf-fa418277dd7b', 'deliveryStreamArn': 'arn:aws:firehose:us-west-2:841154226728:deliverystream/841154226728-us-west-2-lambdacloudtrail-firehose', 'region': 'us-west-2', 'records': [{'recordId': '49636824083378936888143884850852762842396780836895588354000000', 'approximateArrivalTimestamp': 1673386305274, 'data': 'H4sIAAAAAAAAAK1S2U7cQBD8FcvKI8bTM+M59s3sGhSJ2wYkDq3G9njlxLsmPlgO8e9pLwScSChByuNU1XRXV/eTu7RtaxY2ebi17sSdhUk4P4jiONyL3C23Xq9sg7DiAAGnVEiqEK7qxV5T97fI+Gbd+pVZprnxxyqvb721bTuPellV93nXmLLyXoRe0a+yrqxXL6XirrFmibUoocwn4APxr77sh0kUJzdcF6mgIFKmJWci1ZwZCwWhShoCWmKJtk/brClvh4q7ZdXZpnUnV+5Lr+nQPBmaTy/2k3ralGnlZesqHn3yOnpxcn50v5jd3e+6NxtT0Z1ddUOdJ7fM0RuTDKQUwIkOgCghFRFMDNNKwgkhFIAITSSIgGohmaQo5GpIqysx484sMS5AhimBChBi61f2WD5OwtPEObU/epR+zSdOkWaGM6U8LnXuBZnKPUSMB0Va6CyTihnunOOo6H/ivKZ1vXKft/40DJQHPBACJA0kcMF1oISimgZKoWegTAiFfgDn4gQ+Mky1Ghs+buoMHzZ3irKyTssm/t/374cX8X69aH9T+u8r8t8++aNbGIvn7+L5m3g+iAkuIKHAyOX8u7pMQlYfptH0bJaFJ9vf2nq1vXj8KB6J8ws9PLXWQaBAUc4xD60ZDq8CggohuSbIfByPHscTHc4+u83/4E7/o7vT6Pjo8+d23c36xnSbgwNGtxVxlu11t1NWFZ7BmGMb4sAu6+bBictHiyCh3DnYQdTcO6/MGZ7PxFFsgw/j3zz/BOGKdTaQBAAA'}, {'recordId': '49636824083378936888143884870253604395572352512639893506000000', 'approximateArrivalTimestamp': 1673386344938, 'data': 'H4sIAAAAAAAAANVT207bQBD9FcvqI8Y7O3vNWyAB0QYC2JC2gCI72aSunJjaDuEi/r3jQCGopS1SX7pvO3N25syZs3f+zFVVMnXxzaXzW36nHbeH+90oau92/Q2/WM5dSWEjAKTgXGluKJwX092yWFxSJkyWVZgns3SchOuoYFEFS1fVAQ9GebEY12WS5cEDMJgs5qM6K+YPpaK6dMmManHGMWQQAgvP3vXacTeKL4SdpIqDStFqgSq1AhMHE8aNThhYTSWqRVqNyuyyqbiT5bUrK7915j/02m6ax03z7UEvLrbLLM2D0TKP1h4FNR8cnfavp52r6x3/YkWqe+XmdVPnzs/GxA01gtaKcysMNyBBCG6QK2RGaMMBlTUo0TLFBFiLVjLGkDPiV2ekcZ3MSC5QGtEoRGmU3PihPZWP4vZx7B27bwuC7o1bHmOJlBJkoJQZBRIxCVKrRgGznKdjppzizjulUYl/y3tU63zu32/8RNgKOkwSQcuNaHgaAEvUJRBzQ8MwBECaTAnEVwgrpvk64cOyGNHFjb1Jljuvwlb45/2H7UHUK6bVC2T4vKLw6VG45oV18PAZPHwCDxswA2Ax7UF+Hrp535aDD5+EPSl6ycHu5teqmG9Ob38lDzJGgoDUAHTTjARBYYzk3DBtlABagtWSCSmtBDSvyQMC/hd5dva+3Hayk8v3/Y8fDtK9o9/LA4xJIUkKbrhUgEIaRr0UaqssCE4uR9U4VZNf5evyvHBP96DzVrP/A3bqL9kddw/7b/+N53VnUSb16j9yrTat8GbVeb2V5TnZYD2nV4l9NyvKGy/Kbl3LA8aFt79F0eTae8yckH1ansFVvBn/4v47YJMok68FAAA='}]} 57 | 58 | expectedOutput = {'records': [{'recordId': '49636824083378936888143884850852762842396780836895588354000000', 'result': 'Ok', 'data': 'eyAidGltZSI6IDE2NzMzODYyOTYxNjYsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJTVEFSVCBSZXF1ZXN0SWQ6IGZiY2E0Mzg4LTQ3OWQtNWM4ZC1iY2FhLTFmYmY5Y2M3ODNhNCBWZXJzaW9uOiAkTEFURVNUXG4ifXsgInRpbWUiOiAxNjczMzg2Mjk2Mjk4LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUHJvY2Vzc2VkIGZpbGUgczM6Ly84NDExNTQyMjY3MjgtdXMtd2VzdC0yLWNsb3VkdHJhaWwvQVdTTG9ncy84NDExNTQyMjY3MjgvQ2xvdWRUcmFpbC91cy13ZXN0LTIvMjAyMy8wMS8xMC84NDExNTQyMjY3MjhfQ2xvdWRUcmFpbF91cy13ZXN0LTJfMjAyMzAxMTBUMjEzMFpfazhaVEEzb05iRUNVRGNBUS5qc29uLmd6XG4ifXsgInRpbWUiOiAxNjczMzg2Mjk2Mjk5LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiRU5EIFJlcXVlc3RJZDogZmJjYTQzODgtNDc5ZC01YzhkLWJjYWEtMWZiZjljYzc4M2E0XG4ifXsgInRpbWUiOiAxNjczMzg2Mjk2Mjk5LCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUkVQT1JUIFJlcXVlc3RJZDogZmJjYTQzODgtNDc5ZC01YzhkLWJjYWEtMWZiZjljYzc4M2E0XHREdXJhdGlvbjogMTMyLjgwIG1zXHRCaWxsZWQgRHVyYXRpb246IDEzMyBtc1x0TWVtb3J5IFNpemU6IDEwMjQgTUJcdE1heCBNZW1vcnkgVXNlZDogODMgTUJcdFxuIn0='}, {'recordId': '49636824083378936888143884870253604395572352512639893506000000', 'result': 'Ok', 'data': 'eyAidGltZSI6IDE2NzMzODYzMzU4NjUsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICJTVEFSVCBSZXF1ZXN0SWQ6IDAwYTU1NTE1LTY2OGMtNTMzYS1iOTZjLTA5MjJiZDA2ZTYyZSBWZXJzaW9uOiAkTEFURVNUXG4ifXsgInRpbWUiOiAxNjczMzg2MzM2MDcyLCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUHJvY2Vzc2VkIGZpbGUgczM6Ly84NDExNTQyMjY3MjgtdXMtd2VzdC0yLWNsb3VkdHJhaWwvQVdTTG9ncy84NDExNTQyMjY3MjgvQ2xvdWRUcmFpbC91cy13ZXN0LTIvMjAyMy8wMS8xMC84NDExNTQyMjY3MjhfQ2xvdWRUcmFpbF91cy13ZXN0LTJfMjAyMzAxMTBUMjEzNVpfZW5POXJXS1k0OVVvTGFORy5qc29uLmd6XG4ifXsgInRpbWUiOiAxNjczMzg2MzM2MTQxLCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUHJvY2Vzc2VkIGZpbGUgczM6Ly84NDExNTQyMjY3MjgtdXMtd2VzdC0yLWNsb3VkdHJhaWwvQVdTTG9ncy84NDExNTQyMjY3MjgvQ2xvdWRUcmFpbC91cy13ZXN0LTIvMjAyMy8wMS8xMC84NDExNTQyMjY3MjhfQ2xvdWRUcmFpbF91cy13ZXN0LTJfMjAyMzAxMTBUMjEzNVpfRkloekRpVXBKT1hLTmJJUS5qc29uLmd6XG4ifXsgInRpbWUiOiAxNjczMzg2MzM2MTQyLCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiRU5EIFJlcXVlc3RJZDogMDBhNTU1MTUtNjY4Yy01MzNhLWI5NmMtMDkyMmJkMDZlNjJlXG4ifXsgInRpbWUiOiAxNjczMzg2MzM2MTQyLCAiaG9zdCI6ICI4NDExNTQyMjY3MjgiLCAic291cmNlIjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2V0eXBlIjogImF3czpjbG91ZHdhdGNobG9ncyIsICJpbmRleCI6ICJhd3MiLCAiZXZlbnQiOiAiUkVQT1JUIFJlcXVlc3RJZDogMDBhNTU1MTUtNjY4Yy01MzNhLWI5NmMtMDkyMmJkMDZlNjJlXHREdXJhdGlvbjogMjc2Ljk0IG1zXHRCaWxsZWQgRHVyYXRpb246IDI3NyBtc1x0TWVtb3J5IFNpemU6IDEwMjQgTUJcdE1heCBNZW1vcnkgVXNlZDogODMgTUJcdFxuIn0='}]} 59 | 60 | self.assertEqual(self.lambda_module.handler(testInput, "none"), expectedOutput) 61 | 62 | 63 | def test_six(self): 64 | 65 | testInput = {'invocationId': 'becd65b6-252a-4e66-bbb1-1001c091abc3', 'deliveryStreamArn': 'arn:aws:firehose:us-west-2:841154226728:deliverystream/841154226728-us-west-2-lambdacloudtrail-firehose', 'region': 'us-west-2', 'records': [{'recordId': '49636824083378936888143884586489700312348099680449593346000000', 'approximateArrivalTimestamp': 1723672995, 'data': 'H4sIALkxvWYC/51VC2/iRhD+KytUya1U4ydgI2MJQRqhkrtTzLVSy6ly7D3Ynu2lu2seivLfO7trm0B6SVQSkt15fzOzM4+9EnOebvDqtMO9MerNp6vpX3c3STK9ven9jHr0UGEmGbbjev5gOApCOElOQTe3jNY7ybTSA7dywnAmKDtxzPYkw5bAXPT5rqirb/2Mlo1SIhhOS6nlhG7fGQZ9u2+bCc5qRsTpZo8rwaUorx94xshOEFr9QgqBGQelP3vzZEk3fEUTZbj3RVtt9EDgsUdyad0Ph94wcH078DyIGs5B4PheEPiDYOgH4ci2PYkpsMNwGNi+Hw5Cz/OH0rcgkBaRlhKcM3K94cgNwwEwmmxJ+5FyiY5lUfGJsRViN7Ysnm1xmfJ+STJGOf0qJHDrQCrLtW3fsgMLq0D1PyNes3WFUJScuMBlc4P7J0b3JMcMfUhLPDHuWmvm76TK6YF36TKndU4EqTYGuq1JPjEeW9imxG1K4KZEbj6H/mQg6+xM4VjMY3848iKrvXXs3yDxUILYjqz22PGWAKOQHH3o6KuUf4sd3x4MIkudO87HXUZzLFWaU8f5FZ8OlOU8to+hrT6O+oVPZHXMswso0Qw6SeAc6fRJysRwbYXddLyV647twdhx+xKzMvTHS+D3OAPDAFimSn6bFHT0TnxGGcNFKvvxwsoRaqGIULUM+mMxnxi+gVZbiC6XF8dxhheOZ9u0qiBfbRUjq6U8c1buauj5WD2iq5cE8i27U2htdX4i66KrNNp5KtKzjrw1LZbUD3/D6/0MbzchuREnpiN/nMh6j4okGCpUU/754W2tOS1TUl3pva0GT59WC4jPPvpp8Ir8RyWewCTCzIg/rNCSZmmBuixNa7GlOl/qDLkhma5tosfXa8FoCSNe8vQebwikmanImg748adXlD8xsicF3uAl6EGi8Sp76Giv6unuUuA95+FtUZ3d2Xi9bsbGeq1bwnPX64KnnPfx8dJl0/zPKfe4gkkEI2ZRfaVoVheiZuADV+bnxDi7v9OTMZ6iXQsmR80mQIeUI0h/gfO+VJDfpqBjdRddXRbz8VrAvWk/zZ1mGa1h1kpAig3Nohvtkq+bqpXQPaUlVHVa4zBgpmEXhw6xi0M2DEj9n445m5CYm2C/2yJtAA2hCaC56VChzl56SW+svlXUK3ToHv9TQzKQrCErVdSdw6ZaXCf+qh3bsl6N/Gd2/nv4J7jisJj2GHXGEEyK76wDae7lRnjHmGwXZfyOFRlZnfTLpdNSzrRY6aGkzmTiuxV01r1eSpF18VQUtXlPce/py9O/lC1vsGwJAAA='}]} 66 | 67 | expectedOutput = {'records': [{'recordId': '49636824083378936888143884586489700312348099680449593346000000', 'result': 'Ok', 'data': 'eyAidGltZSI6IDE3MjM2NzI5OTUsICJob3N0IjogIjg0MTE1NDIyNjcyOCIsICJzb3VyY2UiOiAiODQxMTU0MjI2NzI4IiwgInNvdXJjZXR5cGUiOiAiYXdzOmNsb3Vkd2F0Y2hsb2dzIiwgImluZGV4IjogImF3cyIsICJldmVudCI6ICI8RXZlbnQgeG1sbnM9J2h0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vd2luLzIwMDQvMDgvZXZlbnRzL2V2ZW50Jz5cclxuICA8U3lzdGVtPlxyXG4gICAgPFByb3ZpZGVyIE5hbWU9J01pY3Jvc29mdC1XaW5kb3dzLVNlY3VyaXR5LUF1ZGl0aW5nJyBHdWlkPSd7NDk2MzY4MjQtMDgzMy03ODkzLTY4ODgtMTQzODg0NTg2NDg5fScgLz5cclxuICAgIDxFdmVudElEPjQ2NzM8L0V2ZW50SUQ+XHJcbiAgICA8VmVyc2lvbj4wPC9WZXJzaW9uPlxyXG4gICAgPExldmVsPjA8L0xldmVsPlxyXG4gICAgPFRhc2s+MTQwNTU8L1Rhc2s+XHJcbiAgICA8T3Bjb2RlPjA8L09wY29kZT5cclxuICAgIDxLZXl3b3Jkcz4weDkwMDAwMDAxMDAxMDAwMDA8L0tleXdvcmRzPlxyXG4gICAgPFRpbWVDcmVhdGVkIFN5c3RlbVRpbWU9JzIwMjQtMDgtMTNUMjI6MDU6MTIuNjQ4OTAwMDAwWicgLz5cclxuICAgIDxFdmVudFJlY29yZElEPjc4OTM3ODkzPC9FdmVudFJlY29yZElEPlxyXG4gICAgPENvcnJlbGF0aW9uIC8+XHJcbiAgICA8RXhlY3V0aW9uIFByb2Nlc3NJRD0nNCcgVGhyZWFkSUQ9JzExMTYnIC8+XHJcbiAgICA8Q2hhbm5lbD5TZWN1cml0eTwvQ2hhbm5lbD5cclxuICAgIDxDb21wdXRlcj50ZXN0LnRlc3Quc3BsdW5rLmNvbTwvQ29tcHV0ZXI+XHJcbiAgICA8U2VjdXJpdHkgLz5cclxuICA8L1N5c3RlbT5cclxuICA8RXZlbnREYXRhPlxyXG4gICAgPERhdGEgTmFtZT0nU3ViamVjdFVzZXJTaWQnPlMtMS0xLTExPC9EYXRhPlxyXG4gICAgPERhdGEgTmFtZT0nU3ViamVjdFVzZXJOYW1lJz50ZXN0LXRlc3QkPC9EYXRhPlxyXG4gICAgPERhdGEgTmFtZT0nU3ViamVjdERvbWFpbk5hbWUnPnRlc3QtdGVzdDwvRGF0YT5cclxuICAgIDxEYXRhIE5hbWU9J1N1YmplY3RMb2dvbklkJz4weDRhODwvRGF0YT5cclxuICAgIDxEYXRhIE5hbWU9J09iamVjdFNlcnZlcic+TlQgTG9jYWwgU2VjdXJpdHkgQXV0aG9yaXR5IC8gQXV0aGVudGljYXRpb24gU2VydmljZTwvRGF0YT5cclxuICAgIDxEYXRhIE5hbWU9J1NlcnZpY2UnPkxzYVJlZ2lzdGVyTG9nb25Qcm9jZXNzKCk8L0RhdGE+XHJcbiAgICA8RGF0YSBOYW1lPSdQcml2aWxlZ2VMaXN0Jz5TZVRjYlByaXZpbGVnZTwvRGF0YT5cclxuICAgIDxEYXRhIE5hbWU9J1Byb2Nlc3NJZCc+MHgzMWI8L0RhdGE+XHJcbiAgICA8RGF0YSBOYW1lPSdQcm9jZXNzTmFtZSc+QzpcXFdpbmRvd3NcXFN5c3RlbTMyXFxsc2Fzcy5leGU8L0RhdGE+XHJcbiAgPC9FdmVudERhdGE+XHJcbiAgPFJlbmRlcmluZ0luZm8gQ3VsdHVyZT0nZW4tVVMnPlxyXG4gICAgPE1lc3NhZ2U+QSBwcml2aWxlZ2VkIHNlcnZpY2Ugd2FzIGNhbGxlZC5cclxuXHJcblN1YmplY3Q6XHJcblx0U2VjdXJpdHkgSUQ6XHRcdFMtMS0xLTExXHJcblx0QWNjb3VudCBOYW1lOlx0XHR0ZXMtdGVzdCRcclxuXHRBY2NvdW50IERvbWFpbjpcdFx0dGVzdC10ZXN0XHJcblx0TG9nb24gSUQ6XHRcdDB4OUE5XHJcblxyXG5TZXJ2aWNlOlxyXG5cdFNlcnZlcjpcdE5UIExvY2FsIFNlY3VyaXR5IEF1dGhvcml0eSAvIEF1dGhlbnRpY2F0aW9uIFNlcnZpY2VcclxuXHRTZXJ2aWNlIE5hbWU6XHRMc2FSZWdpc3RlckxvZ29uUHJvY2VzcygpXHJcblxyXG5Qcm9jZXNzOlxyXG5cdFByb2Nlc3MgSUQ6XHQweDMzYVxyXG5cdFByb2Nlc3MgTmFtZTpcdEM6XFxXaW5kb3dzXFxTeXN0ZW0zMlxcbHNhc3MuZXhlXHJcblxyXG5TZXJ2aWNlIFJlcXVlc3QgSW5mb3JtYXRpb246XHJcblx0UHJpdmlsZWdlczpcdFx0U2VUY2JQcml2aWxlZ2U8L01lc3NhZ2U+XHJcbiAgICA8TGV2ZWw+SW5mb3JtYXRpb248L0xldmVsPlxyXG4gICAgPFRhc2s+U2Vuc2l0aXZlIFByaXZpbGVnZSBVc2U8L1Rhc2s+XHJcbiAgICA8T3Bjb2RlPkluZm88L09wY29kZT5cclxuICAgIDxDaGFubmVsPlNlY3VyaXR5PC9DaGFubmVsPlxyXG4gICAgPFByb3ZpZGVyPk1pY3Jvc29mdC1XaW5kb3dzLVNlY3VyaXR5LUF1ZGl0aW5nPC9Qcm92aWRlcj5cclxuICAgIDxLZXl3b3Jkcz5cclxuICAgICAgPEtleXdvcmQ+QXVkaXQgU3VjY2VzczwvS2V5d29yZD5cclxuICAgIDwvS2V5d29yZHM+XHJcbiAgPC9SZW5kZXJpbmdJbmZvPlxyXG48L0V2ZW50PiJ9'}]} 68 | 69 | self.assertEqual(self.lambda_module.handler(testInput, "none"), expectedOutput) 70 | 71 | if __name__ == '__main__': 72 | unittest.main() -------------------------------------------------------------------------------- /CloudWatchMetrics-Firehose-Resources/cwMetricsToSplunk.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | AWSTemplateFormatVersion: "2010-09-09" 4 | Description: This is a CloudFormation template to configure events coming from CloudWatch Metrics to be sent to Splunk via Kinesis Data Firehose. 5 | 6 | Parameters: 7 | 8 | service: 9 | Type: String 10 | Description: Service name used in tagging AWS resources. 11 | Default: splunk-aws-gdi-toolkit 12 | 13 | stage: 14 | Type: String 15 | Description: Used to distinguish between stages of an environment (dev, test, prod, stage, etc). Only used in AWS resource tagging. 16 | Default: dev 17 | 18 | contact: 19 | Description: Used to identify a contact for the resources created in this stack. Only used in AWS resource tagging. As an example, this could be an email address or username. 20 | Type: String 21 | Default: "" 22 | 23 | splunkHECEndpoint: 24 | Type: String 25 | Description: Destination URL that Firehose will send data to. 26 | 27 | splunkHECToken: 28 | Type: String 29 | Description: HEC token Firehose will use to authenticate data being sent to Splunk. 30 | 31 | cloudWatchAlertEmail: 32 | Type: String 33 | Description: Email address for receiving alerts related to CloudTrail ingestion. Leave empty for no alerting. 34 | Default: "" 35 | 36 | splunkIndex: 37 | Type: String 38 | Description: Name of the index in Splunk events will be sent to. 39 | 40 | splunkEventType: 41 | Type: String 42 | Description: Type of event to be sent to Splunk. 43 | Default: event 44 | AllowedValues: 45 | - metric 46 | - event 47 | 48 | splunkSource: 49 | Type: String 50 | Description: Name of the source for the events that will be sent to Splunk. 51 | 52 | splunkHost: 53 | Type: String 54 | Description: Name of the host for the events that will be sent to Splunk. 55 | 56 | 57 | Conditions: 58 | enableAlerting: !Not 59 | - !Equals 60 | - !Ref cloudWatchAlertEmail 61 | - "" 62 | 63 | Resources: 64 | # Firehose > Splunk resources 65 | firehose: 66 | Type: AWS::KinesisFirehose::DeliveryStream 67 | Properties: 68 | DeliveryStreamName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-firehose" 69 | DeliveryStreamType: DirectPut 70 | SplunkDestinationConfiguration: 71 | BufferingHints: 72 | IntervalInSeconds: 5 73 | SizeInMBs: 1 74 | CloudWatchLoggingOptions: 75 | Enabled: true 76 | LogGroupName: !Ref firehoseLogGroup 77 | LogStreamName: "SplunkDelivery" 78 | HECAcknowledgmentTimeoutInSeconds: 300 79 | HECEndpoint: !Ref splunkHECEndpoint 80 | HECEndpointType: "Event" 81 | HECToken: !Ref splunkHECToken 82 | ProcessingConfiguration: 83 | Enabled: true 84 | Processors: 85 | - Parameters: 86 | - ParameterName: LambdaArn 87 | ParameterValue: !GetAtt lambdaFunction.Arn 88 | - ParameterName: RoleArn 89 | ParameterValue: !GetAtt firehoseIAMRole.Arn 90 | - ParameterName: BufferSizeInMBs 91 | ParameterValue: 1 92 | - ParameterName: BufferIntervalInSeconds 93 | ParameterValue: 5 94 | Type: Lambda 95 | RetryOptions: 96 | DurationInSeconds: 3600 97 | S3BackupMode: "FailedEventsOnly" 98 | S3Configuration: 99 | BucketARN: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-cwmetrics-firehose-backsplash" 100 | BufferingHints: 101 | IntervalInSeconds: 300 102 | SizeInMBs: 16 103 | CompressionFormat: "GZIP" 104 | Prefix: !Sub "$${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-firehose" 105 | RoleARN: !GetAtt firehoseIAMRole.Arn 106 | 107 | firehoseIAMPolicy: 108 | Type: AWS::IAM::ManagedPolicy 109 | Properties: 110 | PolicyDocument: 111 | Version: '2012-10-17' 112 | Statement: 113 | - Effect: Allow 114 | Action: 115 | - logs:Describe* 116 | - logs:PutLogEvents 117 | Resource: !GetAtt firehoseLogGroup.Arn 118 | - Effect: Allow 119 | Action: 120 | - s3:PutObject 121 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-cwmetrics-firehose-backsplash/*" 122 | - Effect: Allow 123 | Action: 124 | - lambda:InvokeFunction 125 | - lambda:GetFunctionConfiguration 126 | Resource: !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-lambda-function" 127 | ManagedPolicyName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-firehose-iam-policy" 128 | 129 | firehoseIAMRole: 130 | Type: AWS::IAM::Role 131 | Properties: 132 | AssumeRolePolicyDocument: 133 | Version: '2012-10-17' 134 | Statement: 135 | - Effect: "Allow" 136 | Principal: 137 | Service: "firehose.amazonaws.com" 138 | Action: 139 | - sts:AssumeRole 140 | ManagedPolicyArns: 141 | - !Ref firehoseIAMPolicy 142 | RoleName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-firehose-iam-role" 143 | Tags: 144 | - Key: service 145 | Value: !Ref service 146 | - Key: stage 147 | Value: !Ref stage 148 | - Key: contact 149 | Value: !Ref contact 150 | 151 | firehoseBacksplashBucket: 152 | Type: AWS::S3::Bucket 153 | Properties: 154 | BucketName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-firehose-backsplash" 155 | AccessControl: Private 156 | PublicAccessBlockConfiguration: 157 | BlockPublicAcls: true 158 | BlockPublicPolicy: true 159 | IgnorePublicAcls: true 160 | RestrictPublicBuckets: true 161 | BucketEncryption: 162 | ServerSideEncryptionConfiguration: 163 | - ServerSideEncryptionByDefault: 164 | SSEAlgorithm: AES256 165 | LifecycleConfiguration: 166 | Rules: 167 | - Id: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-firehose-backsplash-cleanup" 168 | AbortIncompleteMultipartUpload: 169 | DaysAfterInitiation: 1 170 | Status: Enabled 171 | Tags: 172 | - Key: service 173 | Value: !Ref service 174 | - Key: stage 175 | Value: !Ref stage 176 | - Key: contact 177 | Value: !Ref contact 178 | 179 | firehoseLogGroup: 180 | Type: AWS::Logs::LogGroup 181 | Properties: 182 | LogGroupName: !Sub "/aws/kinesisfirehose/${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-firehose" 183 | RetentionInDays: 7 184 | 185 | firehoseLogStream: 186 | Type: AWS::Logs::LogStream 187 | Properties: 188 | LogGroupName: !Ref firehoseLogGroup 189 | LogStreamName: "SplunkDelivery" 190 | 191 | # Lambda resources 192 | lambdaFunction: 193 | Type: AWS::Lambda::Function 194 | DependsOn: lambdaLogGroup 195 | Properties: 196 | Architectures: 197 | - arm64 198 | Code: 199 | ZipFile: | 200 | import json, os, base64, datetime, copy 201 | 202 | SPLUNK_EVENT_TYPE = os.environ['SPLUNK_EVENT_TYPE'] 203 | SPLUNK_SOURCE = os.environ['SPLUNK_SOURCE'] 204 | SPLUNK_HOST = os.environ['SPLUNK_HOST'] 205 | SPLUNK_INDEX = os.environ['SPLUNK_INDEX'] 206 | 207 | # Parse message and return event-formatted record 208 | def parseEventAsEvent(message): 209 | 210 | # Parse as JSON event 211 | jsonMessage = json.loads(message) 212 | 213 | # Create Splunk Event 214 | splunkEvent = {} 215 | splunkEvent["Average"] = jsonMessage['value']['sum'] / jsonMessage['value']['count'] 216 | splunkEvent["Maximum"] = jsonMessage['value']['max'] 217 | splunkEvent["Minimum"] = jsonMessage['value']['min'] 218 | splunkEvent["SampleCount"] = jsonMessage['value']['count'] 219 | splunkEvent["Sum"] = jsonMessage['value']['sum'] 220 | splunkEvent["Unit"] = jsonMessage['unit'] 221 | splunkEvent["account_id"] = jsonMessage['account_id'] 222 | splunkEvent["metric_name"] = jsonMessage['metric_name'] 223 | splunkEvent["Namespace"] = jsonMessage['namespace'] 224 | splunkEvent["timestamp"] = datetime.datetime.fromtimestamp(jsonMessage['timestamp']/1000).isoformat() + "Z" 225 | metric_dimensions = "" 226 | for dimensionKey in jsonMessage['dimensions'].keys(): 227 | metric_dimensions += (str(dimensionKey) + "=[" + str(jsonMessage['dimensions'][str(dimensionKey)]) + "],") 228 | splunkEvent["metric_dimensions"] = metric_dimensions[:-1] 229 | 230 | # Return Splunk event 231 | return '{ "time": ' + str(jsonMessage['timestamp']) + ', "host": "' + SPLUNK_HOST + '", "source": "' + SPLUNK_SOURCE + '", "sourcetype": "aws:cloudwatch", "index": "' + SPLUNK_INDEX + '", "event": ' + json.dumps(splunkEvent) + ' }' 232 | 233 | # Parse message and return metric-formatted record 234 | def parseEventAsMetric(message): 235 | 236 | # Parse as JSON event 237 | jsonMessage = json.loads(message) 238 | 239 | # Create Splunk Event 240 | splunkEvent = {} 241 | splunkEvent["AccountID"] = jsonMessage['account_id'] 242 | splunkEvent["MetricName"] = jsonMessage['metric_name'] 243 | splunkEvent["Namespace"] = jsonMessage['namespace'] 244 | splunkEvent["Unit"] = jsonMessage['unit'] 245 | splunkEvent["Region"] = jsonMessage['region'] 246 | splunkEvent["metric_name:" + jsonMessage['namespace'] + "." + jsonMessage['metric_name'] + ".Average"] = jsonMessage['value']['sum'] / jsonMessage['value']['count'] 247 | splunkEvent["metric_name:" + jsonMessage['namespace'] + "." + jsonMessage['metric_name'] + ".Maximum"] = jsonMessage['value']['max'] 248 | splunkEvent["metric_name:" + jsonMessage['namespace'] + "." + jsonMessage['metric_name'] + ".Minimum"] = jsonMessage['value']['min'] 249 | splunkEvent["metric_name:" + jsonMessage['namespace'] + "." + jsonMessage['metric_name'] + ".SampleCount"] = jsonMessage['value']['count'] 250 | splunkEvent["metric_name:" + jsonMessage['namespace'] + "." + jsonMessage['metric_name'] + ".Sum"] = jsonMessage['value']['sum'] 251 | for dimensionKey in jsonMessage['dimensions'].keys(): 252 | splunkEvent[dimensionKey] = jsonMessage['dimensions'][str(dimensionKey)] 253 | 254 | # Return Splunk event 255 | return '{ "time": ' + str(jsonMessage['timestamp']) + ', "host": "' + SPLUNK_HOST + '", "source": "' + SPLUNK_SOURCE + '", "sourcetype": "aws:cloudwatch:metric", "index": "' + SPLUNK_INDEX + '", "event": ' + json.dumps(splunkEvent) + ' }' 256 | 257 | # Default Lambda handler 258 | def handler(event, context): 259 | 260 | returnRecords = [] 261 | 262 | # Decode events, and split into separate items in a list 263 | for record in event['records']: 264 | 265 | data = base64.b64decode(record['data']).decode('utf-8').splitlines() 266 | 267 | formattedEvents = "" 268 | returnEvent = {} 269 | 270 | for message in data: 271 | 272 | match SPLUNK_EVENT_TYPE: 273 | case "event": # Parse event as event-style message 274 | formattedEvents += parseEventAsEvent(message) + "\n" 275 | case "metric": # Parse event as metric-style message 276 | formattedEvents += parseEventAsMetric(message) + "\n" 277 | 278 | returnEvent['recordId'] = dict(record)['recordId'] 279 | returnEvent['result'] = "Ok" 280 | returnEvent['data'] = base64.b64encode(bytearray(formattedEvents, 'utf-8')) 281 | 282 | returnRecords.append(returnEvent) 283 | 284 | return {'records': returnRecords} 285 | Description: Lambda function for processing Firehose messages into standard CloudWatch Metrics format for Splunk. 286 | Environment: 287 | Variables: 288 | SPLUNK_INDEX: !Ref splunkIndex 289 | SPLUNK_EVENT_TYPE: !Ref splunkEventType 290 | SPLUNK_SOURCE: !Ref splunkSource 291 | SPLUNK_HOST: !Ref splunkHost 292 | FunctionName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-lambda-function" 293 | Handler: index.handler 294 | MemorySize: 512 295 | Role: !GetAtt lambdaIAMRole.Arn 296 | Runtime: python3.12 297 | Tags: 298 | - Key: service 299 | Value: !Ref service 300 | - Key: stage 301 | Value: !Ref stage 302 | - Key: contact 303 | Value: !Ref contact 304 | Timeout: 300 305 | 306 | lambdaIAMPolicy: 307 | Type: AWS::IAM::ManagedPolicy 308 | Properties: 309 | PolicyDocument: 310 | Version: '2012-10-17' 311 | Statement: 312 | - Effect: Allow 313 | Action: 314 | - firehose:PutRecord 315 | - firehose:PutRecordBatch 316 | Resource: !Sub "arn:aws:firehose:${AWS::Region}:${AWS::AccountId}:deliverystream/${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-firehose" 317 | - Effect: Allow 318 | Action: 319 | - logs:CreateLogStream 320 | - logs:PutLogEvents 321 | Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-lambda-function*" 322 | ManagedPolicyName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-lambda-iam-policy" 323 | 324 | lambdaIAMRole: 325 | Type: AWS::IAM::Role 326 | Properties: 327 | AssumeRolePolicyDocument: 328 | Version: '2012-10-17' 329 | Statement: 330 | - Effect: "Allow" 331 | Principal: 332 | Service: "lambda.amazonaws.com" 333 | Action: 334 | - sts:AssumeRole 335 | ManagedPolicyArns: 336 | - !Ref lambdaIAMPolicy 337 | RoleName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-lambda-iam-role" 338 | Tags: 339 | - Key: service 340 | Value: !Ref service 341 | - Key: stage 342 | Value: !Ref stage 343 | - Key: contact 344 | Value: !Ref contact 345 | 346 | lambdaLogGroup: 347 | Type: AWS::Logs::LogGroup 348 | Properties: 349 | LogGroupName: !Sub "/aws/lambda/${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-lambda-function" 350 | RetentionInDays: 7 351 | 352 | # CloudWatch Metrics Stream 353 | cwMetricsStreamIAMPolicy: 354 | Type: AWS::IAM::ManagedPolicy 355 | Properties: 356 | PolicyDocument: 357 | Version: '2012-10-17' 358 | Statement: 359 | - Effect: Allow 360 | Action: 361 | - firehose:PutRecord 362 | - firehose:PutRecordBatch 363 | Resource: !GetAtt firehose.Arn 364 | ManagedPolicyName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-iam-policy" 365 | 366 | cwMetricsStreamIAMRole: 367 | Type: AWS::IAM::Role 368 | Properties: 369 | AssumeRolePolicyDocument: 370 | Version: '2012-10-17' 371 | Statement: 372 | - Effect: "Allow" 373 | Principal: 374 | Service: "streams.metrics.cloudwatch.amazonaws.com" 375 | Action: 376 | - sts:AssumeRole 377 | ManagedPolicyArns: 378 | - !Ref cwMetricsStreamIAMPolicy 379 | RoleName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-iam-role" 380 | Tags: 381 | - Key: service 382 | Value: !Ref service 383 | - Key: stage 384 | Value: !Ref stage 385 | - Key: contact 386 | Value: !Ref contact 387 | 388 | cwMetricsStream: 389 | Type: AWS::CloudWatch::MetricStream 390 | Properties: 391 | FirehoseArn: !GetAtt firehose.Arn 392 | Name: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-stream" 393 | OutputFormat: json 394 | RoleArn: !GetAtt cwMetricsStreamIAMRole.Arn 395 | Tags: 396 | - Key: service 397 | Value: !Ref service 398 | - Key: stage 399 | Value: !Ref stage 400 | - Key: contact 401 | Value: !Ref contact 402 | 403 | # Monitoring resoruces 404 | monitoringSNSTopic: 405 | Condition: enableAlerting 406 | Type: AWS::SNS::Topic 407 | Properties: 408 | DisplayName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-alerting-topic" 409 | TopicName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-alerting-topic" 410 | Subscription: 411 | - Endpoint: !Ref cloudWatchAlertEmail 412 | Protocol: email 413 | 414 | lambdaInvocationAlarm: 415 | Condition: enableAlerting 416 | Type: AWS::CloudWatch::Alarm 417 | Properties: 418 | ActionsEnabled: True 419 | AlarmActions: 420 | - !Ref monitoringSNSTopic 421 | AlarmDescription: !Sub "Alarm if lambda function ${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-lambda-function errors out. Check CloudWatch Logs to verify the function is running correctly." 422 | AlarmName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-lambda-error-invocations" 423 | ComparisonOperator: GreaterThanOrEqualToThreshold 424 | Dimensions: 425 | - Name: FunctionName 426 | Value: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-lambda-function" 427 | EvaluationPeriods: 1 428 | MetricName: Errors 429 | Namespace: AWS/Lambda 430 | Period: 60 431 | Statistic: Sum 432 | Threshold: 1 433 | Unit: Count 434 | 435 | lambdaDurationAlarm: 436 | Condition: enableAlerting 437 | Type: AWS::CloudWatch::Alarm 438 | Properties: 439 | ActionsEnabled: True 440 | AlarmActions: 441 | - !Ref monitoringSNSTopic 442 | AlarmDescription: !Sub "Alarm if lambda function ${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-lambda-function runs over 5 minutes. Consider tuning the lambdaProcessorBatchSize and lambdaProcessorBatchingWindowInSeconds parameters." 443 | AlarmName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-lambda-error-duration" 444 | ComparisonOperator: GreaterThanOrEqualToThreshold 445 | Dimensions: 446 | - Name: FunctionName 447 | Value: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-lambda-function" 448 | EvaluationPeriods: 1 449 | MetricName: Duration 450 | Namespace: AWS/Lambda 451 | Period: 60 452 | Statistic: Maximum 453 | Threshold: 300000 454 | Unit: Milliseconds 455 | 456 | firehoseDeliveryAlarm: 457 | Condition: enableAlerting 458 | Type: AWS::CloudWatch::Alarm 459 | Properties: 460 | ActionsEnabled: True 461 | AlarmActions: 462 | - !Ref monitoringSNSTopic 463 | AlarmDescription: !Sub "Alarm if Firehose ${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-firehose cannot deliver to Splunk." 464 | AlarmName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-firehose-delivery-alarm" 465 | ComparisonOperator: LessThanThreshold 466 | Dimensions: 467 | - Name: DeliveryStreamName 468 | Value: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-${splunkEventType}-firehose" 469 | EvaluationPeriods: 1 470 | MetricName: DeliveryToSplunk.Success 471 | Namespace: AWS/Firehose 472 | Period: 60 473 | Statistic: Maximum 474 | Threshold: 1 475 | Unit: Count 476 | 477 | firehoseThrottlingAlarm: 478 | Condition: enableAlerting 479 | Type: AWS::CloudWatch::Alarm 480 | Properties: 481 | ActionsEnabled: True 482 | AlarmActions: 483 | - !Ref monitoringSNSTopic 484 | AlarmDescription: !Sub "Alarm if Firehose ${AWS::AccountId}-${AWS::Region}-cwmetrics-firehose is throttling events because of quota limits." 485 | AlarmName: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-firehose-throttling-alarm" 486 | ComparisonOperator: GreaterThanThreshold 487 | Dimensions: 488 | - Name: DeliveryStreamName 489 | Value: !Sub "${AWS::AccountId}-${AWS::Region}-cwmetrics-firehose" 490 | EvaluationPeriods: 1 491 | MetricName: ThrottledRecords 492 | Namespace: AWS/Firehose 493 | Period: 60 494 | Statistic: Maximum 495 | Threshold: 0 496 | Unit: Count 497 | 498 | Outputs: 499 | firehoseArn: 500 | Value: !GetAtt firehose.Arn 501 | firehoseIAMRoleArn: 502 | Value: !GetAtt firehoseIAMRole.Arn 503 | firehoseBacksplashBucketArn: 504 | Value: !GetAtt firehoseBacksplashBucket.Arn 505 | firehoseLogGroupArn: 506 | Value: !GetAtt firehoseLogGroup.Arn 507 | cwMetricsStreamArn: 508 | Value: !GetAtt cwMetricsStream.Arn 509 | cwMetricsStreamIAMRoleArn: 510 | Value: !GetAtt cwMetricsStreamIAMRole.Arn 511 | lambdaFunctionArn: 512 | Value: !GetAtt lambdaFunction.Arn 513 | lambdaIAMRoleArn: 514 | Value: !GetAtt lambdaIAMRole.Arn 515 | lambdaLogGroupArn: 516 | Value: !GetAtt lambdaLogGroup.Arn 517 | monitoringSNSTopicArn: 518 | Condition: enableAlerting 519 | Value: !Ref monitoringSNSTopic -------------------------------------------------------------------------------- /CloudWatchMetrics-Firehose-Resources/lambda.py: -------------------------------------------------------------------------------- 1 | import json, os, base64, datetime, copy 2 | 3 | SPLUNK_EVENT_TYPE = os.environ['SPLUNK_EVENT_TYPE'] 4 | SPLUNK_SOURCE = os.environ['SPLUNK_SOURCE'] 5 | SPLUNK_HOST = os.environ['SPLUNK_HOST'] 6 | SPLUNK_INDEX = os.environ['SPLUNK_INDEX'] 7 | 8 | # Parse message and return event-formatted record 9 | def parseEventAsEvent(message): 10 | 11 | # Parse as JSON event 12 | jsonMessage = json.loads(message) 13 | 14 | # Create Splunk Event 15 | splunkEvent = {} 16 | splunkEvent["Average"] = jsonMessage['value']['sum'] / jsonMessage['value']['count'] 17 | splunkEvent["Maximum"] = jsonMessage['value']['max'] 18 | splunkEvent["Minimum"] = jsonMessage['value']['min'] 19 | splunkEvent["SampleCount"] = jsonMessage['value']['count'] 20 | splunkEvent["Sum"] = jsonMessage['value']['sum'] 21 | splunkEvent["Unit"] = jsonMessage['unit'] 22 | splunkEvent["account_id"] = jsonMessage['account_id'] 23 | splunkEvent["metric_name"] = jsonMessage['metric_name'] 24 | splunkEvent["Namespace"] = jsonMessage['namespace'] 25 | splunkEvent["timestamp"] = datetime.datetime.fromtimestamp(jsonMessage['timestamp']/1000).isoformat() + "Z" 26 | metric_dimensions = "" 27 | for dimensionKey in jsonMessage['dimensions'].keys(): 28 | metric_dimensions += (str(dimensionKey) + "=[" + str(jsonMessage['dimensions'][str(dimensionKey)]) + "],") 29 | splunkEvent["metric_dimensions"] = metric_dimensions[:-1] 30 | 31 | # Return Splunk event 32 | return '{ "time": ' + str(jsonMessage['timestamp']) + ', "host": "' + SPLUNK_HOST + '", "source": "' + SPLUNK_SOURCE + '", "sourcetype": "aws:cloudwatch", "index": "' + SPLUNK_INDEX + '", "event": ' + json.dumps(splunkEvent) + ' }' 33 | 34 | # Parse message and return metric-formatted record 35 | def parseEventAsMetric(message): 36 | 37 | # Parse as JSON event 38 | jsonMessage = json.loads(message) 39 | 40 | # Create Splunk Event 41 | splunkEvent = {} 42 | splunkEvent["AccountID"] = jsonMessage['account_id'] 43 | splunkEvent["MetricName"] = jsonMessage['metric_name'] 44 | splunkEvent["Namespace"] = jsonMessage['namespace'] 45 | splunkEvent["Unit"] = jsonMessage['unit'] 46 | splunkEvent["Region"] = jsonMessage['region'] 47 | splunkEvent["metric_name:" + jsonMessage['namespace'] + "." + jsonMessage['metric_name'] + ".Average"] = jsonMessage['value']['sum'] / jsonMessage['value']['count'] 48 | splunkEvent["metric_name:" + jsonMessage['namespace'] + "." + jsonMessage['metric_name'] + ".Maximum"] = jsonMessage['value']['max'] 49 | splunkEvent["metric_name:" + jsonMessage['namespace'] + "." + jsonMessage['metric_name'] + ".Minimum"] = jsonMessage['value']['min'] 50 | splunkEvent["metric_name:" + jsonMessage['namespace'] + "." + jsonMessage['metric_name'] + ".SampleCount"] = jsonMessage['value']['count'] 51 | splunkEvent["metric_name:" + jsonMessage['namespace'] + "." + jsonMessage['metric_name'] + ".Sum"] = jsonMessage['value']['sum'] 52 | for dimensionKey in jsonMessage['dimensions'].keys(): 53 | splunkEvent[dimensionKey] = jsonMessage['dimensions'][str(dimensionKey)] 54 | 55 | # Return Splunk event 56 | return '{ "time": ' + str(jsonMessage['timestamp']) + ', "host": "' + SPLUNK_HOST + '", "source": "' + SPLUNK_SOURCE + '", "sourcetype": "aws:cloudwatch:metric", "index": "' + SPLUNK_INDEX + '", "event": ' + json.dumps(splunkEvent) + ' }' 57 | 58 | # Default Lambda handler 59 | def handler(event, context): 60 | 61 | returnRecords = [] 62 | 63 | # Decode events, and split into separate items in a list 64 | for record in event['records']: 65 | 66 | data = base64.b64decode(record['data']).decode('utf-8').splitlines() 67 | 68 | formattedEvents = "" 69 | returnEvent = {} 70 | 71 | for message in data: 72 | 73 | match SPLUNK_EVENT_TYPE: 74 | case "event": # Parse event as event-style message 75 | formattedEvents += parseEventAsEvent(message) + "\n" 76 | case "metric": # Parse event as metric-style message 77 | formattedEvents += parseEventAsMetric(message) + "\n" 78 | 79 | returnEvent['recordId'] = dict(record)['recordId'] 80 | returnEvent['result'] = "Ok" 81 | returnEvent['data'] = base64.b64encode(bytearray(formattedEvents, 'utf-8')) 82 | 83 | returnRecords.append(returnEvent) 84 | 85 | return {'records': returnRecords} 86 | -------------------------------------------------------------------------------- /CloudWatchMetrics-Firehose-Resources/splunkDashboard-event.json: -------------------------------------------------------------------------------- 1 | { 2 | "visualizations": { 3 | "viz_AcyVeBot": { 4 | "type": "splunk.column", 5 | "showProgressBar": false, 6 | "showLastUpdated": false, 7 | "title": "$sourcetype$ License Usage Over Time (GB)", 8 | "dataSources": { 9 | "primary": "ds_6WBt9iY9" 10 | }, 11 | "options": { 12 | "legendDisplay": "off" 13 | } 14 | }, 15 | "viz_KMF849o7": { 16 | "type": "splunk.singlevalue", 17 | "title": "$sourcetype$ Event License Usage", 18 | "dataSources": { 19 | "primary": "ds_HUEXsOZi" 20 | }, 21 | "options": { 22 | "numberPrecision": 2, 23 | "unit": "GB", 24 | "trendDisplay": "off", 25 | "sparklineDisplay": "off" 26 | } 27 | }, 28 | "viz_OF3a0hX5": { 29 | "type": "splunk.singlevalue", 30 | "title": "$sourcetype$ Event Count", 31 | "dataSources": { 32 | "primary": "ds_LN5KxPiz" 33 | }, 34 | "options": { 35 | "unit": "events", 36 | "trendDisplay": "off", 37 | "sparklineDisplay": "off" 38 | } 39 | }, 40 | "viz_kqPNrs3R": { 41 | "type": "splunk.line", 42 | "showProgressBar": false, 43 | "showLastUpdated": false, 44 | "title": "$sourcetype$ Event Count Over Time", 45 | "dataSources": { 46 | "primary": "ds_7YVGDkt0" 47 | }, 48 | "options": { 49 | "legendDisplay": "off" 50 | } 51 | }, 52 | "viz_eU3T7rCG": { 53 | "type": "splunk.singlevalue", 54 | "title": "AWS Accounts Sending Events", 55 | "dataSources": { 56 | "primary": "ds_noJxIRWy" 57 | } 58 | }, 59 | "viz_nO6QvmkM": { 60 | "type": "splunk.line", 61 | "title": "AWS Accounts Sending $sourcetype$ Events", 62 | "dataSources": { 63 | "primary": "ds_FFly4UzK" 64 | }, 65 | "options": { 66 | "legendDisplay": "off" 67 | } 68 | }, 69 | "viz_NY0AedMk": { 70 | "type": "splunk.line", 71 | "title": "$sourcetype$ Event Latency Over Time", 72 | "showProgressBar": false, 73 | "showLastUpdated": false, 74 | "dataSources": { 75 | "primary": "ds_srz6OcK5" 76 | }, 77 | "options": { 78 | "legendDisplay": "off" 79 | } 80 | }, 81 | "viz_XqNxe7aK": { 82 | "type": "splunk.singlevalue", 83 | "title": "$sourcetype$ Latest Event Latency", 84 | "dataSources": { 85 | "primary": "ds_tCds3VFx" 86 | }, 87 | "options": { 88 | "sparklineDisplay": "off", 89 | "unit": "seconds" 90 | }, 91 | "showProgressBar": false, 92 | "showLastUpdated": false 93 | } 94 | }, 95 | "dataSources": { 96 | "ds_6WBt9iY9": { 97 | "type": "ds.search", 98 | "options": { 99 | "query": "index=_internal source=*license_usage* idx=$eventIndex$ st=$sourcetype$ NOT source=*license_usage_summary.log*\n| timechart sum(b) as ingestedDataGB\n| eval ingestedDataGB = ingestedDataGB/1024/1024/1024", 100 | "queryParameters": { 101 | "earliest": "$global_time.earliest$", 102 | "latest": "$global_time.latest$" 103 | } 104 | }, 105 | "name": "licenseUsageBase" 106 | }, 107 | "ds_HUEXsOZi": { 108 | "type": "ds.chain", 109 | "options": { 110 | "extend": "ds_6WBt9iY9", 111 | "query": "| stats sum(ingestedDataGB) as ingestedDataGB" 112 | }, 113 | "name": "licenseUsageTotal" 114 | }, 115 | "ds_7YVGDkt0": { 116 | "type": "ds.search", 117 | "options": { 118 | "query": "index=_internal series=$sourcetype$ sourcetype=splunkd source=*metrics.log*\n| timechart sum(ev) as eventCount" 119 | }, 120 | "name": "eventCountBase" 121 | }, 122 | "ds_LN5KxPiz": { 123 | "type": "ds.chain", 124 | "options": { 125 | "extend": "ds_7YVGDkt0", 126 | "query": "| stats sum(eventCount) as eventCount" 127 | }, 128 | "name": "eventCountTotal" 129 | }, 130 | "ds_FFly4UzK": { 131 | "type": "ds.search", 132 | "options": { 133 | "query": "index=$eventIndex$ sourcetype=$sourcetype$\n| timechart dc(account_id) as accountCount" 134 | }, 135 | "name": "accountCountTimechart" 136 | }, 137 | "ds_noJxIRWy": { 138 | "type": "ds.search", 139 | "options": { 140 | "query": "index=$eventIndex$ sourcetype=$sourcetype$\n| stats dc(account_id) as accountCount" 141 | }, 142 | "name": "accountCountTotal" 143 | }, 144 | "ds_srz6OcK5": { 145 | "type": "ds.search", 146 | "options": { 147 | "query": "index=$eventIndex$ sourcetype=$sourcetype$ \n| eval eventLatency = abs(_indextime - _time)\n| timechart avg(eventLatency) as eventLatency" 148 | }, 149 | "name": "eventLatencyBase" 150 | }, 151 | "ds_tCds3VFx": { 152 | "type": "ds.search", 153 | "options": { 154 | "query": "index=$eventIndex$ sourcetype=$sourcetype$ | head 1\n| eval eventLatency = abs(_indextime - _time)\n| table eventLatency" 155 | }, 156 | "name": "eventLatencyLatest" 157 | }, 158 | "ds_N5KOizZ9": { 159 | "type": "ds.search", 160 | "options": { 161 | "query": "| tstats count where index=* by index", 162 | "queryParameters": { 163 | "earliest": "$global_time.earliest$", 164 | "latest": "$global_time.latest$" 165 | } 166 | }, 167 | "name": "indexList" 168 | }, 169 | "ds_f9m7vKr7": { 170 | "type": "ds.search", 171 | "options": { 172 | "query": "| tstats count where index=$eventIndex$ by sourcetype" 173 | }, 174 | "name": "sourcetypeList" 175 | } 176 | }, 177 | "defaults": { 178 | "dataSources": { 179 | "ds.search": { 180 | "options": { 181 | "queryParameters": { 182 | "latest": "$global_time.latest$", 183 | "earliest": "$global_time.earliest$" 184 | } 185 | } 186 | } 187 | } 188 | }, 189 | "inputs": { 190 | "input_global_trp": { 191 | "type": "input.timerange", 192 | "options": { 193 | "token": "global_time", 194 | "defaultValue": "-24h@h,now" 195 | }, 196 | "title": "Global Time Range" 197 | }, 198 | "input_9rNFa8NM": { 199 | "options": { 200 | "items": [ 201 | { 202 | "label": "main", 203 | "value": "main" 204 | } 205 | ], 206 | "defaultValue": "main", 207 | "token": "eventIndex" 208 | }, 209 | "title": "Event Index", 210 | "type": "input.dropdown", 211 | "dataSources": { 212 | "primary": "ds_N5KOizZ9" 213 | } 214 | }, 215 | "input_C3SY1WEW": { 216 | "options": { 217 | "items": [ 218 | { 219 | "label": "All", 220 | "value": "*" 221 | } 222 | ], 223 | "defaultValue": "*", 224 | "token": "sourcetype" 225 | }, 226 | "title": "Event Sourcetype", 227 | "type": "input.dropdown", 228 | "dataSources": { 229 | "primary": "ds_f9m7vKr7" 230 | } 231 | } 232 | }, 233 | "layout": { 234 | "type": "grid", 235 | "options": {}, 236 | "structure": [ 237 | { 238 | "item": "viz_eU3T7rCG", 239 | "type": "block", 240 | "position": { 241 | "x": 600, 242 | "y": 0, 243 | "w": 300, 244 | "h": 100 245 | } 246 | }, 247 | { 248 | "item": "viz_XqNxe7aK", 249 | "type": "block", 250 | "position": { 251 | "x": 900, 252 | "y": 0, 253 | "w": 300, 254 | "h": 100 255 | } 256 | }, 257 | { 258 | "item": "viz_KMF849o7", 259 | "type": "block", 260 | "position": { 261 | "x": 0, 262 | "y": 0, 263 | "w": 300, 264 | "h": 100 265 | } 266 | }, 267 | { 268 | "item": "viz_AcyVeBot", 269 | "type": "block", 270 | "position": { 271 | "x": 0, 272 | "y": 100, 273 | "w": 1200, 274 | "h": 300 275 | } 276 | }, 277 | { 278 | "item": "viz_kqPNrs3R", 279 | "type": "block", 280 | "position": { 281 | "x": 0, 282 | "y": 400, 283 | "w": 1200, 284 | "h": 300 285 | } 286 | }, 287 | { 288 | "item": "viz_nO6QvmkM", 289 | "type": "block", 290 | "position": { 291 | "x": 0, 292 | "y": 700, 293 | "w": 1200, 294 | "h": 300 295 | } 296 | }, 297 | { 298 | "item": "viz_NY0AedMk", 299 | "type": "block", 300 | "position": { 301 | "x": 0, 302 | "y": 1000, 303 | "w": 1200, 304 | "h": 300 305 | } 306 | }, 307 | { 308 | "item": "viz_OF3a0hX5", 309 | "type": "block", 310 | "position": { 311 | "x": 300, 312 | "y": 0, 313 | "w": 300, 314 | "h": 100 315 | } 316 | } 317 | ], 318 | "globalInputs": [ 319 | "input_global_trp", 320 | "input_9rNFa8NM", 321 | "input_C3SY1WEW" 322 | ] 323 | }, 324 | "description": "", 325 | "title": "Splunk AWS GDI Toolkit - CloudWatch-Metrics-Firehose as Event" 326 | } -------------------------------------------------------------------------------- /CloudWatchMetrics-Firehose-Resources/splunkDashboard-metric.json: -------------------------------------------------------------------------------- 1 | { 2 | "visualizations": { 3 | "viz_AcyVeBot": { 4 | "type": "splunk.column", 5 | "showProgressBar": false, 6 | "showLastUpdated": false, 7 | "title": "$sourcetype$ License Usage Over Time (GB)", 8 | "dataSources": { 9 | "primary": "ds_6WBt9iY9" 10 | }, 11 | "options": { 12 | "legendDisplay": "off" 13 | } 14 | }, 15 | "viz_KMF849o7": { 16 | "type": "splunk.singlevalue", 17 | "title": "$sourcetype$ Event License Usage", 18 | "dataSources": { 19 | "primary": "ds_HUEXsOZi" 20 | }, 21 | "options": { 22 | "numberPrecision": 2, 23 | "unit": "GB", 24 | "trendDisplay": "off", 25 | "sparklineDisplay": "off" 26 | } 27 | }, 28 | "viz_OF3a0hX5": { 29 | "type": "splunk.singlevalue", 30 | "title": "$sourcetype$ Event Count", 31 | "dataSources": { 32 | "primary": "ds_LN5KxPiz" 33 | }, 34 | "options": { 35 | "unit": "events", 36 | "trendDisplay": "off", 37 | "sparklineDisplay": "off" 38 | } 39 | }, 40 | "viz_kqPNrs3R": { 41 | "type": "splunk.line", 42 | "showProgressBar": false, 43 | "showLastUpdated": false, 44 | "title": "$sourcetype$ Event Count Over Time", 45 | "dataSources": { 46 | "primary": "ds_7YVGDkt0" 47 | }, 48 | "options": { 49 | "legendDisplay": "off" 50 | } 51 | }, 52 | "viz_eU3T7rCG": { 53 | "type": "splunk.singlevalue", 54 | "title": "AWS Accounts Sending Events", 55 | "dataSources": { 56 | "primary": "ds_noJxIRWy" 57 | } 58 | }, 59 | "viz_nO6QvmkM": { 60 | "type": "splunk.line", 61 | "title": "AWS Accounts Sending $sourcetype$ Events", 62 | "dataSources": { 63 | "primary": "ds_FFly4UzK" 64 | }, 65 | "options": { 66 | "legendDisplay": "off" 67 | } 68 | } 69 | }, 70 | "dataSources": { 71 | "ds_6WBt9iY9": { 72 | "type": "ds.search", 73 | "options": { 74 | "query": "index=_internal source=*license_usage* idx=$metricIndex$ st=$sourcetype$ NOT source=*license_usage_summary.log*\n| timechart sum(b) as ingestedDataGB\n| eval ingestedDataGB = ingestedDataGB/1024/1024/1024", 75 | "queryParameters": { 76 | "earliest": "$global_time.earliest$", 77 | "latest": "$global_time.latest$" 78 | } 79 | }, 80 | "name": "licenseUsageBase" 81 | }, 82 | "ds_HUEXsOZi": { 83 | "type": "ds.chain", 84 | "options": { 85 | "extend": "ds_6WBt9iY9", 86 | "query": "| stats sum(ingestedDataGB) as ingestedDataGB" 87 | }, 88 | "name": "licenseUsageTotal" 89 | }, 90 | "ds_7YVGDkt0": { 91 | "type": "ds.search", 92 | "options": { 93 | "query": "index=_internal series=$sourcetype$ sourcetype=splunkd source=*metrics.log*\n| timechart sum(ev) as eventCount" 94 | }, 95 | "name": "eventCountBase" 96 | }, 97 | "ds_LN5KxPiz": { 98 | "type": "ds.chain", 99 | "options": { 100 | "extend": "ds_7YVGDkt0", 101 | "query": "| stats sum(eventCount) as eventCount" 102 | }, 103 | "name": "eventCountTotal" 104 | }, 105 | "ds_FFly4UzK": { 106 | "type": "ds.search", 107 | "options": { 108 | "query": "| mstats count where index=$metricIndex$ metric_name=* sourcetype=$sourcetype$ by AccountID span=15m\n| timechart dc(AccountID) as accountCount" 109 | }, 110 | "name": "accountCountTimechart" 111 | }, 112 | "ds_noJxIRWy": { 113 | "type": "ds.search", 114 | "options": { 115 | "query": "| mstats count where index=$metricIndex$ metric_name=* sourcetype=$sourcetype$c by AccountID\n| stats dc(AccountID) as accountCount" 116 | }, 117 | "name": "accountCountTotal" 118 | }, 119 | "ds_N5KOizZ9": { 120 | "type": "ds.search", 121 | "options": { 122 | "query": "| rest /services/data/indexes datatype=metric\n| table title", 123 | "queryParameters": { 124 | "earliest": "$global_time.earliest$", 125 | "latest": "$global_time.latest$" 126 | } 127 | }, 128 | "name": "indexList" 129 | }, 130 | "ds_f9m7vKr7": { 131 | "type": "ds.search", 132 | "options": { 133 | "query": "| mstats count where index=$metricIndex$ metric_name=* by sourcetype\n| table sourcetype" 134 | }, 135 | "name": "sourcetypeList" 136 | } 137 | }, 138 | "defaults": { 139 | "dataSources": { 140 | "ds.search": { 141 | "options": { 142 | "queryParameters": { 143 | "latest": "$global_time.latest$", 144 | "earliest": "$global_time.earliest$" 145 | } 146 | } 147 | } 148 | } 149 | }, 150 | "inputs": { 151 | "input_global_trp": { 152 | "type": "input.timerange", 153 | "options": { 154 | "token": "global_time", 155 | "defaultValue": "-24h@h,now" 156 | }, 157 | "title": "Global Time Range" 158 | }, 159 | "input_9rNFa8NM": { 160 | "options": { 161 | "items": [ 162 | { 163 | "label": "main", 164 | "value": "main" 165 | } 166 | ], 167 | "defaultValue": "main", 168 | "token": "metricIndex" 169 | }, 170 | "title": "Metric Index", 171 | "type": "input.dropdown", 172 | "dataSources": { 173 | "primary": "ds_N5KOizZ9" 174 | } 175 | }, 176 | "input_C3SY1WEW": { 177 | "options": { 178 | "items": [ 179 | { 180 | "label": "All", 181 | "value": "*" 182 | } 183 | ], 184 | "defaultValue": "*", 185 | "token": "sourcetype" 186 | }, 187 | "title": "Event Sourcetype", 188 | "type": "input.dropdown", 189 | "dataSources": { 190 | "primary": "ds_f9m7vKr7" 191 | } 192 | } 193 | }, 194 | "layout": { 195 | "type": "grid", 196 | "options": {}, 197 | "structure": [ 198 | { 199 | "item": "viz_KMF849o7", 200 | "type": "block", 201 | "position": { 202 | "x": 0, 203 | "y": 0, 204 | "w": 400, 205 | "h": 100 206 | } 207 | }, 208 | { 209 | "item": "viz_AcyVeBot", 210 | "type": "block", 211 | "position": { 212 | "x": 0, 213 | "y": 100, 214 | "w": 1200, 215 | "h": 300 216 | } 217 | }, 218 | { 219 | "item": "viz_kqPNrs3R", 220 | "type": "block", 221 | "position": { 222 | "x": 0, 223 | "y": 400, 224 | "w": 1200, 225 | "h": 300 226 | } 227 | }, 228 | { 229 | "item": "viz_nO6QvmkM", 230 | "type": "block", 231 | "position": { 232 | "x": 0, 233 | "y": 700, 234 | "w": 1200, 235 | "h": 300 236 | } 237 | }, 238 | { 239 | "item": "viz_OF3a0hX5", 240 | "type": "block", 241 | "position": { 242 | "x": 400, 243 | "y": 0, 244 | "w": 400, 245 | "h": 100 246 | } 247 | }, 248 | { 249 | "item": "viz_eU3T7rCG", 250 | "type": "block", 251 | "position": { 252 | "x": 800, 253 | "y": 0, 254 | "w": 400, 255 | "h": 100 256 | } 257 | } 258 | ], 259 | "globalInputs": [ 260 | "input_global_trp", 261 | "input_9rNFa8NM", 262 | "input_C3SY1WEW" 263 | ] 264 | }, 265 | "description": "", 266 | "title": "Splunk AWS GDI Toolkit - CloudWatch-Metrics-Firehose as Metric" 267 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/billingCURToS3.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: This is a CloudFormation template to create a AWS Cost and Usage Billing Report logging infrastructure to S3. This should be deployed at the Organization billing account if AWS Organizations is used, otherwise this should be deployed to the AWS account Billing CUR should be retrieved from. 4 | 5 | 6 | Parameters: 7 | service: 8 | Type: String 9 | Description: service name 10 | Default: splunk-aws-gdi-toolkit 11 | 12 | billingCURS3BucketName: 13 | Description: Destination bucket name that will receive the Billing CUR files. 14 | Type: String 15 | 16 | billingCURS3BucketRegion: 17 | Description: Region of the destination bucket name that will receive the Billing CUR files. 18 | Type: String 19 | 20 | 21 | Resources: 22 | 23 | billingCUR: 24 | Type: AWS::CUR::ReportDefinition 25 | Properties: 26 | AdditionalSchemaElements: 27 | - RESOURCES 28 | Compression: GZIP 29 | Format: textORcsv 30 | RefreshClosedReports: true 31 | ReportName: !Sub "${AWS::AccountId}-${AWS::Region}-billing-report" 32 | ReportVersioning: CREATE_NEW_REPORT 33 | S3Bucket: !Ref billingCURS3BucketName 34 | S3Prefix: !Ref service 35 | S3Region: !Ref billingCURS3BucketRegion 36 | TimeUnit: HOURLY 37 | 38 | 39 | Outputs: 40 | billingCURName: 41 | Value: !Ref billingCUR -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/cloudTrailToS3.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: This is a CloudFormation template to create CloudTrail logging infrastructure to S3. This should be done at the top-level Organization account to capture all of the Organization's CloudTrail logs - https://aws.amazon.com/cloudtrail/ 4 | 5 | 6 | Parameters: 7 | service: 8 | Type: String 9 | Description: service name 10 | Default: splunk-aws-gdi-toolkit 11 | 12 | stage: 13 | Type: String 14 | Description: Used to distinguish between stages of an environment (dev, test, prod, stage, etc). Only used in AWS resource tagging. 15 | Default: dev 16 | 17 | contact: 18 | Description: Used to identify a contact for the resources created in this stack. Only used in AWS resource tagging. As an example, this could be an email address or username. 19 | Type: String 20 | 21 | cloudTrailS3BucketName: 22 | Description: Destination bucket name that will receive CloudTrail events. 23 | Type: String 24 | 25 | cloudTrailLogFileValidation: 26 | Description: Used to enable or disable CloudTrail File Validation - https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-log-file-validation-intro.html . 27 | Default: true 28 | Type: String 29 | AllowedValues: 30 | - true 31 | - false 32 | 33 | cloudTrailIncludeGlobalServiceEvents: 34 | Description: Used to enable or disable logging of CloudTrail global service events - https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-concepts.html#cloudtrail-concepts-global-service-events . 35 | Default: true 36 | Type: String 37 | AllowedValues: 38 | - true 39 | - false 40 | 41 | 42 | Resources: 43 | cloudTrail: 44 | Type: AWS::CloudTrail::Trail 45 | Properties: 46 | EnableLogFileValidation: !Ref cloudTrailLogFileValidation 47 | IncludeGlobalServiceEvents: !Ref cloudTrailIncludeGlobalServiceEvents 48 | IsLogging: true 49 | IsMultiRegionTrail: true 50 | IsOrganizationTrail: true 51 | S3BucketName: !Ref cloudTrailS3BucketName 52 | Tags: 53 | - Key: service 54 | Value: !Ref service 55 | - Key: stage 56 | Value: !Ref stage 57 | - Key: contact 58 | Value: !Ref contact 59 | TrailName: !Sub "${service}-cloudtrail" 60 | 61 | 62 | Outputs: 63 | cloudTrailArn: 64 | Value: !GetAtt cloudTrail.Arn -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/lambda.py: -------------------------------------------------------------------------------- 1 | import boto3, gzip, json, os, sys, shutil, re, dateutil.parser, time, csv, datetime, pandas, pyarrow, urllib.parse, random 2 | 3 | # AWS-related setup 4 | s3Client = boto3.client('s3') 5 | firehoseDeliverySreamName = os.environ['firehoseDeliverySreamName'] 6 | firehoseClient = boto3.client('firehose', region_name=os.environ['AWS_REGION']) 7 | recordBatch = [] 8 | 9 | # Splunk-related setup 10 | SPLUNK_INDEX = os.environ['SPLUNK_INDEX'] 11 | SPLUNK_TIME_PREFIX = os.environ['SPLUNK_TIME_PREFIX'] 12 | SPLUNK_EVENT_DELIMITER = os.environ['SPLUNK_EVENT_DELIMITER'] 13 | SPLUNK_TIME_DELINEATED_FIELD = os.environ['SPLUNK_TIME_DELINEATED_FIELD'] 14 | SPLUNK_TIME_FORMAT = os.environ['SPLUNK_TIME_FORMAT'] 15 | SPLUNK_STRFTIME_FORMAT = os.environ['SPLUNK_STRFTIME_FORMAT'] 16 | SPLUNK_SOURCETYPE = os.environ['SPLUNK_SOURCETYPE'] 17 | SPLUNK_SOURCE = os.environ['SPLUNK_SOURCE'] 18 | SPLUNK_HOST = os.environ['SPLUNK_HOST'] 19 | SPLUNK_JSON_FORMAT = os.environ['SPLUNK_JSON_FORMAT'] 20 | SPLUNK_CSV_TO_JSON = os.environ['SPLUNK_CSV_TO_JSON'] 21 | SPLUNK_IGNORE_FIRST_LINE = os.environ['SPLUNK_IGNORE_FIRST_LINE'] 22 | SPLUNK_REMOVE_EMPTY_CSV_TO_JSON_FIELDS = os.environ['SPLUNK_REMOVE_EMPTY_CSV_TO_JSON_FIELDS'] 23 | 24 | # Lambda things 25 | validFileTypes = ["gz", "gzip", "json", "csv", "log", "parquet", "txt", "ndjson", "jsonl"] 26 | unsupportedFileTypes = ["CloudTrail-Digest", "billing-report-Manifest"] 27 | delimiterMapping = {"space": " ", "tab": " ", "comma": ",", "semicolon": ";"} 28 | maxRetriesToFirehose = 11 29 | 30 | # Create delimiter for delimiting events 31 | def createDelimiter(SPLUNK_EVENT_DELIMITER): 32 | 33 | if SPLUNK_EVENT_DELIMITER in delimiterMapping.keys(): 34 | return delimiterMapping[SPLUNK_EVENT_DELIMITER] 35 | else: 36 | return SPLUNK_EVENT_DELIMITER 37 | 38 | 39 | # Parse SQS message for bucket information 40 | def retrieveObjectInfo(record): 41 | 42 | # Try to parse the record for file information 43 | try: 44 | record = json.loads(record['body']) 45 | bucket = record['Records'][0]['s3']['bucket']['name'] 46 | key = urllib.parse.unquote_plus(record['Records'][0]['s3']['object']['key']) 47 | 48 | # Construct and return the result 49 | result = {} 50 | result["bucket"] = bucket 51 | result["key"] = key 52 | return result 53 | 54 | # Return an error if the record doesn't have a valid file defined in it 55 | except: 56 | return("SQS message did not contain S3 file information. Record: " + str(record)) 57 | 58 | 59 | # Check to see if the file is a valid file type 60 | def isValidFileType(key): 61 | 62 | # Check for invalid file types 63 | for unsupportedFileType in unsupportedFileTypes: 64 | if (unsupportedFileType in key): 65 | return False 66 | 67 | # Define file extension 68 | extension = key.split(".")[-1] 69 | 70 | # Check for valid file types 71 | if extension in validFileTypes: 72 | return True 73 | 74 | # Check for aws:s3:accesslogs 75 | if SPLUNK_SOURCETYPE == "aws:s3:accesslogs" and len(key.split(".")) == 1: 76 | return True 77 | 78 | return False 79 | 80 | 81 | # Retrieve the S3 object, and return the new path 82 | def downloadS3Object(bucket, key): 83 | 84 | try: 85 | # Define the path for the file 86 | path = "/tmp/" + key.split("/")[-1] 87 | 88 | # Download the file from the S3 bucket 89 | s3Client.download_file(bucket, key, path) 90 | 91 | # Return the new file path 92 | return(path) 93 | 94 | except: 95 | return "Unable to download file s3://" + bucket + "/" + key 96 | 97 | 98 | # Uncompress the file if it needs to be uncompressed, then return the path and the new file extension 99 | def uncompressFile(path): 100 | 101 | # Set file extension and new file path (if it gets uncompressed) 102 | extension = path.split(".")[-1] 103 | uncompressedFilePath = path[0:(-1*(len(extension)) - 1)] 104 | 105 | try: 106 | match extension: 107 | case "gz": 108 | 109 | with gzip.open(path, 'rb') as f_in: 110 | with open(uncompressedFilePath, 'wb') as f_out: 111 | shutil.copyfileobj(f_in, f_out) 112 | 113 | # Remove the uncompressed file 114 | os.remove(path) 115 | 116 | return uncompressedFilePath 117 | case "gzip": 118 | 119 | with gzip.open(path, 'rb') as f_in: 120 | with open(uncompressedFilePath, 'wb') as f_out: 121 | shutil.copyfileobj(f_in, f_out) 122 | 123 | # Remove the uncompressed file 124 | os.remove(path) 125 | 126 | return uncompressedFilePath 127 | case "parquet": 128 | 129 | df = pandas.read_parquet(path) 130 | json_array = df.to_json(orient='records', lines=True) 131 | uncompressedFilePath = uncompressedFilePath + ".json" 132 | with open(uncompressedFilePath, "w") as f_out: 133 | f_out.write(json_array) 134 | 135 | # Remove the uncompressed file 136 | os.remove(path) 137 | 138 | return uncompressedFilePath 139 | 140 | except: 141 | return "Unable to uncompress file" 142 | 143 | return path 144 | 145 | 146 | # Split events into a list. Additional file extensions should be added here. 147 | def eventBreak(events, extension, ignoreFirstLine): 148 | 149 | if extension == "csv" or extension == "log" or SPLUNK_SOURCETYPE == "aws:s3:accesslogs": 150 | 151 | splitEvents = events.split("\n") 152 | 153 | # Remove empty last line if it exists 154 | if len(splitEvents[-1]) == 0: 155 | splitEvents = splitEvents[:-1] 156 | 157 | if ignoreFirstLine == "true": 158 | splitEvents = splitEvents[1:] 159 | 160 | events = "" 161 | 162 | return splitEvents 163 | 164 | elif extension == "json" or extension == "txt" or extension=="jsonl": 165 | 166 | if SPLUNK_JSON_FORMAT == "eventsInRecords": 167 | splitEvents = json.loads(events)["Records"] 168 | events = "" 169 | 170 | return splitEvents 171 | 172 | elif SPLUNK_JSON_FORMAT == "NDJSON": 173 | splitEvents = events.split("\n") 174 | events = "" 175 | 176 | if len(splitEvents[-1]) == 0: 177 | splitEvents = splitEvents[:-1] 178 | 179 | return splitEvents 180 | 181 | else: 182 | return "File type invalid" 183 | 184 | 185 | # Clean up first line 186 | def cleanFirstLine(splitEvents): 187 | 188 | # If the sourcetype is aws:billing:cur, remove everything before the "/" in the CSV header 189 | if SPLUNK_SOURCETYPE == "aws:billing:cur": 190 | 191 | header = splitEvents[0] 192 | 193 | newHeader = "" 194 | 195 | for splitHeader in header.split(","): 196 | newHeader += "/".join(splitHeader.split("/")[1:]) + "," 197 | 198 | splitEvents[0] = newHeader[:-1] 199 | 200 | return splitEvents 201 | 202 | 203 | # Handle CSV to JSON conversion, and optionally remove null fields 204 | def csvToJSON(splitEvents): 205 | 206 | newEvents = [] 207 | 208 | # Change CSVs with headers to JSON format 209 | csvSplit = csv.DictReader(splitEvents) 210 | for csvRow in csvSplit: 211 | newEvents.append(csvRow) 212 | 213 | # Remove JSON fields with null or no value 214 | if SPLUNK_REMOVE_EMPTY_CSV_TO_JSON_FIELDS == "true": 215 | 216 | newEventsWithoutEmptyValues = [] 217 | for newEvent in newEvents: 218 | newEventWithoutEmptyValues = {} 219 | 220 | for newEventKey in newEvent.keys(): 221 | if len(newEvent[newEventKey]) > 0: 222 | newEventWithoutEmptyValues[newEventKey] = newEvent[newEventKey] 223 | 224 | newEventsWithoutEmptyValues.append(newEventWithoutEmptyValues) 225 | 226 | newEvents.clear() 227 | return newEventsWithoutEmptyValues 228 | 229 | return newEvents 230 | 231 | 232 | # Set timestamp on event 233 | def getTimestamp(event, delimiter): 234 | 235 | try: 236 | match SPLUNK_TIME_FORMAT: 237 | case "prefix-ISO8601": # For ISO8601 (%Y-%m-%dT%H-%M-%S.%fZ) 238 | 239 | if len(SPLUNK_TIME_PREFIX) > 0: 240 | regexGroupIndex = 2 241 | else: 242 | regexGroupIndex = 0 243 | 244 | iso8601Timestamp = re.search("" + SPLUNK_TIME_PREFIX + r"(.{1,5})?(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{0,10})?Z)", str(event)).group(regexGroupIndex) #fix eventTime 245 | return dateutil.parser.parse(iso8601Timestamp).timestamp() 246 | 247 | case "prefix-epoch":# For prefix epoch formats 248 | 249 | epochTimeString = re.search("" + SPLUNK_TIME_PREFIX + r"(.{1,5})?\d{10,13}", str(event)).group(0) 250 | epochTime = re.search(r"\d{10,13}", str(epochTimeString)).group(0) 251 | if len(epochTime) == 13: 252 | epochTime = float(epochTime) / 1000 253 | 254 | return float(epochTime) 255 | 256 | case "delineated-epoch": # For field-delimited epoch time 257 | 258 | epochTime = float(event.split(delimiter)[int(SPLUNK_TIME_DELINEATED_FIELD)]) 259 | return epochTime 260 | 261 | case "delineated-ISO8601": # For delineated ISO8601 (%Y-%m-%dT%H-%M-%S.%fZ) 262 | 263 | iso8601Timestamp = event.split(delimiter)[int(SPLUNK_TIME_DELINEATED_FIELD)] 264 | return dateutil.parser.parse(iso8601Timestamp).timestamp() 265 | 266 | case "delineated-strftime": # For custom strftime formats 267 | 268 | rawTimeStamp = event.split(delimiter)[int(SPLUNK_TIME_DELINEATED_FIELD)] 269 | return int(datetime.datetime.strptime(rawTimeStamp, SPLUNK_STRFTIME_FORMAT).strftime("%s")) 270 | 271 | except: 272 | # If not standard, set to current time 273 | print("Unable to extract timestamp. Falling back to current time.") 274 | return time.time() 275 | 276 | return time.time() 277 | 278 | 279 | # Buffer and send events to Firehose 280 | def bufferAndSendEventsToFirehose(event, final, objectName, eventBatch): 281 | 282 | # Add current event ot recordBatch 283 | if len(event) > 0: # This will be 0 if it's a final call to clear the buffer 284 | recordBatch.append({"Data": event}) 285 | 286 | # Attempt to send until maxRetriesToFirehose is hit 287 | sendingAttempt = 1 288 | while sendingAttempt <= maxRetriesToFirehose: 289 | 290 | try: 291 | # If there are more than 450 records or 4500000B in the sending queue or this is the final sending and there are records to send, send the event to Splunk and clear the queue 292 | if len(recordBatch) >= 450 or sys.getsizeof(recordBatch) >= 4500000 or (final == True and len(recordBatch) >= 1): 293 | 294 | # Incrmenet eventBatch for logging 295 | if sendingAttempt == 1: 296 | eventBatch[0] += 1 297 | 298 | # Send the event batch 299 | response = firehoseClient.put_record_batch(DeliveryStreamName=firehoseDeliverySreamName, Records=recordBatch) 300 | 301 | # If no messages failed... 302 | if response['FailedPutCount'] == 0: 303 | recordBatch.clear() 304 | return("Sent to Firehose") 305 | # If messages failed... 306 | else: 307 | print("Unable to send file to Firehose. Error message: " + response['RequestResponses'][0]['ErrorMessage']) 308 | 309 | # If the threshold for sending wasn't reached... 310 | else: 311 | if (final == True and len(recordBatch) == 0): 312 | return("Final try, no events buffered") 313 | else: 314 | return("Event buffered") 315 | 316 | # Print exception for debugging 317 | except Exception as e: 318 | print("Unable to send file to Firehose: " + str(e)) 319 | 320 | # Exponential backoff with jitter 321 | sendingAttempt += 1 322 | sleepTime = 2 ** sendingAttempt 323 | jitter = random.uniform(0, sleepTime) 324 | totalSleepTime = sleepTime + jitter 325 | print("Retrying in " + str(round(totalSleepTime, 2)) + " seconds for attempt " + str(sendingAttempt) + " on eventBatch " + str(eventBatch[0]) + " for object " + objectName) 326 | time.sleep(totalSleepTime) 327 | 328 | # Open failure with max retries being reached 329 | return "Max firehose retries reached" 330 | 331 | 332 | # Default Lambda handler 333 | def handler(event, context): 334 | 335 | # Create delineated field break 336 | delimiter = createDelimiter(SPLUNK_EVENT_DELIMITER) 337 | 338 | # Loop through each SQS message 339 | for message in event['Records']: 340 | 341 | # Retrieve bucket name and key from SQS message 342 | objectInfo = retrieveObjectInfo(message) 343 | 344 | # Set eventBatch value, on a per-object basis 345 | eventBatch = [0] 346 | 347 | # If a string was returned instead of a dictionary, print the error and stop this loop 348 | if isinstance(objectInfo, str): 349 | print(objectInfo) 350 | continue 351 | 352 | # Validate file types 353 | isValidFileTypeResult = isValidFileType(objectInfo["key"]) 354 | if not isValidFileTypeResult: 355 | print("Unsupported file type: s3://" + objectInfo["bucket"] + "/" + objectInfo["key"]) 356 | continue 357 | 358 | # Retrieve the S3 object and uncompress it 359 | downloadResult = downloadS3Object(objectInfo["bucket"], objectInfo["key"]) 360 | 361 | # If the file was unable to be downloaded, print the error and stop this loop 362 | if "Unable to download" in downloadResult: 363 | print(downloadResult) 364 | continue 365 | 366 | # Send file info to be uncompressed 367 | uncompressResult = uncompressFile(downloadResult) 368 | 369 | # If the file was unable to be compressed, print the error and stop this loop 370 | if "Unable to uncompress file" in uncompressResult: 371 | print("Unable to uncompress file s3://" + objectInfo["bucket"] + "/" + objectInfo["key"]) 372 | continue 373 | 374 | # Try to read the file contents into memory 375 | try: 376 | with open(uncompressResult, 'r') as f: 377 | events = f.read() 378 | except: 379 | print("Unable to read file contents into memory") 380 | continue 381 | 382 | # Set extension 383 | extension = uncompressResult.split(".")[-1] 384 | 385 | # Split events 386 | splitEvents = eventBreak(events, extension, SPLUNK_IGNORE_FIRST_LINE) 387 | 388 | # Clean up first line of events 389 | if SPLUNK_SOURCETYPE == "aws:billing:cur": 390 | splitEvents = cleanFirstLine(splitEvents) 391 | 392 | # If a string was returned instead of a list, print the error and stop this loop 393 | if isinstance(splitEvents, str): 394 | print("File type unsupported s3://" + objectInfo["bucket"] + "/" + objectInfo["key"]) 395 | continue 396 | 397 | # Transform CSV to JSON 398 | if SPLUNK_CSV_TO_JSON == "true": 399 | splitEvents = csvToJSON(splitEvents) 400 | 401 | # Loop through split events 402 | for splitEvent in splitEvents: 403 | 404 | # Get timestamp 405 | timestamp = getTimestamp(splitEvent, delimiter) 406 | 407 | # Construct event to send to Splunk 408 | splunkEvent = '{ "time": ' + str(timestamp) + ', "host": "' + SPLUNK_HOST + '", "source": "' + SPLUNK_SOURCE + '", "sourcetype": "' + SPLUNK_SOURCETYPE + '", "index": "' + SPLUNK_INDEX + '", "event": ' + json.dumps(splitEvent) + ' }' 409 | 410 | # Buffer and send the events to Firehose 411 | result = bufferAndSendEventsToFirehose(str(splunkEvent), False, objectInfo["key"], eventBatch) 412 | 413 | # Error logging 414 | if result.startswith("Unable to send file to Firehose"): 415 | print(result + " Firehose name: " + firehoseDeliverySreamName + ". File path: s3://" + objectInfo["bucket"] + "/" + objectInfo["key"]) 416 | 417 | # Send the remaining events to Firehose, effectively clearing the buffered events in recordBatch 418 | bufferAndSendEventsToFirehose("", True, objectInfo["key"], eventBatch) 419 | 420 | # Delete the file to clear up space in /tmp to make room for the next one 421 | os.remove(uncompressResult) 422 | 423 | # Logging 424 | print("Processed file s3://" + objectInfo["bucket"] + "/" + objectInfo["key"]) 425 | 426 | -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/route53QueryLogsToS3.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | AWSTemplateFormatVersion: "2010-09-09" 4 | Description: This is a CloudFormation template to configure Route53 DNS Logs for a VPC to be sent to Firehose. 5 | 6 | Parameters: 7 | sourceVPCId: 8 | Description: The VPC ID of the query logging will be enabled for. 9 | Type: AWS::EC2::VPC::Id 10 | 11 | destinationARN: 12 | Description: The ARN of the resource that you want Resolver to send query logs; an Amazon S3 bucket, a CloudWatch Logs log group, or a Kinesis Data Firehose delivery stream. Most likely the S3 ARN created in deploying the eventsInS3ToSplunk CloudFormation Template. 13 | Type: String 14 | 15 | 16 | Resources: 17 | 18 | loggingConfig: 19 | Type: AWS::Route53Resolver::ResolverQueryLoggingConfig 20 | Properties: 21 | DestinationArn: !Ref destinationARN 22 | Name: !Sub "${AWS::AccountId}-${AWS::Region}-queryLoggingConfig-${sourceVPCId}" 23 | 24 | loggingAssociation: 25 | Type: AWS::Route53Resolver::ResolverQueryLoggingConfigAssociation 26 | Properties: 27 | ResolverQueryLogConfigId: !Ref loggingConfig 28 | ResourceId: !Ref sourceVPCId 29 | 30 | 31 | Outputs: 32 | loggingConfigId: 33 | Value: !Ref loggingConfig 34 | loggingAssociationId: 35 | Value: !Ref loggingAssociation -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/sampleElbToS3.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: This is a reference CloudFormation template to show how to create ELBs via CloudFormation that send data to an S3 bucket - https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html 4 | 5 | Parameters: 6 | service: 7 | Type: String 8 | Description: service name 9 | Default: splunk-aws-gdi-toolkit 10 | 11 | stage: 12 | Type: String 13 | Description: Used to distinguish between stages of an environment (dev, test, prod, stage, etc). Only used in AWS resource tagging. 14 | Default: dev 15 | 16 | contact: 17 | Description: Used to identify a contact for the resources created in this stack. Only used in AWS resource tagging. As an example, this could be an email address or username. 18 | Type: String 19 | 20 | s3BucketName: 21 | Description: Destination bucket name that will receive ELB log files. 22 | Type: String 23 | 24 | vpcID: 25 | Description: VPC the security group and ELBs will reside in 26 | Type: AWS::EC2::VPC::Id 27 | 28 | publicSubnetA: 29 | Description: Public subnet in AZ A 30 | Type: AWS::EC2::Subnet::Id 31 | 32 | publicSubnetB: 33 | Description: Public subnet in AZ B 34 | Type: AWS::EC2::Subnet::Id 35 | 36 | publicSubnetC: 37 | Description: Public subnet in AZ C 38 | Type: AWS::EC2::Subnet::Id 39 | 40 | Resources: 41 | 42 | # ELB v2 (ALB / NLB) 43 | elbv2: 44 | Type: AWS::ElasticLoadBalancingV2::LoadBalancer 45 | Properties: 46 | LoadBalancerAttributes: 47 | - Key: access_logs.s3.enabled 48 | Value: true 49 | - Key: access_logs.s3.bucket 50 | Value: !Ref s3BucketName 51 | Scheme: internet-facing 52 | SecurityGroups: 53 | - !Ref loadBalancerSecurityGroup 54 | Subnets: 55 | - !Ref publicSubnetA 56 | - !Ref publicSubnetB 57 | - !Ref publicSubnetC 58 | Tags: 59 | - Key: service 60 | Value: !Ref service 61 | - Key: stage 62 | Value: !Ref stage 63 | - Key: contact 64 | Value: !Ref contact 65 | 66 | # Security Group for load balancers, not required for ELB log ingest 67 | loadBalancerSecurityGroup: 68 | Type: AWS::EC2::SecurityGroup 69 | Properties: 70 | GroupName: splunkAWSGDIToolkitLBSG 71 | GroupDescription: Security group used for sample AWS load balancers. 72 | VpcId: !Ref "vpcID" 73 | SecurityGroupIngress: 74 | - CidrIp: 0.0.0.0/0 75 | Description: "Allow all HTTP traffic in" 76 | FromPort: 80 77 | IpProtocol: tcp 78 | ToPort: 80 79 | Tags: 80 | - Key: service 81 | Value: !Ref service 82 | - Key: stage 83 | Value: !Ref stage 84 | - Key: contact 85 | Value: !Ref contact -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/splunkDashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "visualizations": { 3 | "viz_AcyVeBot": { 4 | "type": "splunk.column", 5 | "showProgressBar": false, 6 | "showLastUpdated": false, 7 | "title": "$sourcetype$ License Usage Over Time (GB)", 8 | "dataSources": { 9 | "primary": "ds_6WBt9iY9" 10 | }, 11 | "options": { 12 | "legendDisplay": "off" 13 | } 14 | }, 15 | "viz_KMF849o7": { 16 | "type": "splunk.singlevalue", 17 | "title": "$sourcetype$ Event License Usage", 18 | "dataSources": { 19 | "primary": "ds_HUEXsOZi" 20 | }, 21 | "options": { 22 | "numberPrecision": 2, 23 | "unit": "GB", 24 | "trendDisplay": "off", 25 | "sparklineDisplay": "off" 26 | } 27 | }, 28 | "viz_OF3a0hX5": { 29 | "type": "splunk.singlevalue", 30 | "title": "$sourcetype$ Event Count", 31 | "dataSources": { 32 | "primary": "ds_LN5KxPiz" 33 | }, 34 | "options": { 35 | "unit": "events", 36 | "trendDisplay": "off", 37 | "sparklineDisplay": "off" 38 | } 39 | }, 40 | "viz_kqPNrs3R": { 41 | "type": "splunk.line", 42 | "showProgressBar": false, 43 | "showLastUpdated": false, 44 | "title": "$sourcetype$ Event Count Over Time", 45 | "dataSources": { 46 | "primary": "ds_7YVGDkt0" 47 | }, 48 | "options": { 49 | "legendDisplay": "off" 50 | } 51 | }, 52 | "viz_eU3T7rCG": { 53 | "type": "splunk.singlevalue", 54 | "title": "AWS Accounts Sending Events", 55 | "dataSources": { 56 | "primary": "ds_noJxIRWy" 57 | } 58 | }, 59 | "viz_nO6QvmkM": { 60 | "type": "splunk.line", 61 | "title": "AWS Accounts Sending $sourcetype$ Events", 62 | "dataSources": { 63 | "primary": "ds_FFly4UzK" 64 | }, 65 | "options": { 66 | "legendDisplay": "off" 67 | } 68 | }, 69 | "viz_NY0AedMk": { 70 | "type": "splunk.line", 71 | "title": "$sourcetype$ Event Latency Over Time", 72 | "showProgressBar": false, 73 | "showLastUpdated": false, 74 | "dataSources": { 75 | "primary": "ds_srz6OcK5" 76 | }, 77 | "options": { 78 | "legendDisplay": "off" 79 | } 80 | }, 81 | "viz_XqNxe7aK": { 82 | "type": "splunk.singlevalue", 83 | "title": "$sourcetype$ Latest Event Latency", 84 | "dataSources": { 85 | "primary": "ds_tCds3VFx" 86 | }, 87 | "options": { 88 | "sparklineDisplay": "off", 89 | "unit": "seconds" 90 | }, 91 | "showProgressBar": false, 92 | "showLastUpdated": false 93 | } 94 | }, 95 | "dataSources": { 96 | "ds_6WBt9iY9": { 97 | "type": "ds.search", 98 | "options": { 99 | "query": "index=_internal source=*license_usage* idx=$eventIndex$ st=$sourcetype$ NOT source=*license_usage_summary.log*\n| timechart sum(b) as ingestedDataGB\n| eval ingestedDataGB = ingestedDataGB/1024/1024/1024", 100 | "queryParameters": { 101 | "earliest": "$global_time.earliest$", 102 | "latest": "$global_time.latest$" 103 | } 104 | }, 105 | "name": "licenseUsageBase" 106 | }, 107 | "ds_HUEXsOZi": { 108 | "type": "ds.chain", 109 | "options": { 110 | "extend": "ds_6WBt9iY9", 111 | "query": "| stats sum(ingestedDataGB) as ingestedDataGB" 112 | }, 113 | "name": "licenseUsageTotal" 114 | }, 115 | "ds_7YVGDkt0": { 116 | "type": "ds.search", 117 | "options": { 118 | "query": "index=_internal series=$sourcetype$ sourcetype=splunkd source=*metrics.log*\n| timechart sum(ev) as eventCount" 119 | }, 120 | "name": "eventCountBase" 121 | }, 122 | "ds_LN5KxPiz": { 123 | "type": "ds.chain", 124 | "options": { 125 | "extend": "ds_7YVGDkt0", 126 | "query": "| stats sum(eventCount) as eventCount" 127 | }, 128 | "name": "eventCountTotal" 129 | }, 130 | "ds_FFly4UzK": { 131 | "type": "ds.search", 132 | "options": { 133 | "query": "index=$eventIndex$ sourcetype=$sourcetype$\n| timechart dc(vendor_account) as accountCount" 134 | }, 135 | "name": "accountCountTimechart" 136 | }, 137 | "ds_noJxIRWy": { 138 | "type": "ds.search", 139 | "options": { 140 | "query": "index=$eventIndex$ sourcetype=$sourcetype$\n| stats dc(vendor_account) as accountCount" 141 | }, 142 | "name": "accountCountTotal" 143 | }, 144 | "ds_srz6OcK5": { 145 | "type": "ds.search", 146 | "options": { 147 | "query": "index=$eventIndex$ sourcetype=$sourcetype$ \n| eval eventLatency = abs(_indextime - _time)\n| timechart avg(eventLatency) as eventLatency" 148 | }, 149 | "name": "eventLatencyBase" 150 | }, 151 | "ds_tCds3VFx": { 152 | "type": "ds.search", 153 | "options": { 154 | "query": "index=$eventIndex$ sourcetype=$sourcetype$ | head 1\n| eval eventLatency = abs(_indextime - _time)\n| table eventLatency" 155 | }, 156 | "name": "eventLatencyLatest" 157 | }, 158 | "ds_N5KOizZ9": { 159 | "type": "ds.search", 160 | "options": { 161 | "query": "| tstats count where index=* by index", 162 | "queryParameters": { 163 | "earliest": "$global_time.earliest$", 164 | "latest": "$global_time.latest$" 165 | } 166 | }, 167 | "name": "indexList" 168 | }, 169 | "ds_f9m7vKr7": { 170 | "type": "ds.search", 171 | "options": { 172 | "query": "| tstats count where index=$eventIndex$ by sourcetype" 173 | }, 174 | "name": "sourcetypeList" 175 | } 176 | }, 177 | "defaults": { 178 | "dataSources": { 179 | "ds.search": { 180 | "options": { 181 | "queryParameters": { 182 | "latest": "$global_time.latest$", 183 | "earliest": "$global_time.earliest$" 184 | } 185 | } 186 | } 187 | } 188 | }, 189 | "inputs": { 190 | "input_global_trp": { 191 | "type": "input.timerange", 192 | "options": { 193 | "token": "global_time", 194 | "defaultValue": "-24h@h,now" 195 | }, 196 | "title": "Global Time Range" 197 | }, 198 | "input_9rNFa8NM": { 199 | "options": { 200 | "items": [ 201 | { 202 | "label": "main", 203 | "value": "main" 204 | } 205 | ], 206 | "defaultValue": "main", 207 | "token": "eventIndex" 208 | }, 209 | "title": "Event Index", 210 | "type": "input.dropdown", 211 | "dataSources": { 212 | "primary": "ds_N5KOizZ9" 213 | } 214 | }, 215 | "input_C3SY1WEW": { 216 | "options": { 217 | "items": [ 218 | { 219 | "label": "All", 220 | "value": "*" 221 | } 222 | ], 223 | "defaultValue": "*", 224 | "token": "sourcetype" 225 | }, 226 | "title": "Event Sourcetype", 227 | "type": "input.dropdown", 228 | "dataSources": { 229 | "primary": "ds_f9m7vKr7" 230 | } 231 | } 232 | }, 233 | "layout": { 234 | "type": "grid", 235 | "options": {}, 236 | "structure": [ 237 | { 238 | "item": "viz_eU3T7rCG", 239 | "type": "block", 240 | "position": { 241 | "x": 600, 242 | "y": 0, 243 | "w": 300, 244 | "h": 100 245 | } 246 | }, 247 | { 248 | "item": "viz_XqNxe7aK", 249 | "type": "block", 250 | "position": { 251 | "x": 900, 252 | "y": 0, 253 | "w": 300, 254 | "h": 100 255 | } 256 | }, 257 | { 258 | "item": "viz_KMF849o7", 259 | "type": "block", 260 | "position": { 261 | "x": 0, 262 | "y": 0, 263 | "w": 300, 264 | "h": 100 265 | } 266 | }, 267 | { 268 | "item": "viz_AcyVeBot", 269 | "type": "block", 270 | "position": { 271 | "x": 0, 272 | "y": 100, 273 | "w": 1200, 274 | "h": 300 275 | } 276 | }, 277 | { 278 | "item": "viz_kqPNrs3R", 279 | "type": "block", 280 | "position": { 281 | "x": 0, 282 | "y": 400, 283 | "w": 1200, 284 | "h": 300 285 | } 286 | }, 287 | { 288 | "item": "viz_nO6QvmkM", 289 | "type": "block", 290 | "position": { 291 | "x": 0, 292 | "y": 700, 293 | "w": 1200, 294 | "h": 300 295 | } 296 | }, 297 | { 298 | "item": "viz_NY0AedMk", 299 | "type": "block", 300 | "position": { 301 | "x": 0, 302 | "y": 1000, 303 | "w": 1200, 304 | "h": 300 305 | } 306 | }, 307 | { 308 | "item": "viz_OF3a0hX5", 309 | "type": "block", 310 | "position": { 311 | "x": 300, 312 | "y": 0, 313 | "w": 300, 314 | "h": 100 315 | } 316 | } 317 | ], 318 | "globalInputs": [ 319 | "input_global_trp", 320 | "input_9rNFa8NM", 321 | "input_C3SY1WEW" 322 | ] 323 | }, 324 | "description": "", 325 | "title": "Splunk AWS GDI Toolkit - S3-SQS-Lambda-Firehose" 326 | } -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/ta_route53/README: -------------------------------------------------------------------------------- 1 | App for props/transforms with AWS Route53 query logs -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/ta_route53/default/app.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Splunk app configuration file 3 | # 4 | 5 | [install] 6 | is_configured = 0 7 | 8 | [ui] 9 | is_visible = 0 10 | label = TA-route53 11 | 12 | [launcher] 13 | author = Paul Reeves 14 | description = App for props/transforms with AWS Route53 query logs 15 | version = 1.0.0 16 | 17 | [package] 18 | id = ta_route53 19 | -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/ta_route53/default/props.conf: -------------------------------------------------------------------------------- 1 | [aws:route53] 2 | DATETIME_CONFIG = CURRENT 3 | LINE_BREAKER = ([\r\n]+) 4 | NO_BINARY_CHECK = true 5 | SHOULD_LINEMERGE = false 6 | category = Network & Security 7 | description = Route 53 Query Logs 8 | pulldown_type = true 9 | AUTO_KV_JSON = true 10 | KV_MODE = JSON 11 | 12 | # Field aliases 13 | FIELDALIAS-query = query_name AS query 14 | FIELDALIAS-src = srcaddr AS src 15 | FIELDALIAS-src_ip = srcaddr AS src_ip 16 | FIELDALIAS-src_port = srcport AS src_port 17 | 18 | # Evals 19 | # EVAL-answer_count 20 | EVAL-message_type = "Response" 21 | EVAL-vendor_product = "AWS Route 53" 22 | EVAL-answer_count = mvcount(answer) 23 | 24 | # Reports 25 | REPORT-aws_route53_answer = aws_route53_answer 26 | -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/ta_route53/default/transforms.conf: -------------------------------------------------------------------------------- 1 | [aws_route53_answer] 2 | REGEX = \"Rdata\":\"([\d.]+)\",\"Type\":\"([A-Za-z0-9]+) 3 | FORMAT = answer::$1 record_type::$2 4 | MV_ADD = true -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/ta_route53/metadata/default.meta: -------------------------------------------------------------------------------- 1 | [] 2 | access = read : [ * ], write : [ sc_admin, power ] 3 | export = system 4 | -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/ta_route53/static/appIcon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-aws-gdi-toolkit/38d8b3047130ed56468f8153c2db7b75c89a0db4/S3-SQS-Lambda-Firehose-Resources/ta_route53/static/appIcon.png -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/ta_route53/static/appIconAlt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-aws-gdi-toolkit/38d8b3047130ed56468f8153c2db7b75c89a0db4/S3-SQS-Lambda-Firehose-Resources/ta_route53/static/appIconAlt.png -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/ta_route53/static/appIconAlt_2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-aws-gdi-toolkit/38d8b3047130ed56468f8153c2db7b75c89a0db4/S3-SQS-Lambda-Firehose-Resources/ta_route53/static/appIconAlt_2x.png -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/ta_route53/static/appIcon_2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-aws-gdi-toolkit/38d8b3047130ed56468f8153c2db7b75c89a0db4/S3-SQS-Lambda-Firehose-Resources/ta_route53/static/appIcon_2x.png -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/test-fixtures/sample-cloudtrail.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-aws-gdi-toolkit/38d8b3047130ed56468f8153c2db7b75c89a0db4/S3-SQS-Lambda-Firehose-Resources/test-fixtures/sample-cloudtrail.json.gz -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/test-fixtures/sample-route53Resolver.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-aws-gdi-toolkit/38d8b3047130ed56468f8153c2db7b75c89a0db4/S3-SQS-Lambda-Firehose-Resources/test-fixtures/sample-route53Resolver.log.gz -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/test-fixtures/sample-s3ServerAccess: -------------------------------------------------------------------------------- 1 | d2b1828e428fd0fc94f09e0df9e09766e034d88de30f07560182746128287467d42d splunk-aws-gdi-toolkit-us-west-2-public-bucket [08/Nov/2022:14:11:33 +0000] 104.129.199.79 arn:aws:sts::841154226728:assumed-role/AWSReservedSSO_SPLKAdministratorAccess_8c0000887680558c/user@splunk.com 1WEG6G1N8100F777 REST.GET.BUCKETVERSIONS - "GET /?versions&max-keys=1&encoding-type=url HTTP/1.1" 200 - 892 - 106 105 "-" "S3Console/0.4, aws-internal/3 aws-sdk-java/1.11.1030 Linux/5.10.144-111.639.amzn2int.x86_64 OpenJDK_64-Bit_Server_VM/25.352-b08 java/1.8.0_352 vendor/Oracle_Corporation cfg/retry-mode/standard" - Pu/icXEeVP6yeYd511MgPL0mGNdJ56728BTlPFSa++5lAcJh5MVtu+M4813cIUk4sl9lWkNzlQ= SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader splunk-aws-gdi-toolkit-us-west-2-public-bucket.s3.us-west-2.amazonaws.com TLSv1.2 - - 2 | d2b1828e428fd0fc94f09e0df9e09766e034d88de30f07560182746128287467d42d splunk-aws-gdi-toolkit-us-west-2-public-bucket [08/Nov/2022:14:11:33 +0000] 104.129.199.79 arn:aws:sts::841154226728:assumed-role/AWSReservedSSO_SPLKAdministratorAccess_8c0000887680558c/user@splunk.com 1WEJTAQ11WXV63QN REST.GET.WEBSITE - "GET /?website HTTP/1.1" 404 NoSuchWebsiteConfiguration 373 - 15 - "-" "S3Console/0.4, aws-internal/3 aws-sdk-java/1.11.1030 Linux/5.10.144-111.639.amzn2int.x86_64 OpenJDK_64-Bit_Server_VM/25.352-b08 java/1.8.0_352 vendor/Oracle_Corporation cfg/retry-mode/standard" - U7AN9+/XFSYBPa0qOEe+0ukJXDcqCJ2222tv1f3EeKdU8iot9gmpMQFhO3PgIsah8Ne3Kf8E1Vk= SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader splunk-aws-gdi-toolkit-us-west-2-public-bucket.s3.us-west-2.amazonaws.com TLSv1.2 - - 3 | d2b1828e428fd0fc94f09e0df9e09766e034d88de30f07560182746128287467d42d splunk-aws-gdi-toolkit-us-west-2-public-bucket [08/Nov/2022:14:11:33 +0000] 104.129.199.79 arn:aws:sts::841154226728:assumed-role/AWSReservedSSO_SPLKAdministratorAccess_8c0000887680558c/user@splunk.com 1WEZPA54227YG1MA REST.GET.BUCKETPOLICY - "GET /?policy HTTP/1.1" 404 NoSuchBucketPolicy 339 - 16 - "-" "S3Console/0.4, aws-internal/3 aws-sdk-java/1.11.1030 Linux/5.10.144-111.639.amzn2int.x86_64 OpenJDK_64-Bit_Server_VM/25.352-b08 java/1.8.0_352 vendor/Oracle_Corporation cfg/retry-mode/standard" - zKJYLtzzrroNp/rzs9y195CTxx5aps4mfZ3uQf3WTsiLC111um4G0bI5LSbDHCW0004jUmcSwdw= SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader splunk-aws-gdi-toolkit-us-west-2-public-bucket.s3.us-west-2.amazonaws.com TLSv1.2 - - 4 | -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/test-fixtures/sample-vpcflow.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-aws-gdi-toolkit/38d8b3047130ed56468f8153c2db7b75c89a0db4/S3-SQS-Lambda-Firehose-Resources/test-fixtures/sample-vpcflow.log.gz -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/test-fixtures/testFile1.log: -------------------------------------------------------------------------------- 1 | 1-0b48139ba00b9b7bb 2 | 1-0b48139ba00b9b7ba 3 | 1-0b48139ba00b9b7bc -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/test-fixtures/testFile1.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-aws-gdi-toolkit/38d8b3047130ed56468f8153c2db7b75c89a0db4/S3-SQS-Lambda-Firehose-Resources/test-fixtures/testFile1.log.gz -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/test-fixtures/testdir/123/testFile2.log: -------------------------------------------------------------------------------- 1 | 2-0b48139ba00b9b7bb 2 | 2-0b48139ba00b9b7ba 3 | 2-0b48139ba00b9b7bc -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/test-fixtures/testdir/123/testFile2.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-aws-gdi-toolkit/38d8b3047130ed56468f8153c2db7b75c89a0db4/S3-SQS-Lambda-Firehose-Resources/test-fixtures/testdir/123/testFile2.log.gz -------------------------------------------------------------------------------- /S3-SQS-Lambda-Firehose-Resources/vpcFlowLogToS3.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: This is a CloudFormation template to create VPC Flow Logging infrastructure to S3. This should be deployed once for each VPC, ENI, or subnet you want to capture VPC Flow Logs from. 4 | 5 | 6 | Parameters: 7 | service: 8 | Type: String 9 | Description: service name 10 | Default: splunk-aws-gdi-toolkit 11 | 12 | stage: 13 | Type: String 14 | Description: Used to distinguish between stages of an environment (dev, test, prod, stage, etc). Only used in AWS resource tagging. 15 | Default: dev 16 | 17 | contact: 18 | Description: Used to identify a contact for the resources created in this stack. Only used in AWS resource tagging. As an example, this could be an email address or username. 19 | Type: String 20 | 21 | vpcFlowLogS3BucketName: 22 | Description: Destination bucket name that will receive VPC Flow Log events. 23 | Type: String 24 | 25 | vpcFlowLogResourceId: 26 | Description: The ID of the subnet, network interface, or VPC for which you want to create a flow log. Taken straight from https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-flowlog.html 27 | Type: String 28 | 29 | vpcFlowLogResourceType: 30 | Description: The type of resource for which to create the flow log. Select NetworkInterface if you specified an ENI for vpcFlowLogResourceId, "Subnet" if you specified a subnet ID for vpcFlowLogResourceId, or "VPC" if you specified a VPC ID for vpcFlowLogResourceId" 31 | Type: String 32 | Default: VPC 33 | AllowedValues: 34 | - NetworkInterface 35 | - Subnet 36 | - VPC 37 | 38 | vpcFlowLogTrafficType: 39 | Description: Type of traffic you want to log. 40 | Type: String 41 | Default: ALL 42 | AllowedValues: 43 | - ACCEPT 44 | - REJECT 45 | - ALL 46 | 47 | 48 | Resources: 49 | vpcFlowLog: 50 | Type: AWS::EC2::FlowLog 51 | Properties: 52 | LogDestination: !Sub "arn:aws:s3:::${vpcFlowLogS3BucketName}" 53 | LogDestinationType: s3 54 | ResourceId: !Ref vpcFlowLogResourceId 55 | ResourceType: !Ref vpcFlowLogResourceType 56 | TrafficType: !Ref vpcFlowLogTrafficType 57 | Tags: 58 | - Key: service 59 | Value: !Ref service 60 | - Key: stage 61 | Value: !Ref stage 62 | - Key: contact 63 | Value: !Ref contact 64 | 65 | 66 | Outputs: 67 | vpcFlowLogId: 68 | Value: !Ref vpcFlowLog -------------------------------------------------------------------------------- /Single Account CloudFormation/cloudTrail.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: This is a CloudFormation template to create CloudTrail logging infrastructure to S3 and CloudWatch Logs - https://aws.amazon.com/cloudtrail/ 4 | 5 | 6 | Parameters: 7 | service: 8 | Type: String 9 | Description: service name 10 | Default: splunk-aws-gdi-toolkit 11 | 12 | stage: 13 | Type: String 14 | Description: Used to distinguish between stages of an environment 15 | Default: dev 16 | 17 | contact: 18 | Description: Used to identify a contact for the resources created in this stack. As an example, this could be an email address or username. 19 | Type: String 20 | 21 | cloudTrailLogFileValidation: 22 | Description: Used to enable or disable CloudTrail File Validation - https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-log-file-validation-intro.html . 23 | Default: true 24 | Type: String 25 | AllowedValues: 26 | - true 27 | - false 28 | 29 | cloudTrailIncludeGlobalServiceEvents: 30 | Description: Used to enable or disable logging of CloudTrail global service events - https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-concepts.html#cloudtrail-concepts-global-service-events . 31 | Default: true 32 | Type: String 33 | AllowedValues: 34 | - true 35 | - false 36 | 37 | cloudTrailMultiRegionLogging: 38 | Description: Used to enable or disable logging of CloudTrail logs from all regions. 39 | Default: true 40 | Type: String 41 | AllowedValues: 42 | - true 43 | - false 44 | 45 | cloudTrailS3FileExpirationInDays: 46 | Description: How many days to keep the CloudTrail files in S3. 47 | Default: 366 48 | Type: String 49 | 50 | 51 | Resources: 52 | cloudTrail: 53 | Type: AWS::CloudTrail::Trail 54 | Properties: 55 | CloudWatchLogsLogGroupArn: !GetAtt cloudTrailLogGroup.Arn 56 | CloudWatchLogsRoleArn: !GetAtt cloudTrailRole.Arn 57 | EnableLogFileValidation: !Ref cloudTrailLogFileValidation 58 | IncludeGlobalServiceEvents: !Ref cloudTrailIncludeGlobalServiceEvents 59 | IsLogging: true 60 | IsMultiRegionTrail: !Ref cloudTrailMultiRegionLogging 61 | S3BucketName: !Ref cloudTrailS3Bucket 62 | Tags: 63 | - Key: service 64 | Value: !Ref service 65 | - Key: stage 66 | Value: !Ref stage 67 | - Key: contact 68 | Value: !Ref contact 69 | TrailName: !Sub "${service}-cloudtrail" 70 | 71 | cloudTrailPolicy: 72 | Type: AWS::IAM::ManagedPolicy 73 | Properties: 74 | PolicyDocument: 75 | Version: '2012-10-17' 76 | Statement: 77 | - Effect: Allow 78 | Action: 79 | - logs:Describe* 80 | - logs:CreateLogStream 81 | - logs:PutLogEvents 82 | Resource: !GetAtt cloudTrailLogGroup.Arn 83 | - Effect: Allow 84 | Action: 85 | - s3:PutObject 86 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-cloudtrail-bucket/*" 87 | - Effect: Allow 88 | Action: 89 | - s3:GetBucketAcl 90 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-cloudtrail-bucket" 91 | ManagedPolicyName: !Sub "${service}-cloudtrail-iam-policy" 92 | 93 | cloudTrailRole: 94 | Type: AWS::IAM::Role 95 | Properties: 96 | AssumeRolePolicyDocument: 97 | Version: '2012-10-17' 98 | Statement: 99 | - Effect: "Allow" 100 | Principal: 101 | Service: "cloudtrail.amazonaws.com" 102 | Action: 103 | - sts:AssumeRole 104 | ManagedPolicyArns: 105 | - !Ref cloudTrailPolicy 106 | RoleName: !Sub "${service}-cloudtrail-iam-role" 107 | Tags: 108 | - Key: service 109 | Value: !Ref service 110 | - Key: stage 111 | Value: !Ref stage 112 | - Key: contact 113 | Value: !Ref contact 114 | 115 | cloudTrailS3Bucket: 116 | Type: AWS::S3::Bucket 117 | Properties: 118 | AccessControl: Private 119 | BucketEncryption: 120 | ServerSideEncryptionConfiguration: 121 | - ServerSideEncryptionByDefault: 122 | SSEAlgorithm: AES256 123 | BucketName: !Sub "${AWS::AccountId}-cloudtrail-bucket" 124 | LifecycleConfiguration: 125 | Rules: 126 | - Id: !Sub "${AWS::AccountId}-cloudtrail-bucket-cleanupPolicy" 127 | AbortIncompleteMultipartUpload: 128 | DaysAfterInitiation: 1 129 | Status: Enabled 130 | - Id: !Sub "${service}-infralog-bucket-expiration" 131 | ExpirationInDays: !Ref cloudTrailS3FileExpirationInDays 132 | Status: Enabled 133 | PublicAccessBlockConfiguration: 134 | BlockPublicAcls: true 135 | BlockPublicPolicy: true 136 | IgnorePublicAcls: true 137 | RestrictPublicBuckets: true 138 | Tags: 139 | - Key: service 140 | Value: !Ref service 141 | - Key: stage 142 | Value: !Ref stage 143 | - Key: contact 144 | Value: !Ref contact 145 | 146 | cloudTrailS3BucketPolicy: 147 | Type: AWS::S3::BucketPolicy 148 | Properties: 149 | Bucket: !Ref cloudTrailS3Bucket 150 | PolicyDocument: 151 | Statement: 152 | - Effect: Allow 153 | Action: 154 | - s3:GetBucketAcl 155 | Principal: 156 | Service: cloudtrail.amazonaws.com 157 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-cloudtrail-bucket" 158 | - Effect: Allow 159 | Action: 160 | - s3:PutObject 161 | Principal: 162 | Service: cloudtrail.amazonaws.com 163 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-cloudtrail-bucket/AWSLogs/${AWS::AccountId}/*" 164 | 165 | cloudTrailLogGroup: 166 | Type: AWS::Logs::LogGroup 167 | Properties: 168 | LogGroupName: !Sub "/${service}-cloudtrail/" 169 | RetentionInDays: 7 170 | 171 | 172 | Outputs: 173 | cloudTrailArn: 174 | Value: !GetAtt cloudTrail.Arn 175 | cloudTrailRoleArn: 176 | Value: !GetAtt cloudTrailRole.Arn 177 | cloudTrailS3BucketArn: 178 | Value: !GetAtt cloudTrailS3Bucket.Arn 179 | cloudTrailLogGroupArn: 180 | Value: !GetAtt cloudTrailLogGroup.Arn -------------------------------------------------------------------------------- /Single Account CloudFormation/guardDuty.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: This is a CloudFormation template to enable GuardDuty - https://aws.amazon.com/guardduty/ 4 | 5 | 6 | Parameters: 7 | service: 8 | Type: String 9 | Description: service name 10 | Default: splunk-aws-gdi-toolkit 11 | 12 | stage: 13 | Type: String 14 | Description: Used to distinguish between stages of an environment 15 | Default: dev 16 | 17 | contact: 18 | Description: Used to identify a contact for the resources created in this stack. As an example, this could be an email address or username. 19 | Type: String 20 | 21 | Resources: 22 | guardDutyDetector: 23 | Type: AWS::GuardDuty::Detector 24 | Properties: 25 | Enable: true 26 | FindingPublishingFrequency: FIFTEEN_MINUTES 27 | Tags: 28 | - Key: service 29 | Value: !Ref service 30 | - Key: stage 31 | Value: !Ref stage 32 | - Key: contact 33 | Value: !Ref contact 34 | 35 | 36 | Outputs: 37 | guardDutyDetector: 38 | Value: !Ref guardDutyDetector -------------------------------------------------------------------------------- /Single Account CloudFormation/iamAccessAnalyzer.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: This is a CloudFormation template to enable IAM Access Analyer - https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html 4 | 5 | 6 | Parameters: 7 | service: 8 | Type: String 9 | Description: service name 10 | Default: splunk-aws-gdi-toolkit 11 | 12 | stage: 13 | Type: String 14 | Description: Used to distinguish between stages of an environment 15 | Default: dev 16 | 17 | contact: 18 | Description: Used to identify a contact for the resources created in this stack. As an example, this could be an email address or username. 19 | Type: String 20 | 21 | 22 | Resources: 23 | iamAccessAnalyzer: 24 | Type: AWS::AccessAnalyzer::Analyzer 25 | Properties: 26 | AnalyzerName: !Sub "${service}-iam-access-analyer" 27 | Tags: 28 | - Key: service 29 | Value: !Ref service 30 | - Key: stage 31 | Value: !Ref stage 32 | - Key: contact 33 | Value: !Ref contact 34 | Type: ACCOUNT 35 | 36 | 37 | Outputs: 38 | iamAccessAnalyzerArn: 39 | Value: !Ref iamAccessAnalyzer -------------------------------------------------------------------------------- /Single Account CloudFormation/scdm-s3-pull.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: This is a CloudFormation template to create AWS reosurces to work with Splunk Cloud Data Manager's S3-based pull connector. 4 | 5 | 6 | Parameters: 7 | 8 | service: 9 | Type: String 10 | Description: Service name used in tagging AWS resources. 11 | Default: splunk-aws-gdi-toolkit 12 | 13 | stage: 14 | Type: String 15 | Description: Used to distinguish between stages of an environment (dev, test, prod, stage, etc). Only used in AWS resource tagging. 16 | Default: dev 17 | 18 | contact: 19 | Description: Used to identify a contact for the resources created in this stack. Only used in AWS resource tagging. As an example, this could be an email address or username. 20 | Type: String 21 | Default: "" 22 | 23 | logType: 24 | Type: String 25 | Description: Log type, used in AWS resource names and for setting bucket policies 26 | AllowedValues: 27 | - cloudtrail 28 | - s3accesslogs 29 | - elbaccesslogs 30 | - cloudfrontaccesslogs 31 | 32 | existingS3BucketName: 33 | Type: String 34 | Description: Name of the existng S3 bucket you want to ingest data from. Leave this blank to create a new S3 bucket. If you specify the name of an existing S3 bucket, you'll need to create the S3 > SQS notification settings. 35 | Default: "" 36 | 37 | s3ObjectExpirationInDays: 38 | Description: How many days to keep the files in S3 before they're deleted. This only applies to new buckets and not existing buckets. 39 | Default: 90 40 | Type: String 41 | 42 | sqsQueueVisibilityTimeoutInSecond: 43 | Type: Number 44 | Description: How long to let SQS messages be taken by the Lambda function before they become available to be processed again. Must be more than lambdaProcessorTimeout. 45 | Default: 630 46 | MinValue: 0 47 | MaxValue: 43200 48 | 49 | 50 | Conditions: 51 | createNewS3Bucket: !Equals 52 | - !Ref existingS3BucketName 53 | - "" 54 | useExistingS3Bucket: !Not 55 | - !Equals 56 | - !Ref existingS3BucketName 57 | - "" 58 | cloudTrail: !Equals 59 | - !Ref logType 60 | - "cloudtrail" 61 | cloudTrailNewBucket: !And 62 | - !Condition cloudTrail 63 | - !Condition createNewS3Bucket 64 | s3AccessLogs: !Equals 65 | - !Ref logType 66 | - "s3accesslogs" 67 | s3AccessLogsNewBucket: !And 68 | - !Condition s3AccessLogs 69 | - !Condition createNewS3Bucket 70 | elbAccessLogs: !Equals 71 | - !Ref logType 72 | - "elbaccesslogs" 73 | elbAccessLogssNewBucket: !And 74 | - !Condition elbAccessLogs 75 | - !Condition createNewS3Bucket 76 | 77 | 78 | Mappings: 79 | RegionMapping: 80 | us-east-1: 81 | elbAccountIDPrincipal: arn:aws:iam::127311923021:root 82 | us-east-2: 83 | elbAccountIDPrincipal: arn:aws:iam::033677994240:root 84 | us-west-1: 85 | elbAccountIDPrincipal: arn:aws:iam::027434742980:root 86 | us-west-2: 87 | elbAccountIDPrincipal: arn:aws:iam::797873946194:root 88 | af-south-1: 89 | elbAccountIDPrincipal: arn:aws:iam::098369216593:root 90 | ca-central-1: 91 | elbAccountIDPrincipal: arn:aws:iam::985666609251:root 92 | eu-central-1: 93 | elbAccountIDPrincipal: arn:aws:iam::054676820928:root 94 | eu-west-1: 95 | elbAccountIDPrincipal: arn:aws:iam::156460612806:root 96 | eu-west-2: 97 | elbAccountIDPrincipal: arn:aws:iam::652711504416:root 98 | eu-south-1: 99 | elbAccountIDPrincipal: arn:aws:iam::635631232127:root 100 | eu-west-3: 101 | elbAccountIDPrincipal: arn:aws:iam::009996457667:root 102 | eu-north-1: 103 | elbAccountIDPrincipal: arn:aws:iam::897822967062:root 104 | ap-east-1: 105 | elbAccountIDPrincipal: arn:aws:iam::754344448648:root 106 | ap-northeast-1: 107 | elbAccountIDPrincipal: arn:aws:iam::582318560864:root 108 | ap-northeast-2: 109 | elbAccountIDPrincipal: arn:aws:iam::600734575887:root 110 | ap-northeast-3: 111 | elbAccountIDPrincipal: arn:aws:iam::383597477331:root 112 | ap-southeast-1: 113 | elbAccountIDPrincipal: arn:aws:iam::114774131450:root 114 | ap-southeast-2: 115 | elbAccountIDPrincipal: arn:aws:iam::783225319266:root 116 | ap-southeast-3: 117 | elbAccountIDPrincipal: arn:aws:iam::589379963580:root 118 | ap-south-1: 119 | elbAccountIDPrincipal: arn:aws:iam::718504428378:root 120 | me-south-1: 121 | elbAccountIDPrincipal: arn:aws:iam::076674570225:root 122 | sa-east-1: 123 | elbAccountIDPrincipal: arn:aws:iam::507241528517:root 124 | us-gov-west-1: 125 | elbAccountIDPrincipal: arn:aws:iam::048591011584:root 126 | us-gov-east-1: 127 | elbAccountIDPrincipal: arn:aws:iam::190560391635:root 128 | cn-north-1: 129 | elbAccountIDPrincipal: arn:aws:iam::638102146993:root 130 | cn-northwest-1: 131 | elbAccountIDPrincipal: arn:aws:iam::037604701340:root 132 | 133 | 134 | Resources: 135 | 136 | # S3 resources 137 | s3Bucket: 138 | Type: AWS::S3::Bucket 139 | Condition: createNewS3Bucket 140 | Properties: 141 | AccessControl: Private 142 | BucketEncryption: 143 | ServerSideEncryptionConfiguration: 144 | - ServerSideEncryptionByDefault: 145 | SSEAlgorithm: AES256 146 | BucketName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}" 147 | LifecycleConfiguration: 148 | Rules: 149 | - Id: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-cleanup" 150 | AbortIncompleteMultipartUpload: 151 | DaysAfterInitiation: 1 152 | Status: Enabled 153 | - Id: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-expiration" 154 | ExpirationInDays: !Ref s3ObjectExpirationInDays 155 | Status: Enabled 156 | NotificationConfiguration: 157 | QueueConfigurations: 158 | - Event: "s3:ObjectCreated:*" 159 | Queue: !GetAtt s3BucketNotificationSQSQueue.Arn 160 | PublicAccessBlockConfiguration: 161 | BlockPublicAcls: true 162 | BlockPublicPolicy: true 163 | IgnorePublicAcls: true 164 | RestrictPublicBuckets: true 165 | Tags: 166 | - Key: service 167 | Value: !Ref service 168 | - Key: stage 169 | Value: !Ref stage 170 | - Key: contact 171 | Value: !Ref contact 172 | - Key: logType 173 | Value: !Ref logType 174 | 175 | s3BucketPolicyCloudTrailLogs: 176 | Type: AWS::S3::BucketPolicy 177 | Condition: cloudTrailNewBucket 178 | Properties: 179 | Bucket: !Ref s3Bucket 180 | PolicyDocument: 181 | Statement: 182 | - Effect: Allow 183 | Action: 184 | - s3:GetBucketAcl 185 | Principal: 186 | Service: cloudtrail.amazonaws.com 187 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-${logType}" 188 | - Effect: Allow 189 | Action: 190 | - s3:PutObject 191 | Principal: 192 | Service: cloudtrail.amazonaws.com 193 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-${logType}/*" 194 | 195 | s3BucketPolicyELBLogs: 196 | Type: AWS::S3::BucketPolicy 197 | Condition: elbAccessLogssNewBucket 198 | Properties: 199 | Bucket: !Ref s3Bucket 200 | PolicyDocument: 201 | Statement: 202 | - Effect: Allow 203 | Action: 204 | - s3:GetBucketAcl 205 | Principal: 206 | Service: logdelivery.elb.amazonaws.com 207 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-${logType}" 208 | - Effect: Allow 209 | Action: 210 | - s3:PutObject 211 | Principal: 212 | AWS: !FindInMap [RegionMapping, !Ref AWS::Region, elbAccountIDPrincipal] 213 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-${logType}/*" 214 | Condition: 215 | StringEquals: 216 | s3:x-amz-acl: bucket-owner-full-control 217 | 218 | s3BucketPolicyS3ServerAccessLogs: 219 | Type: AWS::S3::BucketPolicy 220 | Condition: s3AccessLogsNewBucket 221 | Properties: 222 | Bucket: !Ref s3Bucket 223 | PolicyDocument: 224 | Statement: 225 | - Sid: "S3ServerAccessLogsPolicy" 226 | Effect: Allow 227 | Action: 228 | - s3:PutObject 229 | Principal: 230 | Service: logging.s3.amazonaws.com 231 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-${logType}/*" 232 | Condition: 233 | StringEquals: 234 | aws:SourceAccount: !Ref AWS::AccountId 235 | 236 | # SQS resources 237 | s3BucketNotificationSQSQueue: 238 | Type: AWS::SQS::Queue 239 | Properties: 240 | QueueName: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-sqs-queue" 241 | Tags: 242 | - Key: service 243 | Value: !Ref service 244 | - Key: stage 245 | Value: !Ref stage 246 | - Key: contact 247 | Value: !Ref contact 248 | - Key: logType 249 | Value: !Ref logType 250 | VisibilityTimeout: !Ref sqsQueueVisibilityTimeoutInSecond 251 | 252 | s3BucketNotificationSQSQueuePolicy: 253 | Type: AWS::SQS::QueuePolicy 254 | Properties: 255 | PolicyDocument: 256 | Version: 2012-10-17 257 | Id: !Sub "${AWS::AccountId}-${AWS::Region}-${logType}-sqs-queuePolicy" 258 | Statement: 259 | - 260 | Sid: Send messages to SQS 261 | Effect: Allow 262 | Principal: 263 | AWS: "*" 264 | Action: 265 | - "SQS:SendMessage" 266 | Resource: "*" 267 | Condition: 268 | ArnLike: 269 | "aws:SourceARN": !If [useExistingS3Bucket, !Sub "arn:aws:s3:::${existingS3BucketName}", !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-${logType}"] 270 | Queues: 271 | - !Ref "s3BucketNotificationSQSQueue" 272 | 273 | 274 | Outputs: 275 | s3BucketArn: 276 | Condition: createNewS3Bucket 277 | Value: !GetAtt s3Bucket.Arn 278 | s3BucketNotificationSQSQueueArn: 279 | Value: !GetAtt s3BucketNotificationSQSQueue.Arn -------------------------------------------------------------------------------- /Single Account CloudFormation/securityHub.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: This is a CloudFormation template to enable SecurityHub - https://aws.amazon.com/security-hub/ 4 | 5 | 6 | Parameters: 7 | service: 8 | Type: String 9 | Description: service name 10 | Default: splunk-aws-gdi-toolkit 11 | 12 | stage: 13 | Type: String 14 | Description: Used to distinguish between stages of an environment 15 | Default: dev 16 | 17 | contact: 18 | Description: Used to identify a contact for the resources created in this stack. As an example, this could be an email address or username. 19 | Type: String 20 | 21 | 22 | Resources: 23 | securityHub: 24 | Type: AWS::SecurityHub::Hub 25 | Properties: 26 | Tags: 27 | service: !Ref service 28 | stage: !Ref stage 29 | contact: !Ref contact 30 | name: !Sub "${service}-securityHub" 31 | 32 | 33 | Outputs: 34 | securityHubArn: 35 | Value: !Ref securityHub -------------------------------------------------------------------------------- /VPCFlowLogs-Firehose-Resources/kdfToSplunk.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: This is a CloudFormation template to create infrastructure to send logs that are sent to Kinesis Data Firehose to Splunk. 4 | 5 | Parameters: 6 | service: 7 | Type: String 8 | Description: Service name used in tagging AWS resources. 9 | Default: splunk-aws-gdi-toolkit 10 | 11 | stage: 12 | Type: String 13 | Description: Used to distinguish between stages of an environment (dev, test, prod, stage, etc). Only used in AWS resource tagging. 14 | Default: dev 15 | 16 | contact: 17 | Description: Used to identify a contact for the resources created in this stack. Only used in AWS resource tagging. As an example, this could be an email address or username. 18 | Type: String 19 | Default: "" 20 | 21 | splunkHECEndpoint: 22 | Type: String 23 | Description: Destination URL that Firehose will send data to. 24 | 25 | splunkHECToken: 26 | Type: String 27 | Description: HEC token Firehose will use to authenticate data being sent to Splunk. 28 | 29 | cloudWatchAlertEmail: 30 | Type: String 31 | Description: Email address for receiving alerts related to CloudTrail ingestion. Leave empty for no alerting. 32 | Default: "" 33 | 34 | Conditions: 35 | enableAlerting: !Not 36 | - !Equals 37 | - !Ref cloudWatchAlertEmail 38 | - "" 39 | 40 | Resources: 41 | 42 | # Firehose > Splunk resources 43 | firehose: 44 | Type: AWS::KinesisFirehose::DeliveryStream 45 | Properties: 46 | DeliveryStreamName: !Sub "${AWS::AccountId}-${AWS::Region}-vpcflow-firehose" 47 | DeliveryStreamType: DirectPut 48 | SplunkDestinationConfiguration: 49 | BufferingHints: 50 | IntervalInSeconds: 5 51 | SizeInMBs: 1 52 | CloudWatchLoggingOptions: 53 | Enabled: true 54 | LogGroupName: !Ref firehoseLogGroup 55 | LogStreamName: "SplunkDelivery" 56 | HECAcknowledgmentTimeoutInSeconds: 180 #300 57 | HECEndpoint: !Ref splunkHECEndpoint 58 | HECEndpointType: "Raw" 59 | HECToken: !Ref splunkHECToken 60 | RetryOptions: 61 | DurationInSeconds: 3600 62 | S3BackupMode: "FailedEventsOnly" 63 | S3Configuration: 64 | BucketARN: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-vpcflow-firehose-backsplash" 65 | BufferingHints: 66 | IntervalInSeconds: 300 67 | SizeInMBs: 16 68 | CompressionFormat: "GZIP" 69 | Prefix: !Sub "${AWS::AccountId}-${AWS::Region}-vpcflow-firehose" 70 | RoleARN: !GetAtt firehoseIAMRole.Arn 71 | 72 | firehoseIAMPolicy: 73 | Type: AWS::IAM::ManagedPolicy 74 | Properties: 75 | PolicyDocument: 76 | Version: '2012-10-17' 77 | Statement: 78 | - Effect: Allow 79 | Action: 80 | - logs:Describe* 81 | - logs:PutLogEvents 82 | Resource: !GetAtt firehoseLogGroup.Arn 83 | - Effect: Allow 84 | Action: 85 | - s3:PutObject 86 | Resource: !Sub "arn:aws:s3:::${AWS::AccountId}-${AWS::Region}-vpcflow-firehose-backsplash/*" 87 | ManagedPolicyName: !Sub "${AWS::AccountId}-${AWS::Region}-vpcflow-firehose-iam-policy" 88 | 89 | firehoseIAMRole: 90 | Type: AWS::IAM::Role 91 | Properties: 92 | AssumeRolePolicyDocument: 93 | Version: '2012-10-17' 94 | Statement: 95 | - Effect: "Allow" 96 | Principal: 97 | Service: "firehose.amazonaws.com" 98 | Action: 99 | - sts:AssumeRole 100 | ManagedPolicyArns: 101 | - !Ref firehoseIAMPolicy 102 | RoleName: !Sub "${AWS::AccountId}-${AWS::Region}-vpcflow-firehose-iam-role" 103 | Tags: 104 | - Key: service 105 | Value: !Ref service 106 | - Key: stage 107 | Value: !Ref stage 108 | - Key: contact 109 | Value: !Ref contact 110 | 111 | firehoseBacksplashBucket: 112 | Type: AWS::S3::Bucket 113 | Properties: 114 | BucketName: !Sub "${AWS::AccountId}-${AWS::Region}-vpcflow-firehose-backsplash" 115 | AccessControl: Private 116 | PublicAccessBlockConfiguration: 117 | BlockPublicAcls: true 118 | BlockPublicPolicy: true 119 | IgnorePublicAcls: true 120 | RestrictPublicBuckets: true 121 | BucketEncryption: 122 | ServerSideEncryptionConfiguration: 123 | - ServerSideEncryptionByDefault: 124 | SSEAlgorithm: AES256 125 | LifecycleConfiguration: 126 | Rules: 127 | - Id: !Sub "${AWS::AccountId}-${AWS::Region}-vpcflow-firehose-backsplash-cleanup" 128 | AbortIncompleteMultipartUpload: 129 | DaysAfterInitiation: 1 130 | Status: Enabled 131 | Tags: 132 | - Key: service 133 | Value: !Ref service 134 | - Key: stage 135 | Value: !Ref stage 136 | - Key: contact 137 | Value: !Ref contact 138 | 139 | firehoseLogGroup: 140 | Type: AWS::Logs::LogGroup 141 | Properties: 142 | LogGroupName: !Sub "/aws/kinesisfirehose/${AWS::AccountId}-${AWS::Region}-vpcflow-firehose" 143 | RetentionInDays: 7 144 | 145 | firehoseLogStream: 146 | Type: AWS::Logs::LogStream 147 | Properties: 148 | LogGroupName: !Ref firehoseLogGroup 149 | LogStreamName: "SplunkDelivery" 150 | 151 | # Monitoring resoruces 152 | monitoringSNSTopic: 153 | Condition: enableAlerting 154 | Type: AWS::SNS::Topic 155 | Properties: 156 | DisplayName: !Sub "${AWS::AccountId}-${AWS::Region}-vpcflow-alerting-topic" 157 | TopicName: !Sub "${AWS::AccountId}-${AWS::Region}-vpcflow-alerting-topic" 158 | Subscription: 159 | - Endpoint: !Ref cloudWatchAlertEmail 160 | Protocol: email 161 | 162 | firehoseDeliveryAlarm: 163 | Condition: enableAlerting 164 | Type: AWS::CloudWatch::Alarm 165 | Properties: 166 | ActionsEnabled: True 167 | AlarmActions: 168 | - !Ref monitoringSNSTopic 169 | AlarmDescription: !Sub "Alarm if Firehose ${AWS::AccountId}-${AWS::Region}-vpcflow-firehose cannot deliver to Splunk." 170 | AlarmName: !Sub "${AWS::AccountId}-${AWS::Region}-vpcflow-firehose-delivery-alarm" 171 | ComparisonOperator: LessThanThreshold 172 | Dimensions: 173 | - Name: DeliveryStreamName 174 | Value: !Sub "${AWS::AccountId}-${AWS::Region}-vpcflow-firehose" 175 | EvaluationPeriods: 1 176 | MetricName: DeliveryToSplunk.Success 177 | Namespace: AWS/Firehose 178 | Period: 60 179 | Statistic: Maximum 180 | Threshold: 1 181 | Unit: Count 182 | 183 | firehoseThrottlingAlarm: 184 | Condition: enableAlerting 185 | Type: AWS::CloudWatch::Alarm 186 | Properties: 187 | ActionsEnabled: True 188 | AlarmActions: 189 | - !Ref monitoringSNSTopic 190 | AlarmDescription: !Sub "Alarm if Firehose ${AWS::AccountId}-${AWS::Region}-vpcflow-firehose is throttling events because of quota limits." 191 | AlarmName: !Sub "${AWS::AccountId}-${AWS::Region}-vpcflow-firehose-throttling-alarm" 192 | ComparisonOperator: GreaterThanThreshold 193 | Dimensions: 194 | - Name: DeliveryStreamName 195 | Value: !Sub "${AWS::AccountId}-${AWS::Region}-vpcflow-firehose" 196 | EvaluationPeriods: 1 197 | MetricName: ThrottledRecords 198 | Namespace: AWS/Firehose 199 | Period: 60 200 | Statistic: Maximum 201 | Threshold: 0 202 | Unit: Count 203 | 204 | Outputs: 205 | firehoseArn: 206 | Value: !GetAtt firehose.Arn 207 | firehoseIAMRoleArn: 208 | Value: !GetAtt firehoseIAMRole.Arn 209 | firehoseBacksplashBucketArn: 210 | Value: !GetAtt firehoseBacksplashBucket.Arn 211 | firehoseLogGroupArn: 212 | Value: !GetAtt firehoseLogGroup.Arn 213 | monitoringSNSTopicArn: 214 | Condition: enableAlerting 215 | Value: !Ref monitoringSNSTopic -------------------------------------------------------------------------------- /VPCFlowLogs-Firehose-Resources/splunkDashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "visualizations": { 3 | "viz_AcyVeBot": { 4 | "type": "splunk.column", 5 | "showProgressBar": false, 6 | "showLastUpdated": false, 7 | "title": "$sourcetype$ License Usage Over Time (GB)", 8 | "dataSources": { 9 | "primary": "ds_6WBt9iY9" 10 | }, 11 | "options": { 12 | "legendDisplay": "off" 13 | } 14 | }, 15 | "viz_KMF849o7": { 16 | "type": "splunk.singlevalue", 17 | "title": "$sourcetype$ Event License Usage", 18 | "dataSources": { 19 | "primary": "ds_HUEXsOZi" 20 | }, 21 | "options": { 22 | "numberPrecision": 2, 23 | "unit": "GB", 24 | "trendDisplay": "off", 25 | "sparklineDisplay": "off" 26 | } 27 | }, 28 | "viz_OF3a0hX5": { 29 | "type": "splunk.singlevalue", 30 | "title": "$sourcetype$ Event Count", 31 | "dataSources": { 32 | "primary": "ds_LN5KxPiz" 33 | }, 34 | "options": { 35 | "unit": "events", 36 | "trendDisplay": "off", 37 | "sparklineDisplay": "off" 38 | } 39 | }, 40 | "viz_kqPNrs3R": { 41 | "type": "splunk.line", 42 | "showProgressBar": false, 43 | "showLastUpdated": false, 44 | "title": "$sourcetype$ Event Count Over Time", 45 | "dataSources": { 46 | "primary": "ds_7YVGDkt0" 47 | }, 48 | "options": { 49 | "legendDisplay": "off" 50 | } 51 | }, 52 | "viz_eU3T7rCG": { 53 | "type": "splunk.singlevalue", 54 | "title": "AWS Accounts Sending Events", 55 | "dataSources": { 56 | "primary": "ds_noJxIRWy" 57 | } 58 | }, 59 | "viz_nO6QvmkM": { 60 | "type": "splunk.line", 61 | "title": "AWS Accounts Sending $sourcetype$ Events", 62 | "dataSources": { 63 | "primary": "ds_FFly4UzK" 64 | }, 65 | "options": { 66 | "legendDisplay": "off" 67 | } 68 | }, 69 | "viz_NY0AedMk": { 70 | "type": "splunk.line", 71 | "title": "$sourcetype$ Event Latency Over Time", 72 | "showProgressBar": false, 73 | "showLastUpdated": false, 74 | "dataSources": { 75 | "primary": "ds_srz6OcK5" 76 | }, 77 | "options": { 78 | "legendDisplay": "off" 79 | } 80 | }, 81 | "viz_XqNxe7aK": { 82 | "type": "splunk.singlevalue", 83 | "title": "$sourcetype$ Latest Event Latency", 84 | "dataSources": { 85 | "primary": "ds_tCds3VFx" 86 | }, 87 | "options": { 88 | "sparklineDisplay": "off", 89 | "unit": "seconds" 90 | }, 91 | "showProgressBar": false, 92 | "showLastUpdated": false 93 | } 94 | }, 95 | "dataSources": { 96 | "ds_6WBt9iY9": { 97 | "type": "ds.search", 98 | "options": { 99 | "query": "index=_internal source=*license_usage* idx=$eventIndex$ st=$sourcetype$ NOT source=*license_usage_summary.log*\n| timechart sum(b) as ingestedDataGB\n| eval ingestedDataGB = ingestedDataGB/1024/1024/1024", 100 | "queryParameters": { 101 | "earliest": "$global_time.earliest$", 102 | "latest": "$global_time.latest$" 103 | } 104 | }, 105 | "name": "licenseUsageBase" 106 | }, 107 | "ds_HUEXsOZi": { 108 | "type": "ds.chain", 109 | "options": { 110 | "extend": "ds_6WBt9iY9", 111 | "query": "| stats sum(ingestedDataGB) as ingestedDataGB" 112 | }, 113 | "name": "licenseUsageTotal" 114 | }, 115 | "ds_7YVGDkt0": { 116 | "type": "ds.search", 117 | "options": { 118 | "query": "index=_internal series=$sourcetype$ sourcetype=splunkd source=*metrics.log*\n| timechart sum(ev) as eventCount" 119 | }, 120 | "name": "eventCountBase" 121 | }, 122 | "ds_LN5KxPiz": { 123 | "type": "ds.chain", 124 | "options": { 125 | "extend": "ds_7YVGDkt0", 126 | "query": "| stats sum(eventCount) as eventCount" 127 | }, 128 | "name": "eventCountTotal" 129 | }, 130 | "ds_FFly4UzK": { 131 | "type": "ds.search", 132 | "options": { 133 | "query": "index=$eventIndex$ sourcetype=$sourcetype$\n| timechart dc(vendor_account) as accountCount" 134 | }, 135 | "name": "accountCountTimechart" 136 | }, 137 | "ds_noJxIRWy": { 138 | "type": "ds.search", 139 | "options": { 140 | "query": "index=$eventIndex$ sourcetype=$sourcetype$\n| stats dc(vendor_account) as accountCount" 141 | }, 142 | "name": "accountCountTotal" 143 | }, 144 | "ds_srz6OcK5": { 145 | "type": "ds.search", 146 | "options": { 147 | "query": "index=$eventIndex$ sourcetype=$sourcetype$ \n| eval eventLatency = abs(_indextime - _time)\n| timechart avg(eventLatency) as eventLatency" 148 | }, 149 | "name": "eventLatencyBase" 150 | }, 151 | "ds_tCds3VFx": { 152 | "type": "ds.search", 153 | "options": { 154 | "query": "index=$eventIndex$ sourcetype=$sourcetype$ | head 1\n| eval eventLatency = abs(_indextime - _time)\n| table eventLatency" 155 | }, 156 | "name": "eventLatencyLatest" 157 | }, 158 | "ds_N5KOizZ9": { 159 | "type": "ds.search", 160 | "options": { 161 | "query": "| tstats count where index=* by index", 162 | "queryParameters": { 163 | "earliest": "$global_time.earliest$", 164 | "latest": "$global_time.latest$" 165 | } 166 | }, 167 | "name": "indexList" 168 | }, 169 | "ds_f9m7vKr7": { 170 | "type": "ds.search", 171 | "options": { 172 | "query": "| tstats count where index=$eventIndex$ by sourcetype" 173 | }, 174 | "name": "sourcetypeList" 175 | } 176 | }, 177 | "defaults": { 178 | "dataSources": { 179 | "ds.search": { 180 | "options": { 181 | "queryParameters": { 182 | "latest": "$global_time.latest$", 183 | "earliest": "$global_time.earliest$" 184 | } 185 | } 186 | } 187 | } 188 | }, 189 | "inputs": { 190 | "input_global_trp": { 191 | "type": "input.timerange", 192 | "options": { 193 | "token": "global_time", 194 | "defaultValue": "-24h@h,now" 195 | }, 196 | "title": "Global Time Range" 197 | }, 198 | "input_9rNFa8NM": { 199 | "options": { 200 | "items": [ 201 | { 202 | "label": "main", 203 | "value": "main" 204 | } 205 | ], 206 | "defaultValue": "main", 207 | "token": "eventIndex" 208 | }, 209 | "title": "Event Index", 210 | "type": "input.dropdown", 211 | "dataSources": { 212 | "primary": "ds_N5KOizZ9" 213 | } 214 | }, 215 | "input_C3SY1WEW": { 216 | "options": { 217 | "items": [ 218 | { 219 | "label": "All", 220 | "value": "*" 221 | } 222 | ], 223 | "defaultValue": "*", 224 | "token": "sourcetype" 225 | }, 226 | "title": "Event Sourcetype", 227 | "type": "input.dropdown", 228 | "dataSources": { 229 | "primary": "ds_f9m7vKr7" 230 | } 231 | } 232 | }, 233 | "layout": { 234 | "type": "grid", 235 | "options": {}, 236 | "structure": [ 237 | { 238 | "item": "viz_eU3T7rCG", 239 | "type": "block", 240 | "position": { 241 | "x": 600, 242 | "y": 0, 243 | "w": 300, 244 | "h": 100 245 | } 246 | }, 247 | { 248 | "item": "viz_XqNxe7aK", 249 | "type": "block", 250 | "position": { 251 | "x": 900, 252 | "y": 0, 253 | "w": 300, 254 | "h": 100 255 | } 256 | }, 257 | { 258 | "item": "viz_KMF849o7", 259 | "type": "block", 260 | "position": { 261 | "x": 0, 262 | "y": 0, 263 | "w": 300, 264 | "h": 100 265 | } 266 | }, 267 | { 268 | "item": "viz_AcyVeBot", 269 | "type": "block", 270 | "position": { 271 | "x": 0, 272 | "y": 100, 273 | "w": 1200, 274 | "h": 300 275 | } 276 | }, 277 | { 278 | "item": "viz_kqPNrs3R", 279 | "type": "block", 280 | "position": { 281 | "x": 0, 282 | "y": 400, 283 | "w": 1200, 284 | "h": 300 285 | } 286 | }, 287 | { 288 | "item": "viz_nO6QvmkM", 289 | "type": "block", 290 | "position": { 291 | "x": 0, 292 | "y": 700, 293 | "w": 1200, 294 | "h": 300 295 | } 296 | }, 297 | { 298 | "item": "viz_NY0AedMk", 299 | "type": "block", 300 | "position": { 301 | "x": 0, 302 | "y": 1000, 303 | "w": 1200, 304 | "h": 300 305 | } 306 | }, 307 | { 308 | "item": "viz_OF3a0hX5", 309 | "type": "block", 310 | "position": { 311 | "x": 300, 312 | "y": 0, 313 | "w": 300, 314 | "h": 100 315 | } 316 | } 317 | ], 318 | "globalInputs": [ 319 | "input_global_trp", 320 | "input_9rNFa8NM", 321 | "input_C3SY1WEW" 322 | ] 323 | }, 324 | "description": "", 325 | "title": "Splunk AWS GDI Toolkit - CloudWatch-Events-Firehose" 326 | } -------------------------------------------------------------------------------- /VPCFlowLogs-Firehose-Resources/vpcFlowLogsToKDF.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: This is a CloudFormation template to create VPC Flow Logging infrastructure to Kinesis Data Firehose. This should be deployed once for each VPC, ENI, or subnet you want to capture VPC Flow Logs from. 4 | 5 | 6 | Parameters: 7 | service: 8 | Type: String 9 | Description: service name 10 | Default: splunk-aws-gdi-toolkit 11 | 12 | stage: 13 | Type: String 14 | Description: Used to distinguish between stages of an environment (dev, test, prod, stage, etc). Only used in AWS resource tagging. 15 | Default: dev 16 | 17 | contact: 18 | Description: Used to identify a contact for the resources created in this stack. Only used in AWS resource tagging. As an example, this could be an email address or username. 19 | Type: String 20 | 21 | kdfArn: 22 | Description: ARN of the Kinesis Data Firehose the VPC Flow Logs will be sent to 23 | Type: String 24 | 25 | vpcFlowLogResourceId: 26 | Description: The ID of the subnet, network interface, or VPC for which you want to create a flow log. Taken straight from https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-flowlog.html 27 | Type: String 28 | 29 | vpcFlowLogResourceType: 30 | Description: The type of resource for which to create the flow log. Select NetworkInterface if you specified an ENI for vpcFlowLogResourceId, "Subnet" if you specified a subnet ID for vpcFlowLogResourceId, or "VPC" if you specified a VPC ID for vpcFlowLogResourceId" 31 | Type: String 32 | Default: VPC 33 | AllowedValues: 34 | - NetworkInterface 35 | - Subnet 36 | - VPC 37 | 38 | vpcFlowLogTrafficType: 39 | Description: Type of traffic you want to log. 40 | Type: String 41 | Default: ALL 42 | AllowedValues: 43 | - ACCEPT 44 | - REJECT 45 | - ALL 46 | 47 | 48 | Resources: 49 | vpcFlowLog: 50 | Type: AWS::EC2::FlowLog 51 | Properties: 52 | LogDestination: !Ref kdfArn 53 | LogDestinationType: kinesis-data-firehose 54 | ResourceId: !Ref vpcFlowLogResourceId 55 | ResourceType: !Ref vpcFlowLogResourceType 56 | TrafficType: !Ref vpcFlowLogTrafficType 57 | Tags: 58 | - Key: service 59 | Value: !Ref service 60 | - Key: stage 61 | Value: !Ref stage 62 | - Key: contact 63 | Value: !Ref contact 64 | 65 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Splunk AWS GDI Tookit 2 | 3 | This repo is for individuals and organizations looking to get better visibility, observability, and monitoring in their AWS account or AWS accounts. There are sets of CloudFormation templates here designed to help get data related to AWS accounts and services to Splunk for analysis and alerting. 4 | 5 | PRs are open, and feel free to reach out to me over the [Splunk Usergroups Slack](https://docs.splunk.com/Documentation/Community/current/community/Chat) if you have questions, comments, or concerns! 6 | 7 | Documentation for how to deploy the toolkit, design decisions, diagrams, and FAQs can be found in the [wiki](https://github.com/splunk/splunk-aws-gdi-toolkit/wiki). --------------------------------------------------------------------------------