├── .gitignore
├── log-analysis
    ├── select-all.sql
    ├── create-consolidated-view.sql
    ├── select-top-10.sql
    ├── firehose-policy.json
    ├── create-ecs-table.sql
    ├── create-eks-table.sql
    └── firehose-delivery-policy.json
├── NOTICE
├── ecs
    ├── enable-fluent-log-driver.sh
    ├── Dockerfile
    ├── parsers.conf
    ├── fluent-bit.conf
    ├── load-gen-ecs.sh
    ├── nginx-task-definition.json
    └── ecs-fluent-bit-daemonset.yml
├── .github
    └── PULL_REQUEST_TEMPLATE.md
├── CODE_OF_CONDUCT.md
├── eks
    ├── eks-fluent-bit-daemonset-rbac.yaml
    ├── eks-nginx-app.yaml
    ├── load-gen-eks.sh
    ├── eks-fluent-bit-daemonset-policy.json
    ├── eks-fluent-bit-daemonset.yaml
    └── eks-fluent-bit-configmap.yaml
├── README.md
├── CONTRIBUTING.md
└── LICENSE


/.gitignore:
--------------------------------------------------------------------------------
1 | /.DS_Store
2 | 


--------------------------------------------------------------------------------
/log-analysis/select-all.sql:
--------------------------------------------------------------------------------
1 | SELECT *
2 | FROM fluentbit_consolidated
3 | ORDER BY remote


--------------------------------------------------------------------------------
/NOTICE:
--------------------------------------------------------------------------------
1 | Amazon Ecs Fluent Bit Daemon Service
2 | Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 
3 | 


--------------------------------------------------------------------------------
/ecs/enable-fluent-log-driver.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | echo "ECS_AVAILABLE_LOGGING_DRIVERS=[\"awslogs\",\"fluentd\"]" >> /etc/ecs/ecs.config


--------------------------------------------------------------------------------
/ecs/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM amazon/aws-for-fluent-bit:latest
2 | ADD fluent-bit.conf /fluent-bit/etc/
3 | ADD parsers.conf /fluent-bit/etc/
4 | 


--------------------------------------------------------------------------------
/log-analysis/create-consolidated-view.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE VIEW "fluentbit_consolidated" AS
2 | SELECT * , 'ECS' as source
3 | FROM fluentbit_ecs
4 | UNION
5 | SELECT * , 'EKS' as source
6 | FROM fluentbit_eks


--------------------------------------------------------------------------------
/log-analysis/select-top-10.sql:
--------------------------------------------------------------------------------
1 | SELECT source,
2 |          remote AS IP,
3 |          count(remote) AS num_requests
4 | FROM fluentbit_consolidated
5 | GROUP BY  remote, source
6 | ORDER BY  num_requests DESC LIMIT 10
7 | 


--------------------------------------------------------------------------------
/.github/PULL_REQUEST_TEMPLATE.md:
--------------------------------------------------------------------------------
1 | *Issue #, if available:*
2 | 
3 | *Description of changes:*
4 | 
5 | 
6 | By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
7 | 


--------------------------------------------------------------------------------
/log-analysis/firehose-policy.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "Version": "2012-10-17",
 3 |   "Statement": {
 4 |       "Effect": "Allow",
 5 |       "Principal": {
 6 |         "Service": "firehose.amazonaws.com"
 7 |       },
 8 |       "Action": "sts:AssumeRole"
 9 |   }
10 | }


--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | ## Code of Conduct
2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
4 | opensource-codeofconduct@amazon.com with any additional questions or comments.
5 | 


--------------------------------------------------------------------------------
/ecs/parsers.conf:
--------------------------------------------------------------------------------
1 | [PARSER]
2 |     Name   nginx
3 |     Format regex
4 |     Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")? \"-\"$
5 |     Time_Key time
6 |     Time_Format %d/%b/%Y:%H:%M:%S %z


--------------------------------------------------------------------------------
/ecs/fluent-bit.conf:
--------------------------------------------------------------------------------
 1 | [SERVICE]
 2 |     Parsers_File parsers.conf
 3 | 
 4 | [INPUT]
 5 |     Name forward
 6 |     unix_path /var/run/fluent.sock
 7 | 
 8 | [FILTER]
 9 |     Name parser
10 |     Match **
11 |     Parser nginx
12 |     Key_Name log
13 | 
14 | [OUTPUT]
15 |     Name firehose
16 |     Match **
17 |     delivery_stream ecs-stream
18 |     region us-west-2


--------------------------------------------------------------------------------
/log-analysis/create-ecs-table.sql:
--------------------------------------------------------------------------------
 1 | CREATE EXTERNAL TABLE fluentbit_ecs (
 2 |     agent string,
 3 |     code string,
 4 |     host string,
 5 |     method string,
 6 |     path string,
 7 |     referer string,
 8 |     remote string,
 9 |     size string,
10 |     user string
11 | )
12 | ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
13 | LOCATION 's3://mh9-firelens-demo/ecs2019/'


--------------------------------------------------------------------------------
/log-analysis/create-eks-table.sql:
--------------------------------------------------------------------------------
 1 | CREATE EXTERNAL TABLE fluentbit_eks (
 2 |     agent string,
 3 |     code string,
 4 |     host string,
 5 |     method string,
 6 |     path string,
 7 |     referer string,
 8 |     remote string,
 9 |     size string,
10 |     user string
11 | )
12 | ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
13 | LOCATION 's3://mh9-firelens-demo/eks2019/'


--------------------------------------------------------------------------------
/ecs/load-gen-ecs.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | ################################################################################
 4 | # Generate load for the NGINXs services in ECS
 5 | 
 6 | nginxurls=$(ecs-cli ps --desired-status RUNNING | grep nginx | awk '{ split($3, url, "-") ; print(url[1]) }')
 7 | while true
 8 | do
 9 |     printf "Hit " 
10 |     for nginxurl in $nginxurls
11 |     do
12 |         curl -s $nginxurl  > /dev/null
13 |         printf "$nginxurl "
14 |     done
15 |     printf "\n"
16 |     sleep 2
17 | done


--------------------------------------------------------------------------------
/ecs/nginx-task-definition.json:
--------------------------------------------------------------------------------
 1 | {
 2 | 	"requiresCompatibilities": [
 3 | 		"EC2"
 4 | 	],
 5 | 	"containerDefinitions": [{
 6 | 		"name": "nginx-fluentbitdemo",
 7 | 		"image": "nginx:1.17",
 8 | 		"memory": 100,
 9 | 		"essential": true,
10 | 		"portMappings": [{
11 | 			"hostPort": 80,
12 | 			"protocol": "tcp",
13 | 			"containerPort": 80
14 | 		}],
15 | 		"logConfiguration": {
16 | 			"logDriver": "fluentd",
17 | 			"options": {
18 | 				"fluentd-address": "unix:///var/run/fluent.sock",
19 | 				"tag": "logs-from-nginx"
20 | 			}
21 | 		}
22 | 	}],
23 | 	"family": "nginx-fluentbitdemo"
24 | }
25 | 


--------------------------------------------------------------------------------
/eks/eks-fluent-bit-daemonset-rbac.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: rbac.authorization.k8s.io/v1beta1
 2 | kind: ClusterRole
 3 | metadata:
 4 |   name: pod-log-reader
 5 | rules:
 6 | - apiGroups: [""]
 7 |   resources:
 8 |   - namespaces
 9 |   - pods
10 |   verbs: ["get", "list", "watch"]
11 | ---
12 | apiVersion: rbac.authorization.k8s.io/v1beta1
13 | kind: ClusterRoleBinding
14 | metadata:
15 |   name: pod-log-crb
16 | roleRef:
17 |   apiGroup: rbac.authorization.k8s.io
18 |   kind: ClusterRole
19 |   name: pod-log-reader
20 | subjects:
21 | - kind: ServiceAccount
22 |   name: fluent-bit
23 |   namespace: default


--------------------------------------------------------------------------------
/eks/eks-nginx-app.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: apps/v1
 2 | kind: Deployment
 3 | metadata:
 4 |   name: nginx
 5 |   labels:
 6 |     app.kubernetes.io/name: nginx
 7 | spec:
 8 |   replicas: 4 
 9 |   selector:
10 |     matchLabels:
11 |       app: nginx
12 |   template:
13 |     metadata:
14 |       labels:
15 |         app: nginx
16 |     spec:
17 |       containers:
18 |       - name: nginx
19 |         image: nginx:1.17
20 |         ports:
21 |         - containerPort: 80
22 | ---
23 | apiVersion: v1
24 | kind: Service
25 | metadata:
26 |   name: nginx
27 |   labels:
28 |     app: nginx
29 | spec:
30 |   ports:
31 |   - port: 80
32 |     targetPort: 80
33 |   selector:
34 |     app: nginx


--------------------------------------------------------------------------------
/eks/load-gen-eks.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | ################################################################################
 4 | # Generate load for the NGINXs services in EKS
 5 | 
 6 | # make sure to patch to LB
 7 | kubectl patch svc nginx -p '{"spec": {"type": "LoadBalancer"}}'
 8 | 
 9 | # give the LB 3 minutes to be up and running
10 | echo "Now waiting for 3min until the load balancer is up ..."
11 | sleep 180
12 | 
13 | echo "Starting to hammer the load balancer:"
14 | 
15 | nginxurl=$(kubectl get svc/nginx -o json | jq .status.loadBalancer.ingress[].hostname -r)
16 | while true
17 | do
18 |     printf "Hit " 
19 |         curl -s $nginxurl > /dev/null
20 |         printf "$nginxurl "
21 |     printf "\n"
22 |     sleep 2
23 | done


--------------------------------------------------------------------------------
/eks/eks-fluent-bit-daemonset-policy.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "Version": "2012-10-17",
 3 |     "Statement": [
 4 |         {
 5 |             "Effect": "Allow",
 6 |             "Action": [
 7 |                 "firehose:PutRecordBatch"
 8 |             ],
 9 |             "Resource": "*"
10 |         },
11 |         {
12 |             "Effect": "Allow",
13 |             "Action": "logs:PutLogEvents",
14 |             "Resource": "arn:aws:logs:*:*:log-group:*:*:*"
15 |         },
16 |         {
17 |             "Effect": "Allow",
18 |             "Action": [
19 |                 "logs:CreateLogStream",
20 |                 "logs:DescribeLogStreams",
21 |                 "logs:PutLogEvents"
22 |             ],
23 |             "Resource": "arn:aws:logs:*:*:log-group:*"
24 |         },
25 |         {
26 |             "Effect": "Allow",
27 |             "Action": "logs:CreateLogGroup",
28 |             "Resource": "*"
29 |         }
30 |     ]
31 | }


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Centralized logging in action: multi-cluster log analysis
 2 | 
 3 | This repo contains the container runtime definitions and configurations for an
 4 | end-to-end demo of the Amazon Fluent Bit plugin, showing a multi-cluster (ECS/EKS)
 5 | log analysis, streaming the log data Kinesis Data Firehose to S3, where we then
 6 | query the log data with Amazon Athena.
 7 | 
 8 | See the blog: [Centralized Container Logging with Fluent Bit](https://aws.amazon.com/blogs/opensource/centralized-container-logging-fluent-bit/).
 9 | 
10 | ## Setup for Amazon ECS
11 | 
12 | See the [ecs/](ecs/) directory of this repo for all the source files.
13 | 
14 | ## Setup for Amazon EKS
15 | 
16 | See the [eks/](eks/) directory of this repo for all the source files.
17 | 
18 | ## Setup for log analysis
19 | 
20 | See the [log-analysis/](log-analysis/) directory of this repo for all the source files.
21 | 
22 | ## License
23 | 
24 | All material available under the [Apache License Version 2.0](LICENSE).
25 | 


--------------------------------------------------------------------------------
/eks/eks-fluent-bit-daemonset.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: apps/v1
 2 | kind: DaemonSet
 3 | metadata:
 4 |   name: fluentbit
 5 |   labels:
 6 |     app.kubernetes.io/name: fluentbit
 7 | spec:
 8 |   selector:
 9 |     matchLabels:
10 |       name: fluentbit
11 |   template:
12 |     metadata:
13 |       labels:
14 |         name: fluentbit
15 |     spec:
16 |       serviceAccountName: fluent-bit
17 |       containers:
18 |       - name: aws-for-fluent-bit
19 |         image: 906394416424.dkr.ecr.us-west-2.amazonaws.com/aws-for-fluent-bit:latest
20 |         volumeMounts:
21 |         - name: varlog
22 |           mountPath: /var/log
23 |         - name: varlibdockercontainers
24 |           mountPath: /var/lib/docker/containers
25 |           readOnly: true
26 |         - name: fluent-bit-config
27 |           mountPath: /fluent-bit/etc/
28 |         - name: mnt
29 |           mountPath: /mnt
30 |           readOnly: true
31 |         resources:
32 |           limits:
33 |             memory: 500Mi
34 |           requests:
35 |             cpu: 500m
36 |             memory: 100Mi
37 |       volumes:
38 |       - name: varlog
39 |         hostPath:
40 |           path: /var/log
41 |       - name: varlibdockercontainers
42 |         hostPath:
43 |           path: /var/lib/docker/containers
44 |       - name: fluent-bit-config
45 |         configMap:
46 |           name: fluent-bit-config
47 |       - name: mnt
48 |         hostPath:
49 |           path: /mnt
50 | 


--------------------------------------------------------------------------------
/eks/eks-fluent-bit-configmap.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: v1
 2 | kind: ConfigMap
 3 | metadata:
 4 |   name: fluent-bit-config
 5 |   labels:
 6 |     app.kubernetes.io/name: fluentbit
 7 | data:
 8 |   fluent-bit.conf: |
 9 |     [SERVICE]
10 |         Parsers_File  parsers.conf
11 |     [INPUT]
12 |         Name              tail
13 |         Tag               kube.*
14 |         Path              /var/log/containers/*.log
15 |         Parser            docker
16 |         DB                /var/log/flb_kube.db
17 |         Mem_Buf_Limit     5MB
18 |         Skip_Long_Lines   On
19 |         Refresh_Interval  10
20 |     [FILTER]
21 |         Name parser
22 |         Match **
23 |         Parser nginx
24 |         Key_Name log
25 |     [OUTPUT]
26 |         Name firehose
27 |         Match **
28 |         delivery_stream eks-stream
29 |         region us-west-2 
30 |   parsers.conf: |
31 |     [PARSER]
32 |         Name   nginx
33 |         Format regex
34 |         Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")? \"-\"$
35 |         Time_Key time
36 |         Time_Format %d/%b/%Y:%H:%M:%S %z
37 |     [PARSER]
38 |         Name        docker
39 |         Format      json
40 |         Time_Key    time
41 |         Time_Format %Y-%m-%dT%H:%M:%S.%L
42 |         Time_Keep   On
43 |         # Command      |  Decoder | Field | Optional Action
44 |         # =============|==================|=================
45 |         Decode_Field_As   escaped    log
46 | 


--------------------------------------------------------------------------------
/log-analysis/firehose-delivery-policy.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "Version": "2012-10-17",
 3 |   "Statement": [
 4 |     {
 5 |       "Sid": "",
 6 |       "Effect": "Allow",
 7 |       "Action": [
 8 |         "glue:GetTableVersions"
 9 |       ],
10 |       "Resource": "*"
11 |     },
12 |     {
13 |       "Sid": "",
14 |       "Effect": "Allow",
15 |       "Action": [
16 |         "s3:AbortMultipartUpload",
17 |         "s3:GetBucketLocation",
18 |         "s3:GetObject",
19 |         "s3:ListBucket",
20 |         "s3:ListBucketMultipartUploads",
21 |         "s3:PutObject"
22 |       ],
23 |       "Resource": [
24 |         "arn:aws:s3:::mh9-firelens-demo",
25 |         "arn:aws:s3:::mh9-firelens-demo/*",
26 |         "arn:aws:s3:::%FIREHOSE_BUCKET_NAME%",
27 |         "arn:aws:s3:::%FIREHOSE_BUCKET_NAME%/*"
28 |       ]
29 |     },
30 |     {
31 |       "Sid": "",
32 |       "Effect": "Allow",
33 |       "Action": [
34 |         "lambda:InvokeFunction",
35 |         "lambda:GetFunctionConfiguration"
36 |       ],
37 |       "Resource": "arn:aws:lambda:us-west-2:XXXXXXXXXXXX:function:%FIREHOSE_DEFAULT_FUNCTION%:%FIREHOSE_DEFAULT_VERSION%"
38 |     },
39 |     {
40 |       "Sid": "",
41 |       "Effect": "Allow",
42 |       "Action": [
43 |         "logs:PutLogEvents"
44 |       ],
45 |       "Resource": [
46 |         "arn:aws:logs:us-west-2:XXXXXXXXXXXX:log-group:/aws/kinesisfirehose/eks-stream:log-stream:*"
47 |       ]
48 |     },
49 |     {
50 |       "Sid": "",
51 |       "Effect": "Allow",
52 |       "Action": [
53 |         "kinesis:DescribeStream",
54 |         "kinesis:GetShardIterator",
55 |         "kinesis:GetRecords"
56 |       ],
57 |       "Resource": "arn:aws:kinesis:us-west-2:XXXXXXXXXXXX:stream/%FIREHOSE_STREAM_NAME%"
58 |     },
59 |     {
60 |       "Effect": "Allow",
61 |       "Action": [
62 |         "kms:Decrypt"
63 |       ],
64 |       "Resource": [
65 |         "arn:aws:kms:us-west-2:XXXXXXXXXXXX:key/%SSE_KEY_ID%"
66 |       ],
67 |       "Condition": {
68 |         "StringEquals": {
69 |           "kms:ViaService": "kinesis.%REGION_NAME%.amazonaws.com"
70 |         },
71 |         "StringLike": {
72 |           "kms:EncryptionContext:aws:kinesis:arn": "arn:aws:kinesis:%REGION_NAME%:XXXXXXXXXXXX:stream/%FIREHOSE_STREAM_NAME%"
73 |         }
74 |       }
75 |     }
76 |   ]
77 | }
78 | 


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | # Contributing Guidelines
 2 | 
 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional
 4 | documentation, we greatly value feedback and contributions from our community.
 5 | 
 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary
 7 | information to effectively respond to your bug report or contribution.
 8 | 
 9 | 
10 | ## Reporting Bugs/Feature Requests
11 | 
12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features.
13 | 
14 | When filing an issue, please check [existing open](https://github.com/aws-samples/amazon-ecs-fluent-bit-daemon-service/issues), or [recently closed](https://github.com/aws-samples/amazon-ecs-fluent-bit-daemon-service/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already
15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful:
16 | 
17 | * A reproducible test case or series of steps
18 | * The version of our code being used
19 | * Any modifications you've made relevant to the bug
20 | * Anything unusual about your environment or deployment
21 | 
22 | 
23 | ## Contributing via Pull Requests
24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that:
25 | 
26 | 1. You are working against the latest source on the *master* branch.
27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already.
28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted.
29 | 
30 | To send us a pull request, please:
31 | 
32 | 1. Fork the repository.
33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change.
34 | 3. Ensure local tests pass.
35 | 4. Commit to your fork using clear commit messages.
36 | 5. Send us a pull request, answering any default questions in the pull request interface.
37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation.
38 | 
39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and
40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/).
41 | 
42 | 
43 | ## Finding contributions to work on
44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/aws-samples/amazon-ecs-fluent-bit-daemon-service/labels/help%20wanted) issues is a great place to start.
45 | 
46 | 
47 | ## Code of Conduct
48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
50 | opensource-codeofconduct@amazon.com with any additional questions or comments.
51 | 
52 | 
53 | ## Security issue notifications
54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue.
55 | 
56 | 
57 | ## Licensing
58 | 
59 | See the [LICENSE](https://github.com/aws-samples/amazon-ecs-fluent-bit-daemon-service/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
60 | 
61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes.
62 | 


--------------------------------------------------------------------------------
/ecs/ecs-fluent-bit-daemonset.yml:
--------------------------------------------------------------------------------
  1 | Description: >
  2 |     This template deploys a Fluent Bit Daemonset to an ECS Cluster
  3 | Parameters:
  4 |     EnvironmentName:
  5 |         Description: An environment name that will be prefixed to resource names
  6 |         Type: String
  7 | 
  8 |     DockerImage:
  9 |         Description: The Fluent Bit Docker image.
 10 |         Type: String
 11 | 
 12 |     Cluster:
 13 |         Description: Please provide the ECS Cluster name that this service should run on
 14 |         Type: String
 15 | 
 16 | Resources:
 17 |     Service:
 18 |         Type: AWS::ECS::Service
 19 |         Properties:
 20 |             Cluster: !Ref Cluster
 21 |             LaunchType: EC2
 22 |             TaskDefinition: !Ref TaskDefinition
 23 |             SchedulingStrategy: DAEMON
 24 | 
 25 |     TaskDefinition:
 26 |         Type: AWS::ECS::TaskDefinition
 27 |         Properties:
 28 |             Family: fluentd-aggregator
 29 |             TaskRoleArn: !Ref TaskRole
 30 |             NetworkMode: host
 31 |             Volumes:
 32 |                 - Name: socket
 33 |                   Host:
 34 |                     SourcePath: /var/run
 35 |             ContainerDefinitions:
 36 |                 - Name: fluentd-daemon
 37 |                   Essential: true
 38 |                   Environment:
 39 |                       - Name: FLB_LOG_LEVEL
 40 |                         Value: INFO
 41 |                   Image: !Ref DockerImage
 42 |                   MountPoints:
 43 |                       - ContainerPath: /var/run
 44 |                         SourceVolume: socket
 45 |                   MemoryReservation: 50
 46 |                   LogConfiguration:
 47 |                     LogDriver: awslogs
 48 |                     Options:
 49 |                         awslogs-group: !Ref AWS::StackName
 50 |                         awslogs-region: !Ref AWS::Region
 51 |                         awslogs-stream-prefix: !Ref EnvironmentName
 52 | 
 53 |     CloudWatchLogsGroup:
 54 |         Type: AWS::Logs::LogGroup
 55 |         Properties:
 56 |             LogGroupName: !Ref AWS::StackName
 57 |             RetentionInDays: 14
 58 | 
 59 |     # This IAM Role grants the task access to firehose and CloudWatch Logs
 60 |     TaskRole:
 61 |         Type: AWS::IAM::Role
 62 |         Properties:
 63 |             RoleName: !Sub ecs-task-${AWS::StackName}
 64 |             Path: /
 65 |             AssumeRolePolicyDocument: |
 66 |                 {
 67 |                     "Statement": [{
 68 |                         "Sid": "",
 69 |                         "Effect": "Allow",
 70 |                         "Principal": { "Service": [ "ecs-tasks.amazonaws.com" ]},
 71 |                         "Action": "sts:AssumeRole"
 72 |                     }]
 73 |                 }
 74 |             Policies:
 75 |                 - PolicyName: !Sub ecs-task-${AWS::StackName}
 76 |                   PolicyDocument:
 77 |                     {
 78 |                         "Version": "2012-10-17",
 79 |                         "Statement": [
 80 |                             {
 81 |                                 "Effect": "Allow",
 82 |                                 "Action": [
 83 |                                     "firehose:PutRecordBatch"
 84 |                                 ],
 85 |                                 "Resource": "*"
 86 |                             },
 87 |                             {
 88 |                                 "Effect": "Allow",
 89 |                                 "Action": "logs:PutLogEvents",
 90 |                                 "Resource": "arn:aws:logs:*:*:log-group:*:*:*"
 91 |                             },
 92 |                             {
 93 |                                 "Effect": "Allow",
 94 |                                 "Action": [
 95 |                                     "logs:CreateLogStream",
 96 |                                     "logs:DescribeLogStreams",
 97 |                                     "logs:PutLogEvents"
 98 |                                 ],
 99 |                                 "Resource": "arn:aws:logs:*:*:log-group:*"
100 |                             },
101 |                             {
102 |                                 "Effect": "Allow",
103 |                                 "Action": "logs:CreateLogGroup",
104 |                                 "Resource": "*"
105 |                             }
106 |                         ]
107 |                     }
108 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 | 
  2 |                                  Apache License
  3 |                            Version 2.0, January 2004
  4 |                         http://www.apache.org/licenses/
  5 | 
  6 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  7 | 
  8 |    1. Definitions.
  9 | 
 10 |       "License" shall mean the terms and conditions for use, reproduction,
 11 |       and distribution as defined by Sections 1 through 9 of this document.
 12 | 
 13 |       "Licensor" shall mean the copyright owner or entity authorized by
 14 |       the copyright owner that is granting the License.
 15 | 
 16 |       "Legal Entity" shall mean the union of the acting entity and all
 17 |       other entities that control, are controlled by, or are under common
 18 |       control with that entity. For the purposes of this definition,
 19 |       "control" means (i) the power, direct or indirect, to cause the
 20 |       direction or management of such entity, whether by contract or
 21 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 22 |       outstanding shares, or (iii) beneficial ownership of such entity.
 23 | 
 24 |       "You" (or "Your") shall mean an individual or Legal Entity
 25 |       exercising permissions granted by this License.
 26 | 
 27 |       "Source" form shall mean the preferred form for making modifications,
 28 |       including but not limited to software source code, documentation
 29 |       source, and configuration files.
 30 | 
 31 |       "Object" form shall mean any form resulting from mechanical
 32 |       transformation or translation of a Source form, including but
 33 |       not limited to compiled object code, generated documentation,
 34 |       and conversions to other media types.
 35 | 
 36 |       "Work" shall mean the work of authorship, whether in Source or
 37 |       Object form, made available under the License, as indicated by a
 38 |       copyright notice that is included in or attached to the work
 39 |       (an example is provided in the Appendix below).
 40 | 
 41 |       "Derivative Works" shall mean any work, whether in Source or Object
 42 |       form, that is based on (or derived from) the Work and for which the
 43 |       editorial revisions, annotations, elaborations, or other modifications
 44 |       represent, as a whole, an original work of authorship. For the purposes
 45 |       of this License, Derivative Works shall not include works that remain
 46 |       separable from, or merely link (or bind by name) to the interfaces of,
 47 |       the Work and Derivative Works thereof.
 48 | 
 49 |       "Contribution" shall mean any work of authorship, including
 50 |       the original version of the Work and any modifications or additions
 51 |       to that Work or Derivative Works thereof, that is intentionally
 52 |       submitted to Licensor for inclusion in the Work by the copyright owner
 53 |       or by an individual or Legal Entity authorized to submit on behalf of
 54 |       the copyright owner. For the purposes of this definition, "submitted"
 55 |       means any form of electronic, verbal, or written communication sent
 56 |       to the Licensor or its representatives, including but not limited to
 57 |       communication on electronic mailing lists, source code control systems,
 58 |       and issue tracking systems that are managed by, or on behalf of, the
 59 |       Licensor for the purpose of discussing and improving the Work, but
 60 |       excluding communication that is conspicuously marked or otherwise
 61 |       designated in writing by the copyright owner as "Not a Contribution."
 62 | 
 63 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 64 |       on behalf of whom a Contribution has been received by Licensor and
 65 |       subsequently incorporated within the Work.
 66 | 
 67 |    2. Grant of Copyright License. Subject to the terms and conditions of
 68 |       this License, each Contributor hereby grants to You a perpetual,
 69 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 70 |       copyright license to reproduce, prepare Derivative Works of,
 71 |       publicly display, publicly perform, sublicense, and distribute the
 72 |       Work and such Derivative Works in Source or Object form.
 73 | 
 74 |    3. Grant of Patent License. Subject to the terms and conditions of
 75 |       this License, each Contributor hereby grants to You a perpetual,
 76 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 77 |       (except as stated in this section) patent license to make, have made,
 78 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 79 |       where such license applies only to those patent claims licensable
 80 |       by such Contributor that are necessarily infringed by their
 81 |       Contribution(s) alone or by combination of their Contribution(s)
 82 |       with the Work to which such Contribution(s) was submitted. If You
 83 |       institute patent litigation against any entity (including a
 84 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 85 |       or a Contribution incorporated within the Work constitutes direct
 86 |       or contributory patent infringement, then any patent licenses
 87 |       granted to You under this License for that Work shall terminate
 88 |       as of the date such litigation is filed.
 89 | 
 90 |    4. Redistribution. You may reproduce and distribute copies of the
 91 |       Work or Derivative Works thereof in any medium, with or without
 92 |       modifications, and in Source or Object form, provided that You
 93 |       meet the following conditions:
 94 | 
 95 |       (a) You must give any other recipients of the Work or
 96 |           Derivative Works a copy of this License; and
 97 | 
 98 |       (b) You must cause any modified files to carry prominent notices
 99 |           stating that You changed the files; and
100 | 
101 |       (c) You must retain, in the Source form of any Derivative Works
102 |           that You distribute, all copyright, patent, trademark, and
103 |           attribution notices from the Source form of the Work,
104 |           excluding those notices that do not pertain to any part of
105 |           the Derivative Works; and
106 | 
107 |       (d) If the Work includes a "NOTICE" text file as part of its
108 |           distribution, then any Derivative Works that You distribute must
109 |           include a readable copy of the attribution notices contained
110 |           within such NOTICE file, excluding those notices that do not
111 |           pertain to any part of the Derivative Works, in at least one
112 |           of the following places: within a NOTICE text file distributed
113 |           as part of the Derivative Works; within the Source form or
114 |           documentation, if provided along with the Derivative Works; or,
115 |           within a display generated by the Derivative Works, if and
116 |           wherever such third-party notices normally appear. The contents
117 |           of the NOTICE file are for informational purposes only and
118 |           do not modify the License. You may add Your own attribution
119 |           notices within Derivative Works that You distribute, alongside
120 |           or as an addendum to the NOTICE text from the Work, provided
121 |           that such additional attribution notices cannot be construed
122 |           as modifying the License.
123 | 
124 |       You may add Your own copyright statement to Your modifications and
125 |       may provide additional or different license terms and conditions
126 |       for use, reproduction, or distribution of Your modifications, or
127 |       for any such Derivative Works as a whole, provided Your use,
128 |       reproduction, and distribution of the Work otherwise complies with
129 |       the conditions stated in this License.
130 | 
131 |    5. Submission of Contributions. Unless You explicitly state otherwise,
132 |       any Contribution intentionally submitted for inclusion in the Work
133 |       by You to the Licensor shall be under the terms and conditions of
134 |       this License, without any additional terms or conditions.
135 |       Notwithstanding the above, nothing herein shall supersede or modify
136 |       the terms of any separate license agreement you may have executed
137 |       with Licensor regarding such Contributions.
138 | 
139 |    6. Trademarks. This License does not grant permission to use the trade
140 |       names, trademarks, service marks, or product names of the Licensor,
141 |       except as required for reasonable and customary use in describing the
142 |       origin of the Work and reproducing the content of the NOTICE file.
143 | 
144 |    7. Disclaimer of Warranty. Unless required by applicable law or
145 |       agreed to in writing, Licensor provides the Work (and each
146 |       Contributor provides its Contributions) on an "AS IS" BASIS,
147 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 |       implied, including, without limitation, any warranties or conditions
149 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 |       PARTICULAR PURPOSE. You are solely responsible for determining the
151 |       appropriateness of using or redistributing the Work and assume any
152 |       risks associated with Your exercise of permissions under this License.
153 | 
154 |    8. Limitation of Liability. In no event and under no legal theory,
155 |       whether in tort (including negligence), contract, or otherwise,
156 |       unless required by applicable law (such as deliberate and grossly
157 |       negligent acts) or agreed to in writing, shall any Contributor be
158 |       liable to You for damages, including any direct, indirect, special,
159 |       incidental, or consequential damages of any character arising as a
160 |       result of this License or out of the use or inability to use the
161 |       Work (including but not limited to damages for loss of goodwill,
162 |       work stoppage, computer failure or malfunction, or any and all
163 |       other commercial damages or losses), even if such Contributor
164 |       has been advised of the possibility of such damages.
165 | 
166 |    9. Accepting Warranty or Additional Liability. While redistributing
167 |       the Work or Derivative Works thereof, You may choose to offer,
168 |       and charge a fee for, acceptance of support, warranty, indemnity,
169 |       or other liability obligations and/or rights consistent with this
170 |       License. However, in accepting such obligations, You may act only
171 |       on Your own behalf and on Your sole responsibility, not on behalf
172 |       of any other Contributor, and only if You agree to indemnify,
173 |       defend, and hold each Contributor harmless for any liability
174 |       incurred by, or claims asserted against, such Contributor by reason
175 |       of your accepting any such warranty or additional liability.
176 | 


--------------------------------------------------------------------------------