├── .dockerignore
├── .gitignore
├── Dockerfile
├── README.md
├── build.sh
├── delete.sh
├── deploy.sh
├── display.sh
├── fluent.conf
├── fluentd-cloudwatch-daemonset-template.yaml
└── setversion.sh


/.dockerignore:
--------------------------------------------------------------------------------
1 | *~
2 | *.env
3 | *.yaml
4 | *.sh
5 | README.md
6 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *~
2 | *.env
3 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM fluent/fluentd:latest
 2 | MAINTAINER Eljas Alakulppi <eljas@callstats.io>
 3 | WORKDIR /home/fluent
 4 | ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH
 5 | 
 6 | USER root
 7 | RUN apk --no-cache --update add sudo build-base ruby-dev && \
 8 |     sudo -u fluent gem install fluent-plugin-cloudwatch-logs && \
 9 |     sudo -u fluent gem install fluent-plugin-kubernetes_metadata_filter && \
10 |     rm -rf /home/fluent/.gem/ruby/2.3.0/cache/*.gem && sudo -u fluent gem sources -c && \
11 |     apk del sudo build-base ruby-dev && rm -rf /var/cache/apk/*
12 | 
13 | EXPOSE 24284
14 | 
15 | ADD fluent.conf /fluentd/etc/
16 | CMD exec fluentd -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_OPT
17 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Fluentd image to send Kubernetes logs to CloudWatch
 2 | 
 3 | ## Deployment
 4 | 
 5 | 1. Create AWS IAM user which has permission to store logs to CloudWatch
 6 | 
 7 | 2. Optionally set environment variable to change the default resource names and labels
 8 | 
 9 | - `NAMESPACE` defaults to 'kube-system'
10 | - `APP_NAME` defaults to 'fluentd-cloudwatch'
11 | - `SECRET_NAME`,`CONFIG_NAME` default to `APP_NAME`
12 | - `ENV_NAME` defaults to 'system'
13 | - `CW_LOG_GROUP` defaults to 'kubernetes-cluster'
14 | 
15 | 3. Set environment variables for the AWS IAM user and CloudWatch region or set them just when running `deploy.sh` in the next step
16 | ```
17 | export LOGGING_AWS_ACCESS_KEY_ID=<your key>
18 | export LOGGING_AWS_SECRET_ACCESS_KEY=<your secret>
19 | export LOGGING_AWS_REGION=<your region>
20 | ```
21 | 
22 | 4. Run `deploy.sh` to create the Secret, ConfigMap, and DaemonSet, setting the AWS IAM user and CloudWatch region environments variables if you did not export them in the previous step.
23 | ```
24 | LOGGING_AWS_ACCESS_KEY_ID=<your key> LOGGING_AWS_SECRET_ACCESS_KEY=<your secret> LOGGING_AWS_REGION=<your region> ./deploy.sh
25 | ```
26 | 
27 | 5. Run `display.sh` to check everything is running
28 | 
29 | ## Removal
30 | 
31 | 1. If you set custom values for the namespace or resource name environment variables, 
32 | ensure they are still set to your values (`NAMESPACE`,`APP_NAME`,`SECRET_NAME`,`CONFIG_NAME`)
33 | 
34 | 2. Run `delete.sh`
35 | 
36 | 3. Run `display.sh` to check nothing is left
37 | 


--------------------------------------------------------------------------------
/build.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | docker build . -t fluentd-cloudwatch
4 | 


--------------------------------------------------------------------------------
/delete.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | set -e
 3 | 
 4 | #
 5 | # Delete Secret, ConfigMap, and DaemonSet for fluentd-cloudwatch
 6 | #
 7 | 
 8 | : ${APP_NAME:=fluentd-cloudwatch}
 9 | : ${SECRET_NAME:=$APP_NAME}
10 | : ${CONFIG_NAME:=$APP_NAME}
11 | : ${NAMESPACE:=kube-system}
12 | 
13 | kubectl delete secret $SECRET_NAME --namespace=$NAMESPACE
14 | kubectl delete configmap $CONFIG_NAME --namespace=$NAMESPACE
15 | kubectl delete daemonset $APP_NAME --namespace=$NAMESPACE
16 | 


--------------------------------------------------------------------------------
/deploy.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | set -e
 3 | 
 4 | #
 5 | # Create Secret, ConfigMap, and DaemonSet for fluentd-cloudwatch
 6 | #
 7 | 
 8 | : ${LOGGING_AWS_ACCESS_KEY_ID?"Must define LOGGING_AWS_ACCESS_KEY_ID"}
 9 | : ${LOGGING_AWS_SECRET_ACCESS_KEY?"Must define LOGGING_AWS_SECRET_ACCESS_KEY"}
10 | : ${LOGGING_AWS_REGION?"Must define LOGGING_AWS_REGION"}
11 | 
12 | : ${APP_NAME:=fluentd-cloudwatch}
13 | : ${SECRET_NAME:=$APP_NAME}
14 | : ${CONFIG_NAME:=$APP_NAME}
15 | : ${ENV_NAME:=system}
16 | : ${NAMESPACE:=kube-system}
17 | : ${IMAGE_NAME:=callstats/fluentd-kubernetes-cloudwatch:v1.1.2}
18 | : ${CW_LOG_GROUP:=kubernetes-cluster}
19 | 
20 | #
21 | # Secret
22 | #
23 | 
24 | if [[ "$(kubectl get secret $SECRET_NAME --namespace=$NAMESPACE --output name 2> /dev/null || true)" = "secret/${SECRET_NAME}" ]]; then
25 |   ACTION=replace
26 | else
27 |   ACTION=create
28 | fi
29 | 
30 | kubectl $ACTION -f - <<END
31 | apiVersion: v1
32 | kind: Secret
33 | type: kubernetes.io/opaque
34 | metadata:
35 |   name: $SECRET_NAME
36 |   namespace: $NAMESPACE
37 |   labels:
38 |     app: $APP_NAME
39 |     env: $ENV_NAME
40 | data:
41 |   AWS_ACCESS_KEY_ID: $(echo -n "${LOGGING_AWS_ACCESS_KEY_ID}" | base64 -w 0)
42 |   AWS_SECRET_ACCESS_KEY: $(echo -n "${LOGGING_AWS_SECRET_ACCESS_KEY}" | base64 -w 0)
43 | END
44 | 
45 | #
46 | # ConfigMap
47 | #
48 | 
49 | kubectl apply -f - <<END
50 | apiVersion: v1
51 | kind: ConfigMap
52 | metadata:
53 |   name: $CONFIG_NAME
54 |   namespace: $NAMESPACE
55 |   labels:
56 |     app: $APP_NAME
57 |     env: $ENV_NAME
58 | data:
59 |   AWS_REGION: $LOGGING_AWS_REGION
60 |   CW_LOG_GROUP: $CW_LOG_GROUP
61 | END
62 | 
63 | #
64 | # DaemonSet
65 | #
66 | 
67 | export APP_NAME ENV_NAME CONFIG_NAME SECRET_NAME NAMESPACE IMAGE_NAME
68 | envsubst < fluentd-cloudwatch-daemonset-template.yaml | kubectl apply -f -
69 | 


--------------------------------------------------------------------------------
/display.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Usage:
 4 | #   ./Display.sh
 5 | #   ./Display.sh -o yaml
 6 | 
 7 | : ${APP_NAME:=fluentd-cloudwatch}
 8 | : ${NAMESPACE:=kube-system}
 9 | 
10 | kubectl get secret,configmap,daemonset,pod --selector=app=$APP_NAME --namespace=$NAMESPACE $@
11 | 


--------------------------------------------------------------------------------
/fluent.conf:
--------------------------------------------------------------------------------
 1 | <match fluent.**>
 2 |   @type null
 3 | </match>
 4 | 
 5 | <source>
 6 |   @type tail
 7 |   path /var/log/containers/*.log
 8 |   pos_file /var/log/es-containers.log.pos
 9 |   time_format %Y-%m-%dT%H:%M:%S.%NZ
10 |   tag kubernetes.*
11 |   format json
12 |   read_from_head true
13 | </source>
14 | 
15 | <filter kubernetes.**>
16 |   @type kubernetes_metadata
17 | </filter>
18 | 
19 | <filter kubernetes.**>
20 |   @type record_transformer
21 |   enable_ruby true
22 |   <record>
23 |     kubehost ${record.fetch("kubernetes", Hash.new).fetch("host", "unknown_host")}
24 |   </record>
25 | </filter>
26 | 
27 | 
28 | <match kubernetes.**>
29 |   @type cloudwatch_logs
30 |   log_group_name "#{ENV['CW_LOG_GROUP']}"
31 |   log_stream_name_key kubehost
32 |   remove_log_group_name_key true
33 |   auto_create_stream true
34 |   put_log_events_retry_limit 20
35 | </match>
36 | 


--------------------------------------------------------------------------------
/fluentd-cloudwatch-daemonset-template.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: extensions/v1beta1
 2 | kind: DaemonSet
 3 | metadata:
 4 |   name: $APP_NAME
 5 |   namespace: $NAMESPACE
 6 |   labels:
 7 |     app: $APP_NAME
 8 |     env: $ENV_NAME
 9 |     k8s-app: fluentd-logging
10 |     version: v1
11 |     kubernetes.io/cluster-service: "true"
12 | spec:
13 |   template:
14 |     metadata:
15 |       labels:
16 |         app: $APP_NAME
17 |         env: $ENV_NAME
18 |         k8s-app: fluentd-logging
19 |         version: v1
20 |         kubernetes.io/cluster-service: "true"
21 |     spec:
22 |       containers:
23 |       - name: $APP_NAME
24 |         image: $IMAGE_NAME
25 |         imagePullPolicy: Always
26 |         resources:
27 |           limits:
28 |             memory: 200Mi
29 |           requests:
30 |             cpu: 100m
31 |             memory: 200Mi
32 |         env:
33 |           - name: AWS_ACCESS_KEY_ID
34 |             valueFrom:
35 |               secretKeyRef:
36 |                 name: $SECRET_NAME
37 |                 key: AWS_ACCESS_KEY_ID
38 |           - name: AWS_SECRET_ACCESS_KEY
39 |             valueFrom:
40 |               secretKeyRef:
41 |                 name: $SECRET_NAME
42 |                 key: AWS_SECRET_ACCESS_KEY
43 |           - name: AWS_REGION
44 |             valueFrom:
45 |               configMapKeyRef:
46 |                 name: $CONFIG_NAME
47 |                 key: AWS_REGION
48 |           - name: CW_LOG_GROUP
49 |             valueFrom:
50 |               configMapKeyRef:
51 |                 name: $CONFIG_NAME
52 |                 key: CW_LOG_GROUP
53 |         volumeMounts:
54 |         - name: varlog
55 |           mountPath: /var/log
56 |         - name: varlibdockercontainers
57 |           mountPath: /var/lib/docker/containers
58 |           readOnly: true
59 |       terminationGracePeriodSeconds: 30
60 |       volumes:
61 |       - name: varlog
62 |         hostPath:
63 |           path: /var/log
64 |       - name: varlibdockercontainers
65 |         hostPath:
66 |           path: /var/lib/docker/containers
67 | 


--------------------------------------------------------------------------------
/setversion.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Update the container version tag in scripts and documentation
 4 | 
 5 | : ${1?Must supply a version number (e.g. $0 1.2.3)}
 6 | 
 7 | exp='s/\(callstats\/fluentd-kubernetes-cloudwatch:\)[0-9][0-9.]\+/\1'"$1"'/g'
 8 | 
 9 | sed -i -e "$exp" README.md *.sh
10 | 


--------------------------------------------------------------------------------