├── .gitignore
├── README.md
├── fluent-bit-sidecar.yaml
└── init
    ├── Dockerfile
    ├── fluent-bit.conf
    ├── fluent.conf
    └── parsers.conf


/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Fluent-Bit Sidecar for Kubernetes
 2 | 
 3 | Run [Fluent-Bit](http://fluentbit.io/) as a sidecar to collect logs and output them to elasticsearch in a Kubernetes cluster. Fluent-Bit is configured in this example to tail a named directory (for the example: /mnt/log/reference-logging.txt) and collect all logs from the file.
 4 | 
 5 | This example includes a log generator app that runs with Fluent-Bit in one pod and writes logs to the named directory. 
 6 | 
 7 | ## Usage
 8 | 
 9 | To deploy in a Kubernetes cluster:
10 | 
11 | ```kubectl -f create fluent-bit-sidecar.yaml```
12 | 
13 | [Elasticsearch for Kubernetes](https://github.com/kubernetes/kubernetes/tree/master/examples/elasticsearch)
14 | 
15 | ## History
16 | 
17 | Previously we used FluentD as a log collector and are experimenting with this light-weight Fluent-Bit option. This sidecar example is useful for applications that don't just write logs to STDERR/STDOUT and instead/additionally write logs to a named directory. More details on our experience to come. 


--------------------------------------------------------------------------------
/fluent-bit-sidecar.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: v1
 2 | kind: Pod
 3 | metadata:
 4 |   name: log-app
 5 |   labels:
 6 |     app: log-app
 7 | spec:
 8 |   containers:
 9 |   - name: log-app
10 |     image: quay.io/leahnp/leah_log_app:latest
11 |     ports:
12 |     - containerPort: 8080
13 |     volumeMounts:
14 |     - name: log-storage
15 |       mountPath: /var/log
16 |   - name: sidecar-log-collector
17 |     image: quay.io/leahnp/testing_sidecar:latest
18 |     resources:
19 |       limits:
20 |         cpu: 100m
21 |         memory: 200Mi
22 |     volumeMounts:
23 |     - name: log-storage
24 |       readOnly: true
25 |       mountPath: /mnt/log
26 |   volumes:
27 |   - name: log-storage
28 |     emptyDir: {}


--------------------------------------------------------------------------------
/init/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM gcr.io/google_containers/ubuntu-slim:0.6
 2 | ENV DEBIAN_FRONTEND noninteractive
 3 | 
 4 | # Fluent Bit version
 5 | ENV FLB_MAJOR 0
 6 | ENV FLB_MINOR 11
 7 | ENV FLB_PATCH 0
 8 | ENV FLB_KUBE  2
 9 | ENV FLB_VERSION 0.11.0
10 | 
11 | MAINTAINER Eduardo Silva <eduardo@treasure-data.com>
12 | LABEL Description="Fluent Bit docker image" Vendor="Fluent Organization" Version="1.1"
13 | USER root
14 | 
15 | # Install build tools
16 | RUN apt-get -qq update && \
17 |     apt-get install -y -qq curl ca-certificates build-essential cmake iputils-ping dnsutils make bash sudo wget unzip nano vim valgrind  && \
18 |     apt-get install -y -qq --reinstall lsb-base lsb-release && \
19 |     wget -O "/tmp/fluent-bit-$FLB_VERSION-dev.zip" "http://github.com/fluent/fluent-bit/archive/master.zip" && \
20 |     cd /tmp && \
21 |     unzip "fluent-bit-$FLB_VERSION-dev.zip" && \
22 |     cd "fluent-bit-master/build/" && \
23 |     cmake -DFLB_DEBUG=On -DFLB_TRACE=On \
24 |           -DCMAKE_INSTALL_PREFIX=/fluent-bit/ -DFLB_JEMALLOC=On ../&& \
25 |     make && make install && \
26 |     rm -rf /tmp/* /fluent-bit/include /fluent-bit/lib*
27 | 
28 | COPY fluent-bit.conf /fluent-bit/etc/
29 | COPY parsers.conf /fluent-bit/etc/
30 | 
31 | CMD ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/etc/fluent-bit.conf", "-vv"]


--------------------------------------------------------------------------------
/init/fluent-bit.conf:
--------------------------------------------------------------------------------
 1 | [SERVICE]
 2 |     Flush        1
 3 |     Daemon       Off
 4 |     Log_Level    info
 5 |     Parsers_File parsers.conf
 6 | 
 7 | [INPUT]
 8 |     Name   tail
 9 |     Path   /mnt/log/reference-logging.txt
10 |     Parser        docker
11 |     Tag           kube.*
12 |     Mem_Buf_Limit 10MB
13 | 
14 | [FILTER]
15 |     Name   kubernetes
16 |     Match  kube.**
17 | 
18 | [OUTPUT]
19 |     Name  es
20 |     Match *
21 |     Host  ${FLUENT_ELASTICSEARCH_HOST}
22 |     Port  ${FLUENT_ELASTICSEARCH_PORT}
23 |     Logstash_Format On


--------------------------------------------------------------------------------
/init/fluent.conf:
--------------------------------------------------------------------------------
 1 | [SERVICE]
 2 |     Flush        1
 3 |     Daemon       Off
 4 |     Log_Level    debug
 5 |     Parsers_File parsers.conf
 6 | 
 7 | [INPUT]
 8 |     Name   tail
 9 |     Path   /mnt/log/reference-logging.txt
10 |     Parser        docker
11 |     Tag           kube.*
12 |     Mem_Buf_Limit 10MB
13 | 
14 | [FILTER]
15 |     Name   kubernetes
16 |     Match  kube.**
17 | 
18 | [OUTPUT]
19 |     Name  es
20 |     Match *
21 |     Host  ${FLUENT_ELASTICSEARCH_HOST}
22 |     Port  ${FLUENT_ELASTICSEARCH_PORT}
23 |     Logstash_Format On


--------------------------------------------------------------------------------
/init/parsers.conf:
--------------------------------------------------------------------------------
 1 | [PARSER]
 2 |     Name   apache
 3 |     Format regex
 4 |     Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
 5 |     Time_Key time
 6 |     Time_Format %d/%b/%Y:%H:%M:%S %z
 7 | 
 8 | [PARSER]
 9 |     Name   apache2
10 |     Format regex
11 |     Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
12 |     Time_Key time
13 |     Time_Format %d/%b/%Y:%H:%M:%S %z
14 | 
15 | [PARSER]
16 |     Name   apache_error
17 |     Format regex
18 |     Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
19 | 
20 | [PARSER]
21 |     Name   nginx
22 |     Format regex
23 |     Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
24 |     Time_Key time
25 |     Time_Format %d/%b/%Y:%H:%M:%S %z
26 | 
27 | [PARSER]
28 |     Name   json-test
29 |     Format json
30 |     Time_Key time
31 |     Time_Format %d/%b/%Y:%H:%M:%S %z
32 | 
33 | [PARSER]
34 |     Name        docker
35 |     Format      json
36 |     Time_Key    time
37 |     Time_Format %Y-%m-%dT%H:%M:%S
38 |     Time_Keep   On
39 | 
40 | [PARSER]
41 |     Name        syslog
42 |     Format      regex
43 |     Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
44 |     Time_Key    time
45 |     Time_Format %b %d %H:%M:%S


--------------------------------------------------------------------------------