├── fluent-bit-service-account.yaml ├── fluent-bit-role.yaml ├── fluent-bit-role-binding.yaml ├── readme.md ├── deploy.sh ├── fluent-bit-daemon-set.yaml └── fluent-bit-configmap.yaml /fluent-bit-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: fluent-bit 5 | namespace: logging 6 | 7 | -------------------------------------------------------------------------------- /fluent-bit-role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1beta1 2 | kind: ClusterRole 3 | metadata: 4 | name: fluent-bit-read 5 | rules: 6 | - apiGroups: [""] 7 | resources: 8 | - namespaces 9 | - pods 10 | verbs: ["get", "list", "watch"] 11 | 12 | -------------------------------------------------------------------------------- /fluent-bit-role-binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1beta1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: fluent-bit-read 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: fluent-bit-read 9 | subjects: 10 | - kind: ServiceAccount 11 | name: fluent-bit 12 | namespace: logging 13 | 14 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Fluent Bit Configuration for K8s and Graylog 2 | 3 | > See [this blog post](https://vzurczak.wordpress.com/?p=781) for more details. 4 | 5 | A K8s configuration to install and configure Fluent Bit as a daemon set. 6 | Fluent Bit collects only Docker logs, gets K8s metadata, builds a GEF message 7 | and sends it to a Graylog server. 8 | 9 | * Update the **fluent-bit-configmap.yaml** file. 10 | Replace **192.168.1.18** with the IP address of your Graylog server. 11 | * Then execute the **deploy.sh** script. 12 | 13 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # (Found at https://github.com/fluent/fluent-bit-kubernetes-logging and updated) 4 | # Create the namespace 5 | kubectl create namespace logging 6 | 7 | # Create the base resources 8 | kubectl create -f fluent-bit-service-account.yaml 9 | kubectl create -f fluent-bit-role.yaml 10 | kubectl create -f fluent-bit-role-binding.yaml 11 | 12 | # Create the config map 13 | # See https://github.com/fluent/fluent-bit/issues/851 for the GELF parameters. 14 | kubectl create -f fluent-bit-configmap.yaml 15 | 16 | # Create the daemon set 17 | kubectl create -f fluent-bit-daemon-set.yaml 18 | 19 | -------------------------------------------------------------------------------- /fluent-bit-daemon-set.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: fluent-bit 5 | namespace: logging 6 | labels: 7 | k8s-app: fluent-bit-logging 8 | version: v1 9 | kubernetes.io/cluster-service: "true" 10 | spec: 11 | selector: 12 | matchLabels: 13 | k8s-app: fluent-bit-logging 14 | version: v1 15 | kubernetes.io/cluster-service: "true" 16 | template: 17 | metadata: 18 | labels: 19 | k8s-app: fluent-bit-logging 20 | version: v1 21 | kubernetes.io/cluster-service: "true" 22 | annotations: 23 | prometheus.io/scrape: "true" 24 | prometheus.io/port: "2020" 25 | prometheus.io/path: /api/v1/metrics/prometheus 26 | spec: 27 | containers: 28 | - name: fluent-bit 29 | image: fluent/fluent-bit:1.0.1 30 | imagePullPolicy: Always 31 | ports: 32 | - containerPort: 2020 33 | volumeMounts: 34 | - name: varlog 35 | mountPath: /var/log 36 | - name: varlibdockercontainers 37 | mountPath: /var/lib/docker/containers 38 | readOnly: true 39 | - name: fluent-bit-config 40 | mountPath: /fluent-bit/etc/ 41 | - name: mnt 42 | mountPath: /mnt 43 | readOnly: true 44 | terminationGracePeriodSeconds: 10 45 | volumes: 46 | - name: varlog 47 | hostPath: 48 | path: /var/log 49 | - name: varlibdockercontainers 50 | hostPath: 51 | path: /var/lib/docker/containers 52 | - name: fluent-bit-config 53 | configMap: 54 | name: fluent-bit-config 55 | - name: mnt 56 | hostPath: 57 | path: /mnt 58 | serviceAccountName: fluent-bit 59 | tolerations: 60 | - key: node-role.kubernetes.io/master 61 | operator: Exists 62 | effect: NoSchedule 63 | 64 | -------------------------------------------------------------------------------- /fluent-bit-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: fluent-bit-config 5 | namespace: logging 6 | labels: 7 | k8s-app: fluent-bit 8 | data: 9 | # Configuration files: server, input, filters and output 10 | # ====================================================== 11 | fluent-bit.conf: | 12 | [SERVICE] 13 | Flush 1 14 | Log_Level info 15 | Daemon off 16 | Parsers_File parsers.conf 17 | HTTP_Server On 18 | HTTP_Listen 0.0.0.0 19 | HTTP_Port 2020 20 | 21 | @INCLUDE input-kubernetes.conf 22 | @INCLUDE filter-kubernetes.conf 23 | @INCLUDE output-graylog.conf 24 | 25 | input-kubernetes.conf: | 26 | [INPUT] 27 | Name tail 28 | Tag kube.* 29 | Path /var/log/containers/*.log 30 | Parser docker 31 | DB /var/log/flb_kube.db 32 | Mem_Buf_Limit 5MB 33 | Skip_Long_Lines On 34 | Refresh_Interval 10 35 | 36 | filter-kubernetes.conf: | 37 | [FILTER] 38 | Name kubernetes 39 | Match kube.* 40 | Kube_URL https://kubernetes.default.svc.cluster.local:443 41 | Merge_Log On 42 | K8S-Logging.Parser On 43 | 44 | # ${HOSTNAME} returns the host name. 45 | # But Fluentbit runs in a container. So, it is not meaningful. 46 | # Instead, copy the host name from the Kubernetes object. 47 | [FILTER] 48 | Name nest 49 | Match * 50 | Operation lift 51 | Nested_under kubernetes 52 | 53 | # Remove offending fields, see: https://github.com/fluent/fluent-bit/issues/1291 54 | [FILTER] 55 | Name record_modifier 56 | Match * 57 | Remove_key annotations 58 | Remove_key labels 59 | 60 | output-graylog.conf: | 61 | [OUTPUT] 62 | Name gelf 63 | Match * 64 | Host 192.168.1.18 65 | Port 12201 66 | Mode tcp 67 | Gelf_Short_Message_Key log 68 | 69 | parsers.conf: | 70 | [PARSER] 71 | Name json 72 | Format json 73 | Time_Key time 74 | Time_Format %d/%b/%Y:%H:%M:%S %z 75 | 76 | [PARSER] 77 | Name docker 78 | Format json 79 | Time_Key time 80 | Time_Format %Y-%m-%dT%H:%M:%S.%L 81 | # Command | Decoder | Field | Optional Action 82 | # =============|==================|================= 83 | Decode_Field_As escaped log 84 | 85 | [PARSER] 86 | Name syslog 87 | Format regex 88 | Regex ^\<(?[0-9]+)\>(?