├── .gitignore
├── README.md
├── _config.yml
├── eck.yaml
├── fluent-bit-configmap.yaml
├── fluent-bit-ds.yaml
├── fluent-bit-role-binding.yaml
├── fluent-bit-role.yaml
├── fluent-bit-service-account.yaml
└── namespace.yaml


/.gitignore:
--------------------------------------------------------------------------------
1 | /.idea/
2 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # K8S ECK Logging
 2 | 
 3 | This walk-through guides you to setup an in-cluster Elasticsearch and Kibana suite, with cluster-level logging data gathered by Fluent Bit. You may access and search logs from every pod in cluster, as long as the workload in pod writes log to `stdout` or `stderr`.
 4 | 
 5 | # Prerequisites
 6 | 
 7 | * Kubernetes 1.11 or higher (minikube not working)
 8 | * Predefined storage class called `hdd-ssd` (you may change it in `eck.yaml`)
 9 | 
10 | # Deployment Steps
11 | 
12 | Clone [this repo](https://github.com/nanmu42/k8s-eck-logging) to get necessary yaml files.
13 | 
14 | ## Elasticsearch and Kibana
15 | 
16 | [Elastic Cloud on Kubernetes(ECK)](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-overview.html) is now generally available. ECK makes it easy to deploy Elasticsearch and Kibana on k8s with various topology.
17 | 
18 | * Deploy ECK
19 | 
20 | ```bash
21 | kubectl apply -f https://download.elastic.co/downloads/eck/1.0.1/all-in-one.yaml
22 | ```
23 | 
24 | * Create Namespace `logging`
25 | 
26 | ```bash
27 | kubectl create -f ./namespace.yml
28 | ```
29 | 
30 | * Deploy Elasticsearch and Kibana
31 | 
32 | ```bash
33 | kubectl create -f ./eck.yml
34 | ```
35 | 
36 | ## Fluent Bit
37 | 
38 | FluentBit runs as DaemonSet on every node in cluster, gathering logs from every workload. FluentBit attach metadata like pod name and label to logs delivered to Elasticsearch.
39 | 
40 | Well-structured log(in JSON) can be searched/filtered by term in Elasticsearch.
41 | 
42 | ```bash
43 | kubectl create -f fluent-bit-service-account.yaml
44 | kubectl create -f fluent-bit-role.yaml
45 | kubectl create -f fluent-bit-role-binding.yaml
46 | kubectl create -f fluent-bit-configmap.yaml
47 | kubectl create -f fluent-bit-ds.yaml
48 | ```
49 | 
50 | And off you go.
51 | 
52 | # Reference
53 | 
54 | * [Elastic Cloud on Kubernetes](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-quickstart.html)
55 | * [Kubernetes Logging with Fluent Bit](https://github.com/fluent/fluent-bit-kubernetes-logging)
56 | * [Fluent Bit Manual](https://docs.fluentbit.io/manual/output/elasticsearch)
57 | * [Enabling Native Realms on ECK](https://github.com/elastic/cloud-on-k8s/issues/2036#issuecomment-544838578)
58 | * [Fluent Bit: Elasticsearch output should probably not use a type (flb_type) ](https://github.com/fluent/fluent-bit/issues/1359#issuecomment-553228448)
59 | 


--------------------------------------------------------------------------------
/_config.yml:
--------------------------------------------------------------------------------
1 | theme: jekyll-theme-cayman


--------------------------------------------------------------------------------
/eck.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: elasticsearch.k8s.elastic.co/v1beta1
 2 | kind: Elasticsearch
 3 | metadata:
 4 |   namespace: logging
 5 |   name: logging-elastic
 6 | spec:
 7 |   version: 7.4.2
 8 |   http:
 9 |     tls:
10 |       selfSignedCertificate:
11 |         disabled: true
12 |   nodeSets:
13 |     - name: logging-elastic
14 |       count: 1
15 |       podTemplate:
16 |         spec:
17 |           initContainers:
18 |             - name: sysctl
19 |               securityContext:
20 |                 privileged: true
21 |               command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
22 |           containers:
23 |             - name: elasticsearch
24 |               env:
25 |                 - name: ES_JAVA_OPTS
26 |                   value: "-Xms1536m -Xmx1536m"
27 |               resources:
28 |                 requests:
29 |                   memory: 2Gi
30 |                   cpu: 0.5
31 |                 limits:
32 |                   memory: 3Gi
33 |                   cpu: 2
34 |       volumeClaimTemplates:
35 |         - metadata:
36 |             name: elasticsearch-data
37 |           spec:
38 |             accessModes:
39 |               - ReadWriteOnce
40 |             resources:
41 |               requests:
42 |                 storage: 200Gi
43 |             storageClassName: hdd-ssd
44 |       config:
45 |         node.master: true
46 |         node.data: true
47 |         node.ingest: true
48 |         xpack.monitoring:
49 |           collection.interval: 30s
50 |           history.duration: 2d
51 |         xpack.security.authc:
52 |           realms:
53 |             native:
54 |               native1:
55 |                 order: -101
56 |             file:
57 |               file1:
58 |                 order: -100
59 | 
60 | ---
61 | 
62 | apiVersion: kibana.k8s.elastic.co/v1beta1
63 | kind: Kibana
64 | metadata:
65 |   namespace: logging
66 |   name: logging-kibana
67 | spec:
68 |   version: 7.4.2
69 |   http:
70 |     tls:
71 |       selfSignedCertificate:
72 |         disabled: true
73 |   count: 1
74 |   elasticsearchRef:
75 |     name: logging-elastic
76 |   podTemplate:
77 |     spec:
78 |       containers:
79 |         - name: kibana
80 |           env:
81 |             - name: ES_JAVA_OPTS
82 |               value: "-Xms512m -Xmx1024m"
83 |           resources:
84 |             requests:
85 |               memory: 1Gi
86 |               cpu: 0.5
87 |             limits:
88 |               memory: 2Gi
89 |               cpu: 2
90 | 


--------------------------------------------------------------------------------
/fluent-bit-configmap.yaml:
--------------------------------------------------------------------------------
  1 | apiVersion: v1
  2 | kind: ConfigMap
  3 | metadata:
  4 |   name: fluent-bit-config
  5 |   namespace: logging
  6 |   labels:
  7 |     k8s-app: fluent-bit
  8 | data:
  9 |   # Configuration files: server, input, filters and output
 10 |   # ======================================================
 11 |   fluent-bit.conf: |
 12 |     [SERVICE]
 13 |         Flush         1
 14 |         Log_Level     info
 15 |         Daemon        off
 16 |         Parsers_File  parsers.conf
 17 |         HTTP_Server   On
 18 |         HTTP_Listen   0.0.0.0
 19 |         HTTP_Port     2020
 20 | 
 21 |     @INCLUDE input-kubernetes.conf
 22 |     @INCLUDE filter-kubernetes.conf
 23 |     @INCLUDE output-elasticsearch.conf
 24 | 
 25 |   input-kubernetes.conf: |
 26 |     [INPUT]
 27 |         Name              tail
 28 |         Tag               kube.*
 29 |         Path              /var/log/containers/*.log
 30 |         Parser            docker
 31 |         DB                /var/log/flb_kube.db
 32 |         Mem_Buf_Limit     5MB
 33 |         Skip_Long_Lines   On
 34 |         Refresh_Interval  10
 35 | 
 36 |   filter-kubernetes.conf: |
 37 |     [FILTER]
 38 |         Name                kubernetes
 39 |         Match               kube.*
 40 |         Kube_URL            https://kubernetes.default.svc:443
 41 |         Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
 42 |         Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
 43 |         Kube_Tag_Prefix     kube.var.log.containers.
 44 |         Merge_Log           On
 45 |         Merge_Log_Key       log_processed
 46 |         K8S-Logging.Parser  On
 47 |         K8S-Logging.Exclude Off
 48 | 
 49 |   output-elasticsearch.conf: |
 50 |     [OUTPUT]
 51 |         Name            es
 52 |         Match           *
 53 |         Host            ${FLUENT_ELASTICSEARCH_HOST}
 54 |         Port            ${FLUENT_ELASTICSEARCH_PORT}
 55 |         HTTP_User       ${FLUENT_ELASTICSEARCH_USER}
 56 |         HTTP_Passwd     ${FLUENT_ELASTICSEARCH_PASSWORD}
 57 |         Logstash_Format On
 58 |         Type  _doc
 59 |         Replace_Dots    On
 60 |         Retry_Limit     False
 61 | 
 62 |   parsers.conf: |
 63 |     [PARSER]
 64 |         Name   apache
 65 |         Format regex
 66 |         Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
 67 |         Time_Key time
 68 |         Time_Format %d/%b/%Y:%H:%M:%S %z
 69 | 
 70 |     [PARSER]
 71 |         Name   apache2
 72 |         Format regex
 73 |         Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
 74 |         Time_Key time
 75 |         Time_Format %d/%b/%Y:%H:%M:%S %z
 76 | 
 77 |     [PARSER]
 78 |         Name   apache_error
 79 |         Format regex
 80 |         Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
 81 | 
 82 |     [PARSER]
 83 |         Name   nginx
 84 |         Format regex
 85 |         Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
 86 |         Time_Key time
 87 |         Time_Format %d/%b/%Y:%H:%M:%S %z
 88 | 
 89 |     [PARSER]
 90 |         Name   json
 91 |         Format json
 92 |         Time_Key time
 93 |         Time_Format %d/%b/%Y:%H:%M:%S %z
 94 | 
 95 |     [PARSER]
 96 |         Name        docker
 97 |         Format      json
 98 |         Time_Key    time
 99 |         Time_Format %Y-%m-%dT%H:%M:%S.%L
100 |         Time_Keep   On
101 | 
102 |     [PARSER]
103 |         Name        syslog
104 |         Format      regex
105 |         Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
106 |         Time_Key    time
107 |         Time_Format %b %d %H:%M:%S
108 | 


--------------------------------------------------------------------------------
/fluent-bit-ds.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: extensions/v1beta1
 2 | kind: DaemonSet
 3 | metadata:
 4 |   name: fluent-bit
 5 |   namespace: logging
 6 |   labels:
 7 |     k8s-app: fluent-bit-logging
 8 |     version: v1
 9 |     kubernetes.io/cluster-service: "true"
10 | spec:
11 |   template:
12 |     metadata:
13 |       labels:
14 |         k8s-app: fluent-bit-logging
15 |         version: v1
16 |         kubernetes.io/cluster-service: "true"
17 |       annotations:
18 |         prometheus.io/scrape: "true"
19 |         prometheus.io/port: "2020"
20 |         prometheus.io/path: /api/v1/metrics/prometheus
21 |     spec:
22 |       containers:
23 |       - name: fluent-bit
24 |         image: fluent/fluent-bit:1.2.1
25 |         imagePullPolicy: Always
26 |         ports:
27 |           - containerPort: 2020
28 |         env:
29 |         - name: FLUENT_ELASTICSEARCH_HOST
30 |           value: "logging-elastic-es-http"
31 |         - name: FLUENT_ELASTICSEARCH_PORT
32 |           value: "9200"
33 |         - name: FLUENT_ELASTICSEARCH_USER
34 |           value: "elastic"
35 |         - name: FLUENT_ELASTICSEARCH_PASSWORD
36 |           valueFrom:
37 |             secretKeyRef:
38 |               name: logging-elastic-es-elastic-user
39 |               key: elastic
40 |         volumeMounts:
41 |         - name: varlog
42 |           mountPath: /var/log
43 |         - name: varlibdockercontainers
44 |           mountPath: /var/lib/docker/containers
45 |           readOnly: true
46 |         - name: fluent-bit-config
47 |           mountPath: /fluent-bit/etc/
48 |       terminationGracePeriodSeconds: 10
49 |       volumes:
50 |       - name: varlog
51 |         hostPath:
52 |           path: /var/log
53 |       - name: varlibdockercontainers
54 |         hostPath:
55 |           path: /var/lib/docker/containers
56 |       - name: fluent-bit-config
57 |         configMap:
58 |           name: fluent-bit-config
59 |       serviceAccountName: fluent-bit
60 |       tolerations:
61 |       - key: node-role.kubernetes.io/master
62 |         operator: Exists
63 |         effect: NoSchedule
64 |       - operator: "Exists"
65 |         effect: "NoExecute"
66 |       - operator: "Exists"
67 |         effect: "NoSchedule"
68 | 


--------------------------------------------------------------------------------
/fluent-bit-role-binding.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: rbac.authorization.k8s.io/v1beta1
 2 | kind: ClusterRoleBinding
 3 | metadata:
 4 |   name: fluent-bit-read
 5 | roleRef:
 6 |   apiGroup: rbac.authorization.k8s.io
 7 |   kind: ClusterRole
 8 |   name: fluent-bit-read
 9 | subjects:
10 | - kind: ServiceAccount
11 |   name: fluent-bit
12 |   namespace: logging
13 | 


--------------------------------------------------------------------------------
/fluent-bit-role.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: rbac.authorization.k8s.io/v1beta1
 2 | kind: ClusterRole
 3 | metadata:
 4 |   name: fluent-bit-read
 5 | rules:
 6 | - apiGroups: [""]
 7 |   resources:
 8 |   - namespaces
 9 |   - pods
10 |   verbs: ["get", "list", "watch"]
11 | 


--------------------------------------------------------------------------------
/fluent-bit-service-account.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: ServiceAccount
3 | metadata:
4 |   name: fluent-bit
5 |   namespace: logging
6 | 


--------------------------------------------------------------------------------
/namespace.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Namespace
3 | metadata:
4 |   name: logging


--------------------------------------------------------------------------------