├── Dockerfile ├── LICENSE ├── README.md ├── cron.yaml ├── daemonset.yaml ├── fluent.conf └── rbac.yaml /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:stretch-slim 2 | LABEL maintainer "Joakim Karlsson > /etc/gemrc \ 29 | && gem install \ 30 | fluent-plugin-gelf-hs:1.0.4 \ 31 | fluent-plugin-kubernetes_metadata_filter:1.0.0 \ 32 | fluent-plugin-systemd:0.3.1 \ 33 | fluentd:1.0.2 \ 34 | gelf:3.0.0 \ 35 | json:2.1.0 \ 36 | oj:2.18.3 \ 37 | fluent-plugin-record-modifier \ 38 | && dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \ 39 | && wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_$dpkgArch \ 40 | && chmod +x /usr/bin/dumb-init \ 41 | && wget -O /tmp/jemalloc-4.4.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/4.4.0/jemalloc-4.4.0.tar.bz2 \ 42 | && cd /tmp && tar -xjf jemalloc-4.4.0.tar.bz2 && cd jemalloc-4.4.0/ \ 43 | && ./configure && make \ 44 | && mv lib/libjemalloc.so.2 /usr/lib \ 45 | && apt-get purge -y --auto-remove \ 46 | -o APT::AutoRemove::RecommendsImportant=false \ 47 | $buildDeps \ 48 | && rm -rf /var/lib/apt/lists/* \ 49 | && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem 50 | 51 | ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2" 52 | 53 | # EXPOSE 24224 5140 54 | 55 | ENTRYPOINT ["/usr/bin/dumb-init", "--"] 56 | 57 | CMD fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins $FLUENTD_OPT 58 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Joakim Karlsson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kube-gelf 2 | 3 | [Fluentd](https://www.fluentd.org/) [CoreOS](https://coreos.com/) Kubernetes container logs & [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) log collector with [Graylog](https://www.graylog.org/) output. 4 | Configurable through [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) and with provided cron example to mitigate some fluentd bugs as well as providing option for config reloads 5 | 6 | ## Notes 7 | 8 | This project is automaticly built at [Docker Hub](https://hub.docker.com/r/roffe/kube-gelf/) 9 | 10 | This image has only been tested with CoreOS but should work with any other distribution as long as the paths in fluent.conf & the daemonset is adjusted accordingly. 11 | 12 | ## Installation 13 | 14 | ```sh 15 | kubectl create -f rbac.yaml 16 | 17 | kubectl create configmap \ 18 | --namespace kube-system kube-gelf \ 19 | --from-file fluent.conf \ 20 | --from-literal GELF_HOST= \ 21 | --from-literal GELF_PORT=12201 \ 22 | --from-literal GELF_PROTOCOL= 23 | 24 | kubectl create -f daemonset.yaml 25 | 26 | # optional, see notes below 27 | kubectl create -f cron.yaml 28 | ``` 29 | 30 | After updating the configmap reloading fluentd config on all pods can be done with kubectl access. 31 | Please allow atleast a minute to pass before issuing the command due to Kubernetes not real-time syncing configmap updates to volumes. 32 | 33 | ```sh 34 | for POD in `kubectl get pod --namespace kube-system -l app=kube-gelf | tail -n +2 | awk '{print $1}'`; do echo RELOAD ${POD}; kubectl exec --namespace kube-system ${POD} -- /bin/sh -c 'kill -1 1'; done 35 | ``` 36 | 37 | ## Cron 38 | 39 | As of Kubernetes 1.8 batch/v1beta1 is enabled by default and no additional changes are needed. 40 | 41 | If you are on < 1.8: 42 | 43 | Enable `batch/v2alpha1=true` in the apiserver(s) `--runtime-config=` & restart apiservers + controller-manager. 44 | Also change the apiVersion from `batch/v1beta1` to `batch/v2alpha1` 45 | 46 | The cron.yaml can be used to deploy a cronJob that periodicly tells kube-gelf to reload it's configuration to also works around some fluend bugs. 47 | 48 | I have several images made for different Kubernetes versions and you could adapt your cron.yaml by using any of my avail image tags here: 49 | 50 | ## Fluentd Bugs 51 | 52 | in_tail prevents docker from removing container 53 | . 54 | 55 | in_tail removes untracked file position during startup phase. It means the content of pos_file is growing until restart when you tails lots of files with dynamic path setting. I will fix this problem in the future. Check this issue. 56 | . 57 | 58 | ## Changelog 59 | 60 | ### 1.2 61 | 62 | * Introduced new ENV variable GELF_PROTOCOL for protocol selection. Valid values are "udp" or "tcp". This requires a update of you configmap from earlier verisons 63 | * Changed output plugin to 64 | 65 | ### 1.1 66 | 67 | Got rid of hostNetwork and added a NODENAME env variable utilizing the downward api. All log entries will contain the field `hostname: ` 68 | -------------------------------------------------------------------------------- /cron.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1beta1 2 | kind: CronJob 3 | metadata: 4 | name: kube-gelf-reload-config 5 | namespace: kube-system 6 | spec: 7 | successfulJobsHistoryLimit: 1 8 | failedJobsHistoryLimit: 1 9 | schedule: "*/30 * * * *" 10 | jobTemplate: 11 | spec: 12 | template: 13 | spec: 14 | serviceAccountName: kube-gelf 15 | containers: 16 | - name: cron 17 | image: roffe/kubectl:v1.9.1 18 | args: 19 | - /bin/sh 20 | - -c 21 | - for POD in `kubectl get pod --namespace kube-system -l app=kube-gelf | tail -n +2 | awk '{print $1}'`; do echo RELOAD ${POD}; kubectl exec --namespace kube-system ${POD} -- /bin/sh -c "ps aux|grep '/usr/bin/ruby2.3 /usr/local/bin/fluentd' | grep -v grep | awk {'print \$2'} | xargs kill -1"; done 22 | restartPolicy: Never 23 | -------------------------------------------------------------------------------- /daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: kube-gelf 5 | namespace: kube-system 6 | spec: 7 | updateStrategy: 8 | type: RollingUpdate 9 | rollingUpdate: 10 | maxUnavailable: 1 11 | selector: 12 | matchLabels: 13 | app: kube-gelf 14 | template: 15 | metadata: 16 | labels: 17 | app: kube-gelf 18 | spec: 19 | serviceAccountName: kube-gelf 20 | dnsPolicy: ClusterFirst 21 | containers: 22 | - name: agent 23 | image: roffe/kube-gelf:latest 24 | env: 25 | - name: GELF_HOST 26 | valueFrom: 27 | configMapKeyRef: 28 | name: kube-gelf 29 | key: GELF_HOST 30 | - name: GELF_PORT 31 | valueFrom: 32 | configMapKeyRef: 33 | name: kube-gelf 34 | key: GELF_PORT 35 | - name: GELF_PROTOCOL 36 | valueFrom: 37 | configMapKeyRef: 38 | name: kube-gelf 39 | key: GELF_PROTOCOL 40 | - name: NODENAME 41 | valueFrom: 42 | fieldRef: 43 | fieldPath: spec.nodeName 44 | volumeMounts: 45 | - name: varlog 46 | mountPath: /var/log 47 | readOnly: true 48 | - name: gelf-pos 49 | mountPath: /pos 50 | readOnly: false 51 | - name: varlibdockercontainers 52 | mountPath: /var/lib/docker/containers 53 | readOnly: true 54 | - name: kube-gelf-config 55 | mountPath: /fluentd/etc/fluent.conf 56 | subPath: fluent.conf 57 | tolerations: 58 | - key: node-role.kubernetes.io/master 59 | operator: Exists 60 | effect: NoSchedule 61 | volumes: 62 | - name: varlog 63 | hostPath: 64 | path: /var/log 65 | - name: varlibdockercontainers 66 | hostPath: 67 | path: /var/lib/docker/containers 68 | - name: kube-gelf-config 69 | configMap: 70 | name: kube-gelf 71 | items: 72 | - key: fluent.conf 73 | path: fluent.conf 74 | - name: gelf-pos 75 | hostPath: 76 | path: /var/log/pos 77 | -------------------------------------------------------------------------------- /fluent.conf: -------------------------------------------------------------------------------- 1 | 2 | @type null 3 | 4 | 5 | 6 | @type systemd 7 | tag systemd 8 | path /var/log/journal 9 | 10 | @type local 11 | persistent true 12 | path /pos 13 | 14 | 15 | field_map '{"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"}' 16 | fields_strip_underscores true 17 | fields_lowercase true 18 | 19 | read_from_head true 20 | 21 | 22 | 23 | @type tail 24 | path /var/log/containers/*.log 25 | pos_file /pos/containers.pos 26 | time_key time 27 | time_format %Y-%m-%dT%H:%M:%S.%NZ 28 | tag kubernetes.* 29 | format json 30 | read_from_head true 31 | 32 | 33 | 34 | @type kubernetes_metadata 35 | ca_file /var/run/secrets/kubernetes.io/serviceaccount/ca.crt 36 | bearer_token_file /var/run/secrets/kubernetes.io/serviceaccount/token 37 | 38 | 39 | 40 | @type record_transformer 41 | 42 | hostname "#{ENV['NODENAME']}" 43 | 44 | 45 | 46 | 47 | @type record_modifier 48 | char_encoding utf-8 49 | 50 | 51 | 52 | @type copy 53 | 54 | @type gelf 55 | include_tag_key true 56 | host "#{ENV['GELF_HOST']}" 57 | port "#{ENV['GELF_PORT']}" 58 | protocol "#{ENV['GELF_PROTOCOL']}" 59 | # tls 60 | # tls_options {} 61 | # Supported tls_options: 62 | # 'no_default_ca' [Boolean] prevents OpenSSL from using the systems CA store. 63 | # 'tls_version' [Symbol] any of :TLSv1, :TLSv1_1, :TLSv1_2 (default) 64 | # 'cert' [String, IO] the client certificate file 65 | # 'key' [String, IO] the key for the client certificate 66 | # 'all_ciphers' [Boolean] allows any ciphers to be used, may be insecure 67 | 68 | flush_at_shutdown true 69 | flush_mode immediate 70 | flush_thread_count 4 71 | flush_thread_interval 1 72 | flush_thread_burst_interval 1 73 | retry_forever false 74 | retry_max_times 3 75 | retry_type exponential_backoff 76 | 77 | use_record_host true 78 | 79 | 80 | 81 | # Valid log_level's are: fatal, error, warn, info, debug, trace 82 | 83 | log_level warn 84 | 85 | -------------------------------------------------------------------------------- /rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: kube-gelf 5 | namespace: kube-system 6 | --- 7 | apiVersion: rbac.authorization.k8s.io/v1beta1 8 | kind: ClusterRole 9 | metadata: 10 | annotations: 11 | rbac.authorization.kubernetes.io/autoupdate: "true" 12 | name: kube-gelf-metadata-filter 13 | rules: 14 | - apiGroups: 15 | - "" 16 | resources: 17 | - pods 18 | - pods/exec 19 | - namespaces 20 | verbs: ["get", "list", "watch", "create"] 21 | --- 22 | apiVersion: rbac.authorization.k8s.io/v1beta1 23 | kind: ClusterRoleBinding 24 | metadata: 25 | name: kube-gelf 26 | roleRef: 27 | apiGroup: rbac.authorization.k8s.io 28 | kind: ClusterRole 29 | name: kube-gelf-metadata-filter 30 | subjects: 31 | - kind: ServiceAccount 32 | name: kube-gelf 33 | namespace: kube-system 34 | --------------------------------------------------------------------------------