├── .dockerignore ├── .gitignore ├── .travis.yml ├── Dockerfile.template ├── Makefile ├── README.md ├── build-confd ├── fluentd.daemonset.yaml ├── fluentd.datadog.daemonset.yaml ├── fluentd.rbac.yaml ├── get-confd ├── rootfs ├── etc │ ├── confd │ │ ├── conf.d │ │ │ ├── application_default_credentials.toml │ │ │ └── fluent.conf.toml │ │ └── templates │ │ │ ├── application_default_credentials.json.tmpl │ │ │ └── fluent.conf.tmpl │ ├── cont-finish.d │ │ └── .gitkeep │ ├── cont-init.d │ │ ├── .gitkeep │ │ ├── 04-auto-template-resources │ │ └── 05-confd │ ├── fix-attrs.d │ │ └── .gitkeep │ ├── google │ │ └── auth │ │ │ └── .gitkeep │ └── services.d │ │ └── .gitkeep └── fluentd │ └── etc │ ├── .gitkeep │ ├── conf.d │ └── .gitkeep │ └── confd │ └── templates │ └── .gitkeep └── testdata └── fluentd └── etc └── confd └── templates ├── fluent-s3.conf.tmpl └── test-fluent.conf.tmpl /.dockerignore: -------------------------------------------------------------------------------- 1 | .envrc 2 | *~ 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .envrc 2 | *~ 3 | build/ 4 | *.secret.yaml 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | 3 | services: 4 | - docker 5 | 6 | env: 7 | - FLUENTD_VERSION=0.14.1 8 | 9 | before_install: 10 | - make load-docker-cache 11 | - make build 12 | 13 | script: 14 | - make test 15 | 16 | after_success: 17 | - make save-docker-cache 18 | - if [ "$TRAVIS_BRANCH" == "master" ]; then 19 | docker login -e="$DOCKER_EMAIL" -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"; 20 | make publish; 21 | fi 22 | 23 | cache: 24 | directories: 25 | - docker-cache 26 | -------------------------------------------------------------------------------- /Dockerfile.template: -------------------------------------------------------------------------------- 1 | FROM gcr.io/google_containers/ubuntu-slim:0.14 2 | MAINTAINER Yusuke KUOKA 3 | 4 | # Disable prompts from apt. 5 | ENV DEBIAN_FRONTEND noninteractive 6 | 7 | ENV FLUENTD_VERSION %%FLUENTD_VERSION%% 8 | 9 | # Do not split this into multiple RUN! 10 | # Docker creates a layer for every RUN-Statement 11 | # therefore an 'apt remove --purge -y build*' has no effect 12 | RUN apt-get update -y \ 13 | && apt-get install -y --no-install-recommends ca-certificates wget \ 14 | && wget https://github.com/just-containers/s6-overlay/releases/download/v1.17.1.1/s6-overlay-amd64.tar.gz --no-check-certificate -O /tmp/s6-overlay.tar.gz \ 15 | && tar xvfz /tmp/s6-overlay.tar.gz -C / \ 16 | && rm -f /tmp/s6-overlay.tar.gz \ 17 | && apt-get install --no-install-recommends -y \ 18 | procps \ 19 | build-essential \ 20 | ca-certificates \ 21 | git \ 22 | ruby \ 23 | ruby-dev \ 24 | libjemalloc1 && \ 25 | echo 'gem: --no-document' >> /etc/gemrc && \ 26 | echo "Running ruby: $(ruby -v)" && \ 27 | # minitest 5.x is required by activesupport. It is not a development dependency but a runtime dependency. 28 | # minitest 5.x should be bundled in ruby since 2.2 but ruby from ubuntu seems to miss it therefore install it ourselves 29 | gem install minitest oj && \ 30 | gem install fluentd -v $FLUENTD_VERSION && \ 31 | # We install fluent plugins here because doing so requires build-base and ruby-dev in order to build native extensions 32 | gem install fluent-plugin-google-cloud && \ 33 | gem install fluent-plugin-datadog-log -v 0.1.0.rc18 && \ 34 | gem install fluent-plugin-kubernetes_metadata_filter && \ 35 | gem install fluent-plugin-record-reformer && \ 36 | gem install fluent-plugin-prometheus && \ 37 | apt-get remove --purge -y wget build-essential ruby-dev git libgcc-5-dev cpp-5 && \ 38 | apt-get autoremove -y && \ 39 | apt-get clean -y && \ 40 | rm -rf /tmp/* /var/tmp/* /var/cache/apt/archives/* /var/lib/apt/lists/* 41 | 42 | RUN adduser --disabled-password --disabled-login --gecos "" --uid 1000 --home /home/fluent fluent 43 | RUN chown fluent:fluent -R /home/fluent 44 | 45 | # for log storage (maybe shared with host) 46 | RUN mkdir -p /fluentd/log 47 | # configuration/plugins path (default: copied from .) 48 | RUN mkdir -p /fluentd/etc /fluentd/plugins 49 | 50 | RUN chown -R fluent:fluent /fluentd 51 | 52 | USER fluent 53 | WORKDIR /home/fluent 54 | 55 | # Tell ruby to install packages as user 56 | RUN echo "gem: --user-install --no-document" >> ~/.gemrc 57 | ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH 58 | ENV GEM_PATH /home/fluent/.gem/ruby/2.3.0:$GEM_PATH 59 | 60 | USER root 61 | WORKDIR / 62 | 63 | COPY rootfs / 64 | 65 | ENV JEMALLOC_PATH /usr/lib/x86_64-linux-gnu/libjemalloc.so.1 66 | ENV LD_PRELOAD $JEMALLOC_PATH 67 | 68 | ENV FLUENTD_OUT_KUBEUSER_BUFFER_OVERFLOW_ACTION="block" 69 | ENV FLUENTD_OUT_KUBEUSER_BUFFER_CHUNK_LIMIT_SIZE="2M" 70 | ENV FLUENTD_OUT_KUBEUSER_BUFFER_TOTAL_LIMIT_SIZE="16M" 71 | ENV FLUENTD_OUT_KUBEUSER_BUFFER_FLUSH_INTERVAL="10s" 72 | ENV FLUENTD_OUT_KUBEUSER_BUFFER_RETRY_MAX_INTERVAL="30" 73 | ENV FLUENTD_OUT_KUBEUSER_RETRY_FOREVER="false" 74 | ENV FLUENTD_OUT_KUBEUSER_RETRY_MAX_TIMES="50" 75 | ENV FLUENTD_OUT_KUBEUSER_FLUSH_THREAD_COUNT="2" 76 | 77 | ENV FLUENTD_OUT_KUBESYS_BUFFER_OVERFLOW_ACTION="block" 78 | ENV FLUENTD_OUT_KUBESYS_BUFFER_CHUNK_LIMIT_SIZE="2M" 79 | ENV FLUENTD_OUT_KUBESYS_BUFFER_TOTAL_LIMIT_SIZE="16M" 80 | ENV FLUENTD_OUT_KUBESYS_BUFFER_FLUSH_INTERVAL="30s" 81 | ENV FLUENTD_OUT_KUBESYS_BUFFER_RETRY_MAX_INTERVAL="30" 82 | ENV FLUENTD_OUT_KUBESYS_RETRY_FOREVER="false" 83 | ENV FLUENTD_OUT_KUBESYS_RETRY_MAX_TIMES="50" 84 | ENV FLUENTD_OUT_KUBESYS_FLUSH_THREAD_COUNT="2" 85 | 86 | ENV FLUENTD_ARGS="" 87 | ENV FLUENTD_CONF="fluent.conf" 88 | 89 | EXPOSE 24224 5140 90 | 91 | # Prometheus metrics endpoint 92 | EXPOSE 24231 93 | 94 | ENTRYPOINT [ "/init" ] 95 | 96 | CMD /bin/s6-envuidgid fluent fluentd -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_ARGS 97 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | KUBE_FLUENTD_VERSION ?= 0.9.10-rc.23 2 | FLUENTD_VERSION ?= 1.0.0 3 | 4 | REPOSITORY ?= mumoshu/kube-fluentd 5 | TAG ?= $(FLUENTD_VERSION)-$(KUBE_FLUENTD_VERSION) 6 | IMAGE ?= $(REPOSITORY):$(TAG) 7 | ALIAS ?= $(REPOSITORY):$(FLUENTD_VERSION) 8 | 9 | BUILD_ROOT ?= build/$(TAG) 10 | DOCKERFILE ?= $(BUILD_ROOT)/Dockerfile 11 | DOCKERIGNORE ?= $(BUILD_ROOT)/.dockerignore 12 | ROOTFS ?= $(BUILD_ROOT)/rootfs 13 | DOCKER_CACHE ?= docker-cache 14 | SAVED_IMAGE ?= $(DOCKER_CACHE)/image-$(FLUENTD_VERSION).tar 15 | 16 | .PHONY: build 17 | build: $(DOCKERFILE) $(DOCKERIGNORE) $(ROOTFS) 18 | ./get-confd 19 | cd $(BUILD_ROOT) && docker build -t $(IMAGE) . && docker tag $(IMAGE) $(ALIAS) 20 | 21 | .PHONY: clean 22 | clean: 23 | rm -rf $(BUILD_ROOT) 24 | 25 | publish: 26 | docker push $(IMAGE) && docker push $(ALIAS) 27 | 28 | $(DOCKERFILE): $(BUILD_ROOT) 29 | sed 's/%%FLUENTD_VERSION%%/'"$(FLUENTD_VERSION)"'/g;' Dockerfile.template > $(DOCKERFILE) 30 | 31 | $(DOCKERIGNORE): $(BUILD_ROOT) 32 | cp .dockerignore $(DOCKERIGNORE) 33 | 34 | $(ROOTFS): $(BUILD_ROOT) 35 | cp -R rootfs $(ROOTFS) 36 | find $(ROOTFS) -type f -name '*~' -exec rm {} + 37 | 38 | $(BUILD_ROOT): 39 | mkdir -p $(BUILD_ROOT) 40 | 41 | travis-env: 42 | travis env set DOCKER_EMAIL $(DOCKER_EMAIL) 43 | travis env set DOCKER_USERNAME $(DOCKER_USERNAME) 44 | travis env set DOCKER_PASSWORD $(DOCKER_PASSWORD) 45 | 46 | test: build 47 | @echo There are no tests available for now. Skipping 48 | 49 | save-docker-cache: $(DOCKER_CACHE) 50 | docker save $(IMAGE) $(shell docker history -q $(IMAGE) | tail -n +2 | grep -v \ | tr '\n' ' ') > $(SAVED_IMAGE) 51 | ls -lah $(DOCKER_CACHE) 52 | 53 | load-docker-cache: $(DOCKER_CACHE) 54 | if [ -e $(SAVED_IMAGE) ]; then docker load < $(SAVED_IMAGE); fi 55 | 56 | $(DOCKER_CACHE): 57 | mkdir -p $(DOCKER_CACHE) 58 | 59 | docker-run-gcp: DOCKER_CMD ?= 60 | docker-run-gcp: 61 | docker run --rm -it \ 62 | -v /mnt/sda1:/mnt/sda1 \ 63 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 64 | -v /var/log:/var/log \ 65 | -e GOOGLE_FLUENTD_PRIVATE_KEY_ID="$(GOOGLE_FLUENTD_PRIVATE_KEY_ID)" \ 66 | -e GOOGLE_FLUENTD_PRIVATE_KEY="$(GOOGLE_FLUENTD_PRIVATE_KEY)" \ 67 | -e GOOGLE_FLUENTD_PROJECT_ID="$(GOOGLE_FLUENTD_PROJECT_ID)" \ 68 | -e GOOGLE_FLUENTD_CLIENT_EMAIL="$(GOOGLE_FLUENTD_CLIENT_EMAIL)" \ 69 | -e GOOGLE_FLUENTD_CLIENT_ID="$(GOOGLE_FLUENTD_CLIENT_ID)" \ 70 | -e GOOGLE_FLUENTD_CLIENT_X509_CERT_URL="$(GOOGLE_FLUENTD_CLIENT_X509_CERT_URL)" \ 71 | -e FLUENTD_OUT_KUBESYS_BUFFER_CHUNK_LIMIT="$(FLUENTD_OUT_KUBESYS_BUFFER_CHUNK_LIMIT)" \ 72 | -e FLUENTD_OUT_KUBESYS_BUFFER_QUEUE_LIMIT="$(FLUENTD_OUT_KUBESYS_BUFFER_QUEUE_LIMIT)" \ 73 | -e FLUENTD_OUT_KUBESYS_NUM_THREADS="$(FLUENTD_OUT_KUBESYS_NUM_THREADS)" \ 74 | -e FLUENTD_OUT_KUBESYS_BUFFER_FLUSH_INTERVAL="$(FLUENTD_OUT_KUBESYS_BUFFER_FLUSH_INTERVAL)" \ 75 | -e FLUENTD_OUT_KUBESYS_RETRY_LIMIT_DISABLE="$(FLUENTD_OUT_KUBESYS_RETRY_LIMIT_DISABLE)" \ 76 | -e FLUENTD_OUT_KUBESYS_RETRY_LIMIT="$(FLUENTD_OUT_KUBESYS_RETRY_LIMIT)" \ 77 | -e FLUENTD_OUT_KUBEUSER_BUFFER_CHUNK_LIMIT="$(FLUENTD_OUT_KUBEUSER_BUFFER_CHUNK_LIMIT)" \ 78 | -e FLUENTD_OUT_KUBEUSER_BUFFER_QUEUE_LIMIT="$(FLUENTD_OUT_KUBEUSER_BUFFER_QUEUE_LIMIT)" \ 79 | -e FLUENTD_OUT_KUBEUSER_NUM_THREADS="$(FLUENTD_OUT_KUBEUSER_NUM_THREADS)" \ 80 | -e FLUENTD_OUT_KUBEUSER_BUFFER_FLUSH_INTERVAL="$(FLUENTD_OUT_KUBEUSER_BUFFER_FLUSH_INTERVAL)" \ 81 | -e FLUENTD_OUT_KUBEUSER_RETRY_LIMIT_DISABLE="$(FLUENTD_OUT_KUBEUSER_RETRY_LIMIT_DISABLE)" \ 82 | -e FLUENTD_OUT_KUBEUSER_RETRY_LIMIT="$(FLUENTD_OUT_KUBEUSER_RETRY_LIMIT)" \ 83 | $(IMAGE) $(DOCKER_CMD) 84 | 85 | define SECRET_YAML 86 | apiVersion: v1 87 | kind: Secret 88 | metadata: 89 | name: fluentd 90 | type: Opaque 91 | data: 92 | private.key.id: $(shell bash -c 'echo -n "$$GOOGLE_FLUENTD_PRIVATE_KEY_ID" | base64') 93 | private.key: $(shell bash -c 'echo -n "$$GOOGLE_FLUENTD_PRIVATE_KEY" | base64') 94 | project.id: $(shell bash -c 'echo -n "$$GOOGLE_FLUENTD_PROJECT_ID" | base64') 95 | client.email: $(shell bash -c 'echo -n "$$GOOGLE_FLUENTD_CLIENT_EMAIL" | base64') 96 | client.id: $(shell bash -c 'echo -n "$$GOOGLE_FLUENTD_CLIENT_ID" | base64') 97 | client.x509.cert.url: $(shell bash -c 'echo -n "$$GOOGLE_FLUENTD_CLIENT_X509_CERT_URL" | base64') 98 | endef 99 | export SECRET_YAML 100 | 101 | fluentd.secret.yaml: 102 | echo "$$SECRET_YAML" > fluentd.secret.yaml 103 | 104 | docker-run-dd: DOCKER_CMD ?= sh 105 | docker-run-dd: 106 | docker run --rm -it \ 107 | -v /mnt/sda1:/mnt/sda1 \ 108 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 109 | -v /var/log:/var/log \ 110 | -v $(shell pwd)/testdata/fluentd/etc/confd/templates:/fluentd/etc/confd/templates \ 111 | -e DD_API_KEY="$(DD_API_KEY)" \ 112 | -e KUBE_ENV=test \ 113 | -e KUBE_CLUSTER=k8s1 \ 114 | -e AWS_ACCESS_KEY_ID=dummyid \ 115 | -e AWS_SECERET_ACCESS_KEY=dummysecret \ 116 | -e S3_BUCKET=mykubeawscluster-test-k8s1 \ 117 | $(IMAGE) $(DOCKER_CMD) 118 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kube-fluentd 2 | 3 | [![Build Status](https://travis-ci.org/mumoshu/kube-fluentd.svg?branch=master)](https://travis-ci.org/mumoshu/kube-fluentd) 4 | 5 | [Docker Hub](https://hub.docker.com/r/mumoshu/kube-fluentd) 6 | 7 | A docker image for running fluentd in Kubernetes pods. 8 | 9 | Based on: 10 | 11 | * [gcr.io/google_containers/ubuntu-slim](https://console.cloud.google.com/kubernetes/images/tags/ubuntu-slim?location=GLOBAL&project=google-containers) 12 | * [Fluentd](https://github.com/fluent/fluentd) 13 | * [s6-overlay](https://github.com/just-containers/s6-overlay) 14 | * [confd](https://github.com/kelseyhightower/confd) 15 | 16 | ## Supported tags 17 | 18 | * `latest`/`0.14.22-0.9.9` (Fluentd v0.14.22) 19 | 20 | Naming convention for images is `$FLUENTD_VERSION`-`$KUBE_FLUENTD_VERSION` 21 | 22 | ## Changelog 23 | 24 | * `0.9.10-rc.17` 25 | * Bump Fluentd to 1.0.0 26 | * Support for Datadog Log Management 27 | * Expose Prometheus metrics 28 | * `0.9.10-rc.1` 29 | * Bump Fluentd to 0.14.22 30 | * Support for credentials from volume mounts (See `fluentd.daemonset.credfromfile.yaml`) 31 | * `0.9.9` 32 | * Bump Fluentd to 0.14.14 to incorporate the fix for https://github.com/fluent/fluentd/issues/1485 33 | * `0.9.8` 34 | * Set buffer_queue_full_action to `block` for tail plugins so that we won't drop messages anymore 35 | * `0.9.7` 36 | * fluentd.conf is customizable via environment variables (#3, thanks to @cw-hayashi) 37 | * `0.9.6` 38 | * Fix the fluentd OOM issue when ingesting a lot of log 39 | * Also see https://github.com/fluent/fluentd/issues/1260 for why we don't use `overflow_action block` 40 | * `0.9.5` 41 | * Remove /var/lib/apt/lists/* to save approximately 20MB of image size 42 | * `0.9.4` 43 | * Build confd outside of containers to save 1MB of image size 44 | * `0.9.3` 45 | * Switch from alpine to ubuntu-slim:0.4 46 | * Use jemalloc 47 | * Fluentd 0.14.9 48 | * `0.9.2` 49 | * I had fixed the wrong part of fluent.conf in 0.9.1. Now I've done it right. 50 | * Bump Fluentd to 0.14.2 to fix [the issue reported in the metadata filter repo](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/issues/33#issuecomment-238377746) 51 | * `0.9.1` 52 | * Use multiple threads for processing as per [a change in GoogleCloudPlatform/google-fluentd](https://github.com/GoogleCloudPlatform/google-fluentd/commit/283eb7052d3a256078f37d03e8ea3a496794a28f) 53 | 54 | ## Usage in Kubernetes 55 | 56 | (1) Create a new GCP service account key in https://console.cloud.google.com/apis/credentials and obtain `${gcp_project_name}-${service_account_key_id}.json1` 57 | 58 | (2) Run the following commands: 59 | 60 | For Google Stackdriver Logging: 61 | 62 | ``` 63 | $ cp ${gcp_project_name}-${service_account_key_id}.json application_default_credentials.json 64 | $ kubectl kubectl create secret generic kube-fluentd-google-application-default-credentials --from-file application_default_credentials.json --namespace kube-system 65 | $ kubectl create -f fluentd.rbac.yaml 66 | $ kubectl create -f fluentd.daemonset.credfromfile.yaml 67 | ``` 68 | 69 | For Datadog Log Management: 70 | 71 | ``` 72 | $ kubectl create secret generic datadog --from-literal=api-key=$DD_API_KEY 73 | $ kubectl create -f fluentd.rbac.yaml 74 | $ kubectl create -f fluentd.datadog.daemonset.yaml 75 | ``` 76 | -------------------------------------------------------------------------------- /build-confd: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -vxe 4 | 5 | # Until confd package arrives to apt packages, build it ourselves: 6 | # Installation process initially inspired by https://github.com/smebberson/docker-alpine/blob/master/alpine-confd/Dockerfile 7 | # and later adapted to Ubuntu. 8 | # CONF_VERSION can be any git ref like commit id, branch, tag 9 | # ENV CONFD_VERSION=master 10 | export CONFD_VERSION=v0.11.0 11 | 12 | export GOPATH=$(pwd)/go/confd 13 | export REPO_PATH=$GOPATH/src/github.com/kelseyhightower 14 | mkdir -p $REPO_PATH 15 | 16 | export CONFD_PATH=$REPO_PATH/confd 17 | 18 | if [ -d "$CONFD_PATH" ]; then 19 | rm -rf $CONFD_PATH 20 | fi 21 | 22 | git clone https://github.com/kelseyhightower/confd.git $GOPATH/src/github.com/kelseyhightower/confd 23 | 24 | pushd $GOPATH/src/github.com/kelseyhightower/confd 25 | 26 | git checkout -q --detach "$CONFD_VERSION" 27 | 28 | # For legacy confd branch with non-standard file structure 29 | if [ -d src/github.com/kelseyhightower/confd ]; then echo fixing file tree to conform this project to standard golang project structure; mv src/github.com/kelseyhightower/confd/* .; fi && \ 30 | if [ -d vendor/src ]; then echo fixing vendor to conform this project to standard golang project structure; mv vendor/src/* vendor/; fi && \ 31 | 32 | CGO_ENABLED=0 GOOS=linux go build -a -tags netgo -ldflags '-w' . 33 | 34 | chmod +x confd 35 | 36 | popd 37 | 38 | mkdir -p rootfs/bin 39 | cp $CONFD_PATH/confd rootfs/bin/ 40 | -------------------------------------------------------------------------------- /fluentd.daemonset.yaml: -------------------------------------------------------------------------------- 1 | # Originally copied from: https://github.com/kubernetes/kubernetes/tree/master/cluster/saltbase/salt/fluentd-gcp 2 | # Also see the post: http://blog.kubernetes.io/2015/06/cluster-level-logging-with-kubernetes.html 3 | 4 | apiVersion: extensions/v1beta1 5 | kind: DaemonSet 6 | metadata: 7 | name: fluentd-cloud-logging 8 | namespace: kube-system 9 | spec: 10 | template: 11 | metadata: 12 | labels: 13 | k8s-app: fluentd-logging 14 | spec: 15 | serviceAccountName: fluentd-cloud-logging 16 | tolerations: 17 | - operator: Exists 18 | effect: NoSchedule 19 | - operator: Exists 20 | effect: NoExecute 21 | - operator: Exists 22 | dnsPolicy: Default 23 | containers: 24 | - name: fluentd-cloud-logging 25 | image: mumoshu/kube-fluentd:0.14.22-0.9.9 26 | resources: 27 | limits: 28 | memory: 200Mi 29 | requests: 30 | # Any change here should be accompanied by a proportional change in CPU 31 | # requests of other per-node add-ons (e.g. kube-proxy). 32 | cpu: 80m 33 | memory: 200Mi 34 | env: 35 | - name: FLUENTD_ARGS 36 | value: -v 37 | - name: GOOGLE_FLUENTD_PRIVATE_KEY_ID 38 | valueFrom: 39 | secretKeyRef: 40 | name: fluentd 41 | key: private.key.id 42 | - name: GOOGLE_FLUENTD_PRIVATE_KEY 43 | valueFrom: 44 | secretKeyRef: 45 | name: fluentd 46 | key: private.key 47 | - name: GOOGLE_FLUENTD_PROJECT_ID 48 | valueFrom: 49 | secretKeyRef: 50 | name: fluentd 51 | key: project.id 52 | - name: GOOGLE_FLUENTD_CLIENT_EMAIL 53 | valueFrom: 54 | secretKeyRef: 55 | name: fluentd 56 | key: client.email 57 | - name: GOOGLE_FLUENTD_CLIENT_ID 58 | valueFrom: 59 | secretKeyRef: 60 | name: fluentd 61 | key: client.id 62 | - name: GOOGLE_FLUENTD_CLIENT_X509_CERT_URL 63 | valueFrom: 64 | secretKeyRef: 65 | name: fluentd 66 | key: client.x509.cert.url 67 | ports: 68 | - containerPort: 24231 69 | name: prometheus-metrics 70 | volumeMounts: 71 | - name: varlog 72 | mountPath: /var/log 73 | - name: varlibdockercontainers 74 | mountPath: /var/lib/docker/containers 75 | readOnly: true 76 | - name: mntsda1 77 | mountPath: /mnt/sda1 78 | terminationGracePeriodSeconds: 30 79 | volumes: 80 | - name: varlog 81 | hostPath: 82 | path: /var/log 83 | - name: varlibdockercontainers 84 | hostPath: 85 | path: /var/lib/docker/containers 86 | - name: mntsda1 87 | hostPath: 88 | path: /mnt/sda1 89 | -------------------------------------------------------------------------------- /fluentd.datadog.daemonset.yaml: -------------------------------------------------------------------------------- 1 | # Originally copied from: https://github.com/kubernetes/kubernetes/tree/master/cluster/saltbase/salt/fluentd-gcp 2 | # Also see the post: http://blog.kubernetes.io/2015/06/cluster-level-logging-with-kubernetes.html 3 | 4 | apiVersion: extensions/v1beta1 5 | kind: DaemonSet 6 | metadata: 7 | name: fluentd-cloud-logging 8 | namespace: kube-system 9 | spec: 10 | updateStrategy: 11 | type: RollingUpdate 12 | rollingUpdate: 13 | maxUnavailable: 1 14 | template: 15 | metadata: 16 | labels: 17 | k8s-app: fluentd-logging 18 | spec: 19 | serviceAccountName: fluentd-cloud-logging 20 | tolerations: 21 | - operator: Exists 22 | effect: NoSchedule 23 | - operator: Exists 24 | effect: NoExecute 25 | - operator: Exists 26 | dnsPolicy: Default 27 | containers: 28 | - name: fluentd-cloud-logging 29 | image: mumoshu/kube-fluentd:1.0.0-0.9.10-rc.23 30 | resources: 31 | limits: 32 | memory: 200Mi 33 | requests: 34 | # Any change here should be accompanied by a proportional change in CPU 35 | # requests of other per-node add-ons (e.g. kube-proxy). 36 | cpu: 80m 37 | memory: 200Mi 38 | env: 39 | - name: DD_API_KEY 40 | valueFrom: 41 | secretKeyRef: 42 | name: datadog 43 | key: api-key 44 | - name: DD_TAGS 45 | value: | 46 | ["env:test", "kube_cluster:k8s1"] 47 | ports: 48 | - containerPort: 24231 49 | name: prometheus 50 | volumeMounts: 51 | - name: varlog 52 | mountPath: /var/log 53 | - name: varlibdockercontainers 54 | mountPath: /var/lib/docker/containers 55 | readOnly: true 56 | - name: mntsda1 57 | mountPath: /mnt/sda1 58 | terminationGracePeriodSeconds: 30 59 | volumes: 60 | - name: varlog 61 | hostPath: 62 | path: /var/log 63 | - name: varlibdockercontainers 64 | hostPath: 65 | path: /var/lib/docker/containers 66 | - name: mntsda1 67 | hostPath: 68 | path: /mnt/sda1 69 | -------------------------------------------------------------------------------- /fluentd.rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | labels: 5 | k8s-app: fluentd-logging 6 | name: fluentd-cloud-logging 7 | namespace: kube-system 8 | --- 9 | apiVersion: rbac.authorization.k8s.io/v1beta1 10 | kind: ClusterRoleBinding 11 | metadata: 12 | name: fluentd-cloud-logging 13 | labels: 14 | k8s-app: fluentd-logging 15 | roleRef: 16 | apiGroup: rbac.authorization.k8s.io 17 | kind: ClusterRole 18 | name: cluster-admin 19 | subjects: 20 | - kind: ServiceAccount 21 | name: fluentd-cloud-logging 22 | namespace: kube-system 23 | -------------------------------------------------------------------------------- /get-confd: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -vxe 4 | 5 | export CONFD_VERSION=0.14.0 6 | 7 | mkdir -p confd/$CONFD_VERSION 8 | 9 | if [ ! -f confd/$CONFD_VERSION/confd ]; then 10 | curl -o confd/$CONFD_VERSION/confd -L https://github.com/kelseyhightower/confd/releases/download/v$CONFD_VERSION/confd-$CONFD_VERSION-linux-amd64 11 | fi 12 | 13 | mkdir -p rootfs/bin 14 | 15 | cp confd/$CONFD_VERSION/confd rootfs/bin/ 16 | -------------------------------------------------------------------------------- /rootfs/etc/confd/conf.d/application_default_credentials.toml: -------------------------------------------------------------------------------- 1 | [template] 2 | src = "application_default_credentials.json.tmpl" 3 | dest = "/etc/google/auth/application_default_credentials.json" 4 | keys = [ 5 | "/google/fluentd/project/id", 6 | "/google/fluentd/private/key/id", 7 | "/google/fluentd/private/key", 8 | "/google/fluentd/client/email", 9 | "/google/fluentd/client/id", 10 | "/google/fluentd/client/x509/cert/url", 11 | ] 12 | -------------------------------------------------------------------------------- /rootfs/etc/confd/conf.d/fluent.conf.toml: -------------------------------------------------------------------------------- 1 | [template] 2 | src = "fluent.conf.tmpl" 3 | dest = "/fluentd/etc/fluent.conf" 4 | keys = [ 5 | "/dd/api/key", 6 | "/dd/service", 7 | "/dd/source", 8 | "/dd/sourcecategory", 9 | "/dd/tags", 10 | "/fluentd/out/kubeuser/buffer/overflow/action", 11 | "/fluentd/out/kubeuser/buffer/chunk/limit/size", 12 | "/fluentd/out/kubeuser/buffer/total/limit/size", 13 | "/fluentd/out/kubeuser/buffer/flush/interval", 14 | "/fluentd/out/kubeuser/buffer/retry/max/interval", 15 | "/fluentd/out/kubeuser/retry/forever", 16 | "/fluentd/out/kubeuser/retry/max/times", 17 | "/fluentd/out/kubeuser/flush/thread/count", 18 | "/fluentd/out/kubesys/buffer/overflow/action", 19 | "/fluentd/out/kubesys/buffer/chunk/limit/size", 20 | "/fluentd/out/kubesys/buffer/total/limit/size", 21 | "/fluentd/out/kubesys/buffer/flush/interval", 22 | "/fluentd/out/kubesys/buffer/retry/max/interval", 23 | "/fluentd/out/kubesys/retry/forever", 24 | "/fluentd/out/kubesys/retry/max/times", 25 | "/fluentd/out/kubesys/flush/thread/count", 26 | ] 27 | -------------------------------------------------------------------------------- /rootfs/etc/confd/templates/application_default_credentials.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "type": "service_account", 3 | "project_id": "{{getv "/google/fluentd/project/id"}}", 4 | "private_key_id": "{{getv "/google/fluentd/private/key/id"}}", 5 | "private_key": "{{getv "/google/fluentd/private/key"}}", 6 | "client_email": "{{getv "/google/fluentd/client/email"}}", 7 | "client_id": "{{getv "/google/fluentd/client/id"}}", 8 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 9 | "token_uri": "https://accounts.google.com/o/oauth2/token", 10 | "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", 11 | "client_x509_cert_url": "{{getv "/google/fluentd/client/x509/cert/url"}}" 12 | } 13 | -------------------------------------------------------------------------------- /rootfs/etc/confd/templates/fluent.conf.tmpl: -------------------------------------------------------------------------------- 1 | # This configuration file for Fluentd / td-agent is used 2 | # to watch changes to Docker log files that live in the 3 | # directory /var/lib/docker/containers/ and are symbolically 4 | # linked to from the /var/log directory using names that capture the 5 | # pod name and container name. These logs are then submitted to 6 | # Google Cloud Logging which assumes the installation of the cloud-logging plug-in. 7 | # 8 | # Example 9 | # ======= 10 | # A line in the Docker log file might like like this JSON: 11 | # 12 | # {"log":"2014/09/25 21:15:03 Got request with path wombat\n", 13 | # "stream":"stderr", 14 | # "time":"2014-09-25T21:15:03.499185026Z"} 15 | # 16 | # Currently, the log information is ingested as plain text rather than JSON. 17 | # TODO: When Cloud Logging supports it, ingest as JSON. 18 | # 19 | # The record reformer is used to write the tag to focus on the pod name 20 | # and the Kubernetes container name. For example a Docker container's logs 21 | # might be in the directory: 22 | # /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b 23 | # and in the file: 24 | # 997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log 25 | # where 997599971ee6... is the Docker ID of the running container. 26 | # The Kubernetes kubelet makes a symbolic link to this file on the host machine 27 | # in the /var/log/containers directory which includes the pod name and the Kubernetes 28 | # container name: 29 | # synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log 30 | # -> 31 | # /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log 32 | # The /var/log directory on the host is mapped to the /var/log directory in the container 33 | # running this instance of Fluentd and we end up collecting the file: 34 | # /var/log/containers/synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log 35 | # This results in the tag: 36 | # var.log.containers.synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log 37 | # The record reformer is used is discard the var.log.containers prefix and 38 | # the Docker container ID suffix and "kubernetes." is pre-pended giving the 39 | # final tag which is ingested into Elasticsearch: 40 | # kubernetes.synthetic-logger-0.25lps-pod_default-synth-lgr 41 | # This makes it easier for users to search for logs by pod name or by 42 | # the name of the Kubernetes container regardless of how many times the 43 | # Kubernetes pod has been restarted (resulting in a several Docker container IDs). 44 | 45 | # See https://github.com/fluent/fluentd/blob/v1.0.0/lib/fluent/system_config.rb#L50 for available config params 46 | 47 | 48 | format json 49 | 50 | 51 | 52 | # Do not directly collect fluentd's own logs to avoid infinite loops. 53 | 54 | @type null 55 | 56 | 57 | # Expose prometheus metrics 58 | 59 | @type prometheus 60 | 61 | 62 | # Collect prometheus metrics for output buffers 63 | # See https://github.com/fluent/fluent-plugin-prometheus#prometheus_monitor-input-plugin for more details 64 | 65 | @type monitor_agent 66 | 67 | 68 | @type prometheus_monitor 69 | 70 | host ${hostname} 71 | 72 | 73 | 74 | # Collect prometheus metrics for outputs 75 | 76 | @type prometheus_output_monitor 77 | 78 | host ${hostname} 79 | 80 | 81 | 82 | # Example: 83 | # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"} 84 | 85 | @type tail 86 | format json 87 | time_key time 88 | path /var/log/containers/*.log 89 | pos_file /var/log/gcp-containers.log.pos 90 | time_format %Y-%m-%dT%H:%M:%S.%NZ 91 | # tag reform.* 92 | tag kubernetes.* 93 | read_from_head true 94 | 95 | 96 | # 97 | # @type record_reformer 98 | # enable_ruby true 99 | # tag kubernetes.${tag_suffix[4].split('-')[0..-2].join('-')} 100 | # 101 | 102 | # Example: 103 | # 2015-12-21 23:17:22,066 [salt.state ][INFO ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081 104 | 105 | @type tail 106 | format /^(?