├── filebeat.yml ├── Procfile ├── start-kube-gen.sh ├── kill-filebeat.sh ├── kube-filebeat-daemonset.yaml ├── example-pod.yaml ├── Dockerfile ├── LICENSE ├── .github └── workflows │ └── publish.yml ├── README.md └── filebeat.yml.tmpl /filebeat.yml: -------------------------------------------------------------------------------- 1 | filebeat: 2 | prospectors: 3 | 4 | output: 5 | console: 6 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | kubeproxy: kubectl proxy 2 | kubegen: /app/start-kube-gen.sh 3 | filebeat: filebeat -e -c /app/filebeat.yml 4 | -------------------------------------------------------------------------------- /start-kube-gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sleep 1 4 | exec kube-gen -watch -type pods -wait 2s:10s -post-cmd '/app/kill-filebeat.sh' -host http://localhost:8001 /app/filebeat.yml.tmpl /app/filebeat.yml 5 | -------------------------------------------------------------------------------- /kill-filebeat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cnt=0 4 | pid=$(pgrep -x filebeat) 5 | while kill -0 $pid > /dev/null 2>&1 6 | do 7 | if [ $cnt -gt "0" ]; then 8 | sleep $((2**$cnt)) 9 | fi 10 | 11 | if [ $cnt -lt "3" ]; then 12 | echo "Killing filebeat pid:$pid" 13 | kill $pid > /dev/null 2>&1 14 | else 15 | echo "Force-killing filebeat pid:$pid" 16 | kill -9 $pid > /dev/null 2>&1 17 | fi 18 | cnt=$((cnt+1)) 19 | done 20 | -------------------------------------------------------------------------------- /kube-filebeat-daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | name: "kube-filebeat" 5 | annotations: 6 | description: "automated log shipper powered by annotations" 7 | spec: 8 | template: 9 | spec: 10 | containers: 11 | - 12 | name: "kube-filebeat" 13 | image: "kylemcc/kube-filebeat:latest" 14 | env: 15 | - 16 | name: LOGSTASH_HOSTS 17 | value: logstash.default.svc.cluster.local:5044 18 | - 19 | name: KUBERNETES_API_URL 20 | value: http://10.1.2.3:8080 21 | volumeMounts: 22 | - name: docker 23 | mountPath: /var/lib/docker 24 | imagePullPolicy: "Always" 25 | restartPolicy: "Always" 26 | volumes: 27 | - name: docker 28 | hostPath: 29 | path: /var/lib/docker 30 | -------------------------------------------------------------------------------- /example-pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | annotations: 5 | kube_filebeat: > 6 | [ 7 | { 8 | "log": "/var/log/example-app/output.log", 9 | "ignore_older": "24h", 10 | "close_older": "24h", 11 | "fields": { 12 | "app": "example-app", 13 | "version": "1.2.3" 14 | }, 15 | "multiline": { 16 | "pattern": "^(([[:alpha:]]{3} [0-9]{1,2}, [0-9]{4} [0-9]{1,2}:[0-9]{2}:[0-9]{2})|([0-9]{4}-[0-9]{2}-[0-9]{2}))", 17 | "negate": true, 18 | "match": "after" 19 | } 20 | }, 21 | { 22 | "log": "/var/log/nginx/access.log", 23 | "exclude_lines": [".*Go-http-client/1\\.1.*"], 24 | "ignore_older": "24h", 25 | "close_older": "24h", 26 | "fields": { 27 | "app": "example-app", 28 | "version": "1.2.3", 29 | "type": "access_log" 30 | } 31 | } 32 | ] 33 | name: example-app 34 | spec: 35 | containers: 36 | - image: example-app:1.2.3 37 | name: example-app 38 | 39 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:jessie 2 | 3 | LABEL version="0.2.0" 4 | 5 | # install forego, kube-gen, kubectl, and filebeat 6 | ENV KUBE_GEN_VERSION 0.4.0 7 | ENV FILEBEAT_VERSION 5.4.0 8 | ADD https://storage.googleapis.com/kubernetes-release/release/v1.8.15/bin/linux/amd64/kubectl /usr/local/bin/ 9 | ADD https://bin.equinox.io/c/ekMN3bCZFUn/forego-stable-linux-amd64.tgz /tmp 10 | ADD https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-$FILEBEAT_VERSION-linux-x86_64.tar.gz /tmp 11 | ADD https://github.com/kylemcc/kube-gen/releases/download/$KUBE_GEN_VERSION/kube-gen-linux-amd64-$KUBE_GEN_VERSION.tar.gz /tmp 12 | RUN tar -C /usr/local/bin -xzvf /tmp/forego-stable-linux-amd64.tgz \ 13 | && rm /tmp/forego-stable-linux-amd64.tgz \ 14 | && tar -C /tmp -xvzf /tmp/filebeat-$FILEBEAT_VERSION-linux-x86_64.tar.gz \ 15 | && mv /tmp/filebeat-$FILEBEAT_VERSION-linux-x86_64/filebeat /usr/local/bin \ 16 | && rm -r /tmp/filebeat-$FILEBEAT_VERSION-linux-x86_64 /tmp/filebeat-$FILEBEAT_VERSION-linux-x86_64.tar.gz \ 17 | && tar -C /usr/local/bin -xvzf /tmp/kube-gen-linux-amd64-$KUBE_GEN_VERSION.tar.gz \ 18 | && rm /tmp/kube-gen-linux-amd64-$KUBE_GEN_VERSION.tar.gz \ 19 | && chmod +x /usr/local/bin/forego \ 20 | && chmod +x /usr/local/bin/filebeat \ 21 | && chmod +x /usr/local/bin/kubectl \ 22 | && chmod +x /usr/local/bin/kube-gen 23 | 24 | COPY . /app/ 25 | WORKDIR /app/ 26 | 27 | ENTRYPOINT ["forego", "start", "-r"] 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Kyle McCullough 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of the Kyle McCullough nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - master 5 | tags: 6 | - '*' 7 | pull_request: {} 8 | name: Build and Publish 9 | jobs: 10 | build_and_publish: 11 | name: Build and Publish 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@master 15 | - name: '[Dockerhub] Docker login' 16 | uses: actions/docker/login@master 17 | env: 18 | DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} 19 | DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} 20 | - name: '[Github] Docker login' 21 | uses: actions/docker/login@master 22 | env: 23 | DOCKER_PASSWORD: '${{ secrets.GH_DOCKER_PASSWORD }}' 24 | DOCKER_USERNAME: $GITHUB_ACTOR 25 | DOCKER_REGISTRY_URL: docker.pkg.github.com 26 | - name: Build Image 27 | uses: actions/docker/cli@master 28 | with: 29 | args: build -t kube-filebeat . 30 | - name: '[Dockerhub] Tag Image' 31 | uses: actions/docker/tag@master 32 | if: github.ref == 'refs/heads/master' || github.event_name == 'release' 33 | with: 34 | args: --env kube-filebeat kylemcc/kube-filebeat 35 | - name: '[Github] Tag Image' 36 | uses: actions/docker/tag@master 37 | if: github.ref == 'refs/heads/master' || github.event_name == 'release' 38 | with: 39 | args: --env kube-filebeat docker.pkg.github.com/kylemcc/kube-filebeat/kube-filebeat 40 | - name: '[Dockerhub] Push Image' 41 | uses: actions/docker/cli@master 42 | if: github.ref == 'refs/heads/master' || github.event_name == 'release' 43 | with: 44 | args: push kylemcc/kube-filebeat 45 | - name: '[Github] Push Image' 46 | uses: actions/docker/cli@master 47 | if: github.ref == 'refs/heads/master' || github.event_name == 'release' 48 | with: 49 | args: push docker.pkg.github.com/kylemcc/kube-filebeat/kube-filebeat 50 | - name: Send Slack Notification 51 | uses: kylemcc/actions/slack-webhook@master 52 | if: always() && (github.ref == 'refs/heads/master' || github.event_name == 'release') 53 | env: 54 | SLACK_MESSAGE: '$GITHUB_REPOSITORY: Build ${{ job.status }}' 55 | SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kube-filebeat 2 | 3 | ![License BSD](https://img.shields.io/badge/license-BSD-red.svg?style=flat) [![](https://img.shields.io/docker/stars/kylemcc/kube-filebeat.svg?style=flat)](https://hub.docker.com/r/kylemcc/kube-filebeat 'DockerHub') [![](https://img.shields.io/docker/pulls/kylemcc/kube-filebeat.svg?style=flat)](https://hub.docker.com/r/kylemcc/kube-filebeat 'DockerHub') 4 | 5 | `kube-filebeat` is a Docker container running [filebeat][1] and [kube-gen][2]. `kube-gen` watches for events on the Kubernetes API and generates filebeat configurations (based on Pod annotations) to harvest logs from applications running in Kubernetes and ship them to logstash. 6 | 7 | **Note**: This project is mostly experimental. It relies on and exploits the mechanics of Docker's filesystem layer. The implementation here only works for Docker versions >= 1.10.0 and may break at any time. 8 | 9 | ## Usage 10 | 11 | Due to the mechanics of how `kube-filebeat` operates, it needs to be running on any node from which you would like to collect logs. The recommended way to acheive this is to run `kube-filebeat` as a [Daemon Set][3]. [For example][4]: 12 | 13 | ```yaml 14 | apiVersion: extensions/v1beta1 15 | kind: DaemonSet 16 | metadata: 17 | name: "kube-filebeat" 18 | annotations: 19 | description: "automated log shipper powered by annotations" 20 | spec: 21 | template: 22 | spec: 23 | containers: 24 | - 25 | name: "kube-filebeat" 26 | image: "kylemcc/kube-filebeat:latest" 27 | env: 28 | - 29 | name: LOGSTASH_HOSTS 30 | value: logstash.default.svc.cluster.local:5044 31 | - 32 | name: KUBERNETES_API_URL 33 | value: http://10.1.2.3:8080 34 | volumeMounts: 35 | - name: docker 36 | mountPath: /var/lib/docker 37 | imagePullPolicy: "Always" 38 | restartPolicy: "Always" 39 | volumes: 40 | - name: docker 41 | hostPath: 42 | path: /var/lib/docker 43 | ``` 44 | 45 | ### Configuration 46 | 47 | Annotations are used to inform `kube-filebeat` of files that should be harvested. [For example][5]: 48 | 49 | ```yaml 50 | apiVersion: v1 51 | kind: Pod 52 | metadata: 53 | annotations: 54 | kube_filebeat: > 55 | [ 56 | { 57 | "log": "/var/log/example-app/output.log", 58 | "ignore_older": "24h", 59 | "close_older": "24h", 60 | "fields": { 61 | "app": "example-app", 62 | "version": "1.2.3" 63 | }, 64 | "multiline": { 65 | "pattern": "^(([[:alpha:]]{3} [0-9]{1,2}, [0-9]{4} [0-9]{1,2}:[0-9]{2}:[0-9]{2})|([0-9]{4}-[0-9]{2}-[0-9]{2}))", 66 | "negate": true, 67 | "match": "after" 68 | } 69 | }, 70 | { 71 | "log": "/var/log/nginx/access.log", 72 | "exclude_lines": [".*Go-http-client/1\\.1.*"], 73 | "ignore_older": "24h", 74 | "close_older": "24h", 75 | "fields": { 76 | "app": "example-app", 77 | "version": "1.2.3", 78 | "type": "access_log" 79 | } 80 | } 81 | ] 82 | name: example-app 83 | spec: 84 | containers: 85 | - image: example-app:1.2.3 86 | name: example-app 87 | 88 | ``` 89 | 90 | For multi-container pods, specify the container name in each filebeat config. E.g.: 91 | ```yaml 92 | apiVersion: v1 93 | kind: Pod 94 | metadata: 95 | annotations: 96 | kube_filebeat: > 97 | [ 98 | { 99 | "container": "example-app", 100 | "log": "/var/log/app/logfile", 101 | ... 102 | }, 103 | { 104 | "container": "nginx", 105 | "log": "/var/log/nginx/access.log", 106 | ... 107 | } 108 | ] 109 | spec: 110 | containers: 111 | - image: example-app:1.2.3 112 | name: example-app 113 | - image: nginx:latest 114 | name: nginx 115 | ``` 116 | 117 | [1]: https://github.com/elastic/beats/tree/master/filebeat 118 | [2]: https://github.com/kylemcc/kube-gen 119 | [3]: http://kubernetes.io/docs/admin/daemons/ 120 | [4]: https://github.com/kylemcc/kube-filebeat/blob/master/kube-filebeat-daemonset.yaml 121 | [5]: https://github.com/kylemcc/kube-filebeat/blob/master/example-pod.yaml 122 | -------------------------------------------------------------------------------- /filebeat.yml.tmpl: -------------------------------------------------------------------------------- 1 | {{ $pods := whereExist .Pods "ObjectMeta.Annotations.kube_filebeat" -}} 2 | 3 | filebeat: 4 | prospectors: 5 | {{- range $pod := $pods -}} 6 | {{ $containerMap := groupBy $pod.Status.ContainerStatuses "Name" }} 7 | {{ $configs := parseJsonSafe $pod.ObjectMeta.Annotations.kube_filebeat -}} 8 | {{- range $conf := $configs }} 9 | {{ $container := coalesce (first (index $containerMap (coalesce $conf.container ""))) (first $pod.Status.ContainerStatuses) -}} 10 | {{ $containerId := trimPrefix $container.ContainerID "docker://" -}} 11 | {{ if exists (printf "/var/lib/docker/containers/%s" $containerId) }} 12 | 13 | {{ $res := shell (printf "cat /var/lib/docker/image/overlay2/layerdb/mounts/%s/mount-id" $containerId) -}} 14 | {{ $baseDir := printf "/var/lib/docker/overlay2/%s/merged" $res.Stdout }} 15 | 16 | - 17 | # Paths that should be crawled and fetched. Glob based paths. 18 | # To fetch all ".log" files from a specific level of subdirectories 19 | # /var/log/*/*.log can be used. 20 | # For each file found under this path, a harvester is started. 21 | # Make sure not file is defined twice as this can lead to unexpected behaviour. 22 | paths: 23 | {{- $containerConfig := parseJsonSafe (shell (printf "cat /var/lib/docker/containers/%s/config.v2.json" $containerId)).Stdout -}} 24 | {{- $logFileData := split $conf.log "/" -}} 25 | {{- $logFile := last $logFileData -}} 26 | {{- $logPath := pathJoin "/" (pathJoinSlice (slice $logFileData 0 (add (len $logFileData) -1))) -}} 27 | {{- if hasField $containerConfig.MountPoints $logPath }} 28 | - {{ pathJoin (printf "/var/lib/docker/volumes/%s/_data" (index $containerConfig.MountPoints $logPath).Name) $logFile -}} 29 | {{- else }} 30 | - {{ pathJoin $baseDir $conf.log }} 31 | {{- end }} 32 | 33 | input_type: log 34 | 35 | # Exclude files. A list of regular expressions to match. Filebeat drops the files that 36 | # are matching any regular expression from the list. By default, no files are dropped. 37 | {{- if hasField $conf "exclude_files" }} 38 | exclude_files: 39 | {{- range $f := $conf.exclude_files }} 40 | - '{{ $f }}' 41 | {{- end }} 42 | {{ else }} 43 | exclude_files: [".gz$"] 44 | {{- end }} 45 | 46 | # Ignore files which were modified more then the defined timespan in the past. 47 | # In case all files on your system must be read you can set this value very large. 48 | # Time strings like 2h (2 hours), 5m (5 minutes) can be used. 49 | ignore_older: {{ coalesce $conf.ignore_older "" }} 50 | 51 | # Close older closes the file handler for which were not modified 52 | # for longer then close_older 53 | # Time strings like 2h (2 hours), 5m (5 minutes) can be used. 54 | close_older: {{ coalesce $conf.close_older "" }} 55 | 56 | # Exclude lines. A list of regular expressions to match. It drops the lines that are 57 | # matching any regular expression from the list. The include_lines is called before 58 | # exclude_lines. By default, no lines are dropped. 59 | exclude_lines: 60 | {{- range $l := $conf.exclude_lines }} 61 | - '{{ $l }}' 62 | {{- end }} 63 | 64 | fields: 65 | path: {{ $conf.log }} 66 | pod: {{ $pod.ObjectMeta.Name }} 67 | namespace: {{ $pod.ObjectMeta.Namespace }} 68 | {{- range $key, $value := $conf.fields }} 69 | {{ $key }}: {{ $value }} 70 | {{- end }}{{/* end range fields */}} 71 | 72 | fields_under_root: true 73 | 74 | {{- if hasField $conf "multiline" }} 75 | {{- with $conf.multiline }} 76 | multiline: 77 | pattern: '{{ .pattern }}' 78 | negate: {{ coalesce .negate false }} 79 | match: {{ coalesce .match "after" }} 80 | max_lines: {{ coalesce .max_lines "500" }} 81 | timeout: {{ coalesce .timeout "5s" }} 82 | {{- end }}{{/* end with */}} 83 | {{- end -}}{{/* end multiline */}} 84 | 85 | {{- end }}{{/* end directory exists */}} 86 | 87 | {{- end }}{{/* end range configs */}} 88 | 89 | {{- end }}{{/* end range $pods */}} 90 | 91 | # Defines how often the spooler is flushed. After idle_timeout the spooler is 92 | # Flush even though spool_size is not reached. 93 | idle_timeout: 5s 94 | ############################# Output ########################################## 95 | 96 | # Configure what outputs to use when sending the data collected by the beat. 97 | # Multiple outputs may be used. 98 | output: 99 | {{- if mapContains .Env "LOGSTASH_HOSTS" }} 100 | logstash: 101 | # The Logstash hosts 102 | hosts: 103 | {{- range $host := split .Env.LOGSTASH_HOSTS "," }} 104 | - {{ $host }} 105 | {{- end }} 106 | 107 | # Number of workers per Logstash host. 108 | worker: 1 109 | 110 | # Set gzip compression level. 111 | compression_level: 3 112 | 113 | # Optional load balance the events between the Logstash hosts 114 | loadbalance: true 115 | 116 | # Optional index name. The default index name depends on the each beat. 117 | # For Packetbeat, the default is set to packetbeat, for Topbeat 118 | # top topbeat and for Filebeat to filebeat. 119 | #index: filebeat 120 | {{ end -}} 121 | {{- if mapContains .Env "CONSOLE_OUTPUT" }} 122 | console: 123 | {{ end -}} 124 | --------------------------------------------------------------------------------