├── .gitignore ├── log-generator ├── .dockerignore ├── go.mod ├── Dockerfile ├── go.sum └── log-generator.go ├── log-output ├── .dockerignore ├── go.mod ├── go.sum ├── Dockerfile ├── log-output.go └── pkg │ └── index │ ├── template.go │ ├── index.go │ └── service.go ├── charts └── rancher-logging-example │ ├── templates │ ├── NOTES.txt │ ├── senarios │ │ ├── all-logs.yaml │ │ ├── namespace-logs.yaml │ │ └── logs-by-label.yaml │ ├── validate-rancher-logging-installed.yaml │ └── _helpers.tpl │ ├── .helmignore │ ├── charts │ ├── log-output │ │ ├── templates │ │ │ ├── service.yaml │ │ │ ├── ingress.yaml │ │ │ ├── _helpers.tpl │ │ │ └── deployment.yaml │ │ ├── .helmignore │ │ ├── values.yaml │ │ └── Chart.yaml │ └── log-generator │ │ ├── .helmignore │ │ ├── values.yaml │ │ ├── Chart.yaml │ │ └── templates │ │ ├── deployment.yaml │ │ ├── labeled-deployment.yaml │ │ └── _helpers.tpl │ ├── values.yaml │ └── Chart.yaml ├── .github └── workflows │ ├── validate.yaml │ └── release.yaml ├── scripts └── validate ├── Makefile └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .idea -------------------------------------------------------------------------------- /log-generator/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | 3 | !/go.mod 4 | !/go.sum 5 | !/log-generator.go -------------------------------------------------------------------------------- /log-output/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/pkg/** 3 | !/go.mod 4 | !/go.sum 5 | !/log-output.go 6 | -------------------------------------------------------------------------------- /log-output/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/paynejacob/rancher-logging-examples/log-output 2 | 3 | go 1.15 4 | 5 | require github.com/gorilla/mux v1.8.0 6 | -------------------------------------------------------------------------------- /log-output/go.sum: -------------------------------------------------------------------------------- 1 | github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= 2 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 3 | -------------------------------------------------------------------------------- /log-generator/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/paynejacob/log-generator 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/drhodes/golorem v0.0.0-20160418191928-ecccc744c2d9 7 | github.com/kelseyhightower/envconfig v1.4.0 8 | ) 9 | -------------------------------------------------------------------------------- /log-output/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15.7-alpine3.13 as build 2 | WORKDIR /build 3 | COPY . . 4 | RUN go build -o log-output log-output.go 5 | 6 | FROM alpine:3.13 7 | COPY --from=build /build/log-output /usr/local/bin/log-output 8 | ENTRYPOINT ["log-output"] -------------------------------------------------------------------------------- /charts/rancher-logging-example/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 1. Get the Logging output URL by running the following command: 2 | 3 | kubectl port-forward -n {{ .Release.Namespace }} svc/{{ .Release.Name }}-log-output 8080:80 4 | 5 | 2. Go to http://localhost:8080 to view your logs. 6 | -------------------------------------------------------------------------------- /.github/workflows/validate.yaml: -------------------------------------------------------------------------------- 1 | name: validate 2 | 3 | on: pull_request 4 | 5 | jobs: 6 | validate: 7 | name: validate 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - uses: actions/setup-go@v2 12 | - run: scripts/validate -------------------------------------------------------------------------------- /log-generator/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15.7-alpine3.13 as build 2 | WORKDIR /build 3 | COPY go.mod . 4 | COPY go.sum . 5 | COPY log-generator.go . 6 | RUN go build -o log-generator log-generator.go 7 | 8 | FROM alpine:3.13 9 | COPY --from=build /build/log-generator /usr/local/bin/log-generator 10 | ENTRYPOINT ["log-generator"] -------------------------------------------------------------------------------- /log-generator/go.sum: -------------------------------------------------------------------------------- 1 | github.com/drhodes/golorem v0.0.0-20160418191928-ecccc744c2d9 h1:EQOZw/LCQ0SM4sNez3EhUf9gQalQrLrs4mPtmQa+d58= 2 | github.com/drhodes/golorem v0.0.0-20160418191928-ecccc744c2d9/go.mod h1:NsKVpF4h4j13Vm6Cx7Kf0V03aJKjfaStvm5rvK4+FyQ= 3 | github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= 4 | github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= 5 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-output/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "log-output.fullname" . }} 5 | labels: 6 | {{- include "log-output.labels" . | nindent 4 }} 7 | spec: 8 | type: {{ .Values.service.type }} 9 | ports: 10 | - port: {{ .Values.service.port }} 11 | targetPort: http 12 | protocol: TCP 13 | name: http 14 | selector: 15 | {{- include "log-output.selectorLabels" . | nindent 4 }} 16 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-generator/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-output/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /log-output/log-output.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | 7 | "github.com/gorilla/mux" 8 | . "github.com/paynejacob/rancher-logging-examples/log-output/pkg/index" 9 | ) 10 | 11 | func main() { 12 | service := NewIndexService() 13 | 14 | r := mux.NewRouter() 15 | r.HandleFunc("/", service.ListIndices).Methods("GET") 16 | r.HandleFunc("/{index_name}/", service.ListLogs).Methods("GET") 17 | r.HandleFunc("/{index_name}/", service.WriteLog).Methods("POST") 18 | 19 | http.Handle("/", r) 20 | 21 | log.Fatal(http.ListenAndServe(":80", nil)) 22 | } 23 | -------------------------------------------------------------------------------- /scripts/validate: -------------------------------------------------------------------------------- 1 | pushd log-generator 2 | go mod tidy 3 | go fmt log-generator.go 4 | popd 5 | 6 | pushd log-output 7 | go mod tidy 8 | go fmt log-output.go 9 | go fmt pkg/index 10 | popd 11 | 12 | if [ -n "$(git status --porcelain --untracked-files=no)" ]; then 13 | echo "bad go formatting or out of date go.mod file!" 14 | git status 15 | exit 1 16 | fi 17 | 18 | docker build -f log-generator/Dockerfile log-generator || exit 1 19 | docker build -f log-output/Dockerfile log-output || exit 1 20 | 21 | helm lint --with-subcharts charts/rancher-logging-example 22 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-generator/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for log-generator. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | replicaCount: 1 6 | 7 | image: 8 | repository: paynejacob/log-generator 9 | pullPolicy: Always 10 | # Overrides the image tag whose default is the chart appVersion. 11 | tag: "latest" 12 | 13 | imagePullSecrets: [] 14 | nameOverride: "" 15 | fullnameOverride: "" 16 | 17 | podAnnotations: {} 18 | 19 | podSecurityContext: {} 20 | 21 | securityContext: {} 22 | 23 | resources: {} 24 | 25 | nodeSelector: {} 26 | 27 | tolerations: [] 28 | 29 | affinity: {} 30 | -------------------------------------------------------------------------------- /log-output/pkg/index/template.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import "time" 4 | 5 | const listPageTemplate = ` 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | {{range .}} 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | {{ end }} 24 | 25 |
NameFirst LogLast LogLog Count
{{.Name}}{{.FirstLog | DateFormat }}{{.LastLog | DateFormat}}{{.LogCount}}view logs
26 | 27 | ` 28 | 29 | func DateFormat(t time.Time) string { 30 | return t.Format(time.RFC3339) 31 | } -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | UNAME:=$(shell uname -s) 2 | MAKEPATH:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) 3 | NAMESPACE:=rancher-logging-example 4 | NAME:=$(NAMESPACE) 5 | 6 | install: uninstall 7 | helm install -n $(NAMESPACE) --create-namespace $(NAME) $(MAKEPATH)/charts/rancher-logging-example 8 | 9 | uninstall: 10 | -helm uninstall -n $(NAMESPACE) $(NAME) 11 | -kubectl delete namespace $(NAMESPACE) 12 | 13 | port-forward: 14 | kubectl port-forward -n $(NAMESPACE) svc/$(NAMESPACE)-log-output 8080:80 15 | 16 | open-log-output: 17 | ifeq ($(UNAME), Darwin) 18 | open http://localhost:8080 19 | else ifeq ($(UNAME), Linux) 20 | xdg-open http://localhost:8080 21 | else 22 | explorer http://localhost:8080 23 | endif 24 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/templates/senarios/all-logs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: logging.banzaicloud.io/v1beta1 2 | kind: ClusterFlow 3 | metadata: 4 | name: all-logs 5 | namespace: cattle-logging-system 6 | labels: 7 | {{- include "rancher-logging-example.labels" . | nindent 4 }} 8 | spec: 9 | globalOutputRefs: 10 | - all-logs 11 | --- 12 | apiVersion: logging.banzaicloud.io/v1beta1 13 | kind: ClusterOutput 14 | metadata: 15 | name: all-logs 16 | namespace: cattle-logging-system 17 | labels: 18 | {{- include "rancher-logging-example.labels" . | nindent 4 }} 19 | spec: 20 | http: 21 | endpoint: http://{{ .Release.Name }}-log-output.{{ .Release.Namespace }}.svc/all-logs/ 22 | buffer: 23 | flush_interval: 10s 24 | flush_mode: interval 25 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/templates/senarios/namespace-logs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: logging.banzaicloud.io/v1beta1 2 | kind: Flow 3 | metadata: 4 | name: {{ .Release.Namespace }}-namespace 5 | labels: 6 | {{- include "rancher-logging-example.labels" . | nindent 4 }} 7 | spec: 8 | localOutputRefs: 9 | - {{ .Release.Namespace }}-namespace 10 | --- 11 | apiVersion: logging.banzaicloud.io/v1beta1 12 | kind: Output 13 | metadata: 14 | name: {{ .Release.Namespace }}-namespace 15 | labels: 16 | {{- include "rancher-logging-example.labels" . | nindent 4 }} 17 | spec: 18 | http: 19 | endpoint: http://{{ .Release.Name }}-log-output.{{ .Release.Namespace }}.svc/{{ .Release.Namespace }}-namespace/ 20 | buffer: 21 | flush_interval: 10s 22 | flush_mode: interval 23 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for rancher-logging-example. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | log-generator: 5 | replicaCount: 2 6 | resources: 7 | limits: 8 | memory: 16Mi 9 | requests: 10 | memory: 4Mi 11 | nodeSelector: 12 | kubernetes.io/os: linux 13 | tolerations: 14 | - key: cattle.io/os 15 | operator: "Equal" 16 | value: "linux" 17 | effect: NoSchedule 18 | log-output: 19 | replicaCount: 1 20 | resources: 21 | limits: 22 | memory: 16Mi 23 | requests: 24 | memory: 8Mi 25 | nodeSelector: 26 | kubernetes.io/os: linux 27 | tolerations: 28 | - key: cattle.io/os 29 | operator: "Equal" 30 | value: "linux" 31 | effect: NoSchedule 32 | -------------------------------------------------------------------------------- /log-output/pkg/index/index.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | const MaxIndexLogs = 100 8 | 9 | type Index struct { 10 | Name string `json:"name"` 11 | FirstLog time.Time `json:"first_log"` 12 | LastLog time.Time `json:"last_log"` 13 | LogCount int64 `json:"log_count"` 14 | Logs [][]byte `json:"-"` 15 | } 16 | 17 | func NewIndex(name string) Index { 18 | return Index{ 19 | Name: name, 20 | Logs: make([][]byte, MaxIndexLogs), 21 | } 22 | } 23 | 24 | func WriteLog(index *Index, _log []byte) { 25 | if index.FirstLog.IsZero() { 26 | index.FirstLog = time.Now() 27 | } 28 | index.LastLog = time.Now() 29 | 30 | index.LogCount++ 31 | 32 | if index.LogCount > MaxIndexLogs { 33 | index.Logs = append(index.Logs[1:], _log) 34 | } else { 35 | index.Logs[index.LogCount-1] = _log 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/templates/senarios/logs-by-label.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: logging.banzaicloud.io/v1beta1 2 | kind: Flow 3 | metadata: 4 | name: log-generator-app 5 | labels: 6 | {{- include "rancher-logging-example.labels" . | nindent 4 }} 7 | spec: 8 | localOutputRefs: 9 | - log-generator-app 10 | match: 11 | - select: 12 | labels: 13 | rancher.logging.example/app: log-generator 14 | --- 15 | apiVersion: logging.banzaicloud.io/v1beta1 16 | kind: Output 17 | metadata: 18 | name: log-generator-app 19 | labels: 20 | {{- include "rancher-logging-example.labels" . | nindent 4 }} 21 | spec: 22 | http: 23 | endpoint: http://{{ .Release.Name }}-log-output.{{ .Release.Namespace }}.svc/log-generator-app/ 24 | buffer: 25 | flush_interval: 10s 26 | flush_mode: interval 27 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-output/values.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | repository: paynejacob/log-output 3 | pullPolicy: Always 4 | # Overrides the image tag whose default is the chart appVersion. 5 | tag: "latest" 6 | 7 | imagePullSecrets: [] 8 | nameOverride: "" 9 | fullnameOverride: "" 10 | 11 | podAnnotations: {} 12 | 13 | podSecurityContext: {} 14 | 15 | securityContext: {} 16 | 17 | service: 18 | type: ClusterIP 19 | port: 80 20 | 21 | ingress: 22 | enabled: false 23 | annotations: {} 24 | # kubernetes.io/ingress.class: nginx 25 | # kubernetes.io/tls-acme: "true" 26 | hosts: 27 | - host: chart-example.local 28 | paths: [] 29 | tls: [] 30 | # - secretName: chart-example-tls 31 | # hosts: 32 | # - chart-example.local 33 | 34 | resources: {} 35 | 36 | nodeSelector: {} 37 | 38 | tolerations: [] 39 | 40 | affinity: {} 41 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/templates/validate-rancher-logging-installed.yaml: -------------------------------------------------------------------------------- 1 | #{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} 2 | # {{- $found := dict -}} 3 | # {{- set $found "logging.banzaicloud.io/v1beta1/ClusterFlow" false -}} 4 | # {{- set $found "logging.banzaicloud.io/v1beta1/ClusterOutput" false -}} 5 | # {{- set $found "logging.banzaicloud.io/v1beta1/Flow" false -}} 6 | # {{- set $found "logging.banzaicloud.io/v1beta1/Logging" false -}} 7 | # {{- set $found "logging.banzaicloud.io/v1beta1/Output" false -}} 8 | # {{- range .Capabilities.APIVersions -}} 9 | # {{- if hasKey $found (toString .) -}} 10 | # {{- set $found (toString .) true -}} 11 | # {{- end -}} 12 | # {{- end -}} 13 | # {{- range $_, $exists := $found -}} 14 | # {{- if (eq $exists false) -}} 15 | # {{- required "Required CRDs are missing. Please install the rancher-logging chart before installing this chart." "" -}} 16 | # {{- end -}} 17 | # {{- end -}} 18 | #{{- end -}} -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-output/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $fullName := include "log-output.fullname" . -}} 3 | {{- $svcPort := .Values.service.port -}} 4 | {{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} 5 | apiVersion: networking.k8s.io/v1beta1 6 | {{- else -}} 7 | apiVersion: extensions/v1beta1 8 | {{- end }} 9 | kind: Ingress 10 | metadata: 11 | name: {{ $fullName }} 12 | labels: 13 | {{- include "log-output.labels" . | nindent 4 }} 14 | {{- with .Values.ingress.annotations }} 15 | annotations: 16 | {{- toYaml . | nindent 4 }} 17 | {{- end }} 18 | spec: 19 | rules: 20 | {{- range .Values.ingress.hosts }} 21 | - host: {{ .host | quote }} 22 | http: 23 | paths: 24 | {{- range .paths }} 25 | - path: {{ .path }} 26 | backend: 27 | serviceName: {{ $fullName }} 28 | servicePort: {{ $svcPort }} 29 | {{- end }} 30 | {{- end }} 31 | {{- end }} 32 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | push-log-generator: 10 | name: push-image 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: docker/login-action@v1 15 | with: 16 | username: ${{ secrets.DOCKERHUB_USERNAME }} 17 | password: ${{ secrets.DOCKERHUB_TOKEN }} 18 | - run: docker build -t paynejacob/log-generator:latest -f log-generator/Dockerfile log-generator 19 | - run: docker push paynejacob/log-generator:latest 20 | push-log-output: 21 | name: push-image 22 | runs-on: ubuntu-latest 23 | steps: 24 | - uses: actions/checkout@v2 25 | - uses: docker/login-action@v1 26 | with: 27 | username: ${{ secrets.DOCKERHUB_USERNAME }} 28 | password: ${{ secrets.DOCKERHUB_TOKEN }} 29 | - run: docker build -t paynejacob/log-output:latest -f log-output/Dockerfile log-output 30 | - run: docker push paynejacob/log-output:latest -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-output/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: log-output 3 | description: A Helm chart for Kubernetes 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.0 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "1.16.0" 25 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-generator/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: log-generator 3 | description: A Helm chart for Kubernetes 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.0 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "1.16.0" 25 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: rancher-logging-example 3 | description: A Helm chart for Kubernetes 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.0 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "1.16.0" 25 | 26 | dependencies: 27 | - name: log-generator 28 | version: 0.1.0 29 | - name: log-output 30 | version: 0.1.0 -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-generator/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "log-generator.fullname" . }} 5 | labels: 6 | {{- include "log-generator.labels" . | nindent 4 }} 7 | spec: 8 | replicas: {{ .Values.replicaCount }} 9 | selector: 10 | matchLabels: 11 | {{- include "log-generator.selectorLabels" . | nindent 6 }} 12 | template: 13 | metadata: 14 | {{- with .Values.podAnnotations }} 15 | annotations: 16 | {{- toYaml . | nindent 8 }} 17 | {{- end }} 18 | labels: 19 | {{- include "log-generator.selectorLabels" . | nindent 8 }} 20 | spec: 21 | {{- with .Values.imagePullSecrets }} 22 | imagePullSecrets: 23 | {{- toYaml . | nindent 8 }} 24 | {{- end }} 25 | securityContext: 26 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 27 | containers: 28 | - name: {{ .Chart.Name }} 29 | securityContext: 30 | {{- toYaml .Values.securityContext | nindent 12 }} 31 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 32 | imagePullPolicy: {{ .Values.image.pullPolicy }} 33 | resources: 34 | {{- toYaml .Values.resources | nindent 12 }} 35 | {{- with .Values.nodeSelector }} 36 | nodeSelector: 37 | {{- toYaml . | nindent 8 }} 38 | {{- end }} 39 | {{- with .Values.affinity }} 40 | affinity: 41 | {{- toYaml . | nindent 8 }} 42 | {{- end }} 43 | {{- with .Values.tolerations }} 44 | tolerations: 45 | {{- toYaml . | nindent 8 }} 46 | {{- end }} 47 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-generator/templates/labeled-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "log-generator.fullname" . }}-labeled 5 | labels: 6 | {{- include "log-generator.labels" . | nindent 4 }} 7 | spec: 8 | replicas: {{ .Values.replicaCount }} 9 | selector: 10 | matchLabels: 11 | {{- include "log-generator.selectorLabels" . | nindent 6 }} 12 | template: 13 | metadata: 14 | {{- with .Values.podAnnotations }} 15 | annotations: 16 | {{- toYaml . | nindent 8 }} 17 | {{- end }} 18 | labels: 19 | {{- include "log-generator.selectorLabels" . | nindent 8 }} 20 | rancher.logging.example/app: log-generator 21 | spec: 22 | {{- with .Values.imagePullSecrets }} 23 | imagePullSecrets: 24 | {{- toYaml . | nindent 8 }} 25 | {{- end }} 26 | securityContext: 27 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 28 | containers: 29 | - name: {{ .Chart.Name }} 30 | securityContext: 31 | {{- toYaml .Values.securityContext | nindent 12 }} 32 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 33 | imagePullPolicy: {{ .Values.image.pullPolicy }} 34 | resources: 35 | {{- toYaml .Values.resources | nindent 12 }} 36 | {{- with .Values.nodeSelector }} 37 | nodeSelector: 38 | {{- toYaml . | nindent 8 }} 39 | {{- end }} 40 | {{- with .Values.affinity }} 41 | affinity: 42 | {{- toYaml . | nindent 8 }} 43 | {{- end }} 44 | {{- with .Values.tolerations }} 45 | tolerations: 46 | {{- toYaml . | nindent 8 }} 47 | {{- end }} 48 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-output/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "log-output.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "log-output.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "log-output.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "log-output.labels" -}} 37 | helm.sh/chart: {{ include "log-output.chart" . }} 38 | {{ include "log-output.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "log-output.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "log-output.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-output/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "log-output.fullname" . }} 5 | labels: 6 | {{- include "log-output.labels" . | nindent 4 }} 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | {{- include "log-output.selectorLabels" . | nindent 6 }} 12 | template: 13 | metadata: 14 | {{- with .Values.podAnnotations }} 15 | annotations: 16 | {{- toYaml . | nindent 8 }} 17 | {{- end }} 18 | labels: 19 | {{- include "log-output.selectorLabels" . | nindent 8 }} 20 | spec: 21 | {{- with .Values.imagePullSecrets }} 22 | imagePullSecrets: 23 | {{- toYaml . | nindent 8 }} 24 | {{- end }} 25 | securityContext: 26 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 27 | containers: 28 | - name: {{ .Chart.Name }} 29 | securityContext: 30 | {{- toYaml .Values.securityContext | nindent 12 }} 31 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 32 | imagePullPolicy: {{ .Values.image.pullPolicy }} 33 | ports: 34 | - name: http 35 | containerPort: 80 36 | protocol: TCP 37 | resources: 38 | {{- toYaml .Values.resources | nindent 12 }} 39 | {{- with .Values.nodeSelector }} 40 | nodeSelector: 41 | {{- toYaml . | nindent 8 }} 42 | {{- end }} 43 | {{- with .Values.affinity }} 44 | affinity: 45 | {{- toYaml . | nindent 8 }} 46 | {{- end }} 47 | {{- with .Values.tolerations }} 48 | tolerations: 49 | {{- toYaml . | nindent 8 }} 50 | {{- end }} 51 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/charts/log-generator/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "log-generator.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "log-generator.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "log-generator.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "log-generator.labels" -}} 37 | helm.sh/chart: {{ include "log-generator.chart" . }} 38 | {{ include "log-generator.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "log-generator.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "log-generator.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | -------------------------------------------------------------------------------- /log-generator/log-generator.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | golorem "github.com/drhodes/golorem" 6 | "github.com/kelseyhightower/envconfig" 7 | "log" 8 | "math/rand" 9 | "os" 10 | "text/template" 11 | "time" 12 | ) 13 | 14 | type config struct { 15 | LogFormat string `default:"[{{.Timestamp}}] @{{.Level}} {{.Message}} {{.Number}} ::{{.Decimal}}"` 16 | 17 | Frequency time.Duration `default:"1s"` 18 | ConstantRate bool `default:"true"` 19 | } 20 | 21 | type logLevel string 22 | 23 | const ( 24 | DEBUG logLevel = "DEBUG" 25 | INFO logLevel = "INFO" 26 | WARNING logLevel = "WARNING" 27 | ERROR logLevel = "ERROR" 28 | CRITICAL logLevel = "CRITICAL" 29 | ) 30 | 31 | func main() { 32 | logLevels := []logLevel{DEBUG, INFO, WARNING, ERROR, CRITICAL} 33 | var logCount int64 34 | var delay time.Duration 35 | var config = &config{} 36 | 37 | r := rand.New(rand.NewSource(time.Now().Unix())) 38 | 39 | if err := envconfig.Process("", config); err != nil { 40 | log.Fatalf("failed to parse configuration from environment: %v", err) 41 | } 42 | 43 | for { 44 | levelIndex := r.Intn(len(logLevels)) 45 | 46 | _ = template.Must(template.New("").Parse(config.LogFormat+"\n")).Execute(os.Stdout, map[string]string{ 47 | "Timestamp": time.Now().Format(time.RFC3339), 48 | "Level": string(logLevels[levelIndex]), 49 | "Message": golorem.Sentence(5, 10), 50 | "Number": fmt.Sprintf("%d", logCount), 51 | "Decimal": fmt.Sprintf("%f", r.Float64()), 52 | }) 53 | 54 | if config.ConstantRate { 55 | delay, _ = time.ParseDuration(fmt.Sprintf("%ds", int(config.Frequency.Seconds()))) 56 | } else { 57 | delay, _ = time.ParseDuration(fmt.Sprintf("%ds", r.Intn(int(config.Frequency.Seconds())))) 58 | } 59 | 60 | logCount++ 61 | 62 | time.Sleep(delay) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /charts/rancher-logging-example/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "rancher-logging-example.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "rancher-logging-example.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "rancher-logging-example.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "rancher-logging-example.labels" -}} 37 | helm.sh/chart: {{ include "rancher-logging-example.chart" . }} 38 | {{ include "rancher-logging-example.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "rancher-logging-example.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "rancher-logging-example.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rancher Logging Examples 2 | 3 | This repository contains an example chart for testing your logging setup. 4 | 5 | ## Charts 6 | 7 | ### Rancher Logging Example 8 | The `rancher-logging-example` is an end to end setup of an application, multiple configurations for log collection, and a simple output. This chart is recommended for running an example payload for the [rancher-logging](https://rancher.com/docs/rancher/v2.x/en/logging/v2.5/) chart. 9 | 10 | This chart demonstrates 3 scenarios: 11 | 12 | 1. `all-logs` collects all logs in the cluster 13 | 2. `logs-by-label` collects all logs for pods with the label `rancher.logging.example/app: log-generator` 14 | 3. `namespace-logs` collects all logs for the namespace this chart is deployed in. 15 | 16 | #### Installation 17 | To install this chart run: 18 | 19 | ```shell 20 | git clone https://github.com/paynejacob/rancher-logging-examples 21 | cd rancher-logging-examples 22 | make install 23 | ``` 24 | 25 | 26 | #### Checking the output 27 | 28 | To see information about the logs being collected you will need a proxy to the log output service. The helm notes will include instructions for setting this up after a successful installation. Then, you can view your logs by going to [http://localhost:8080](http://localhost:8080). It can take a few seconds for logs to show up. 29 | 30 | ## Log Generator 31 | 32 | `docker run paynejacob/log-generator:latest` 33 | 34 | The log generator is a simple application that will constantly output logs. 35 | 36 | You can tweak the log output by setting the following environment variables: 37 | 38 | - `LOGFORMAT` a go template string for the generated logs 39 | - `FREQUENCY` the maximum delay between log messages 40 | - `CONSTANTRATE` by default the log generator will output messages every `FREQUENCY` seconds. If this variable is set to true messages will be logged every `FREQUENCY` seconds. 41 | 42 | ## Log Output 43 | 44 | `docker run -p 80:80 paynejacob/log-output:latest` 45 | 46 | The log output is a simple http service that can receive logs and be used to test a logging setup. A `GET /` will return a summary of all logs received by the service. Logs are organized into indices. Think of an index as a folder for logs. The logs are written by preforming `POST //` with one or more log messages. A `GET //` will return the last 100 logs received by the service for the given index. **Logs are stored in memory and all statistics and logs will be lost if the pod is restarted.** 47 | -------------------------------------------------------------------------------- /log-output/pkg/index/service.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "net/http" 7 | "sync" 8 | "text/template" 9 | 10 | "github.com/gorilla/mux" 11 | ) 12 | 13 | type Service struct { 14 | m *sync.RWMutex 15 | indices map[string]Index 16 | } 17 | 18 | func NewIndexService() *Service { 19 | return &Service{ 20 | m: &sync.RWMutex{}, 21 | indices: make(map[string]Index), 22 | } 23 | } 24 | 25 | func (s Service) ListLogs(w http.ResponseWriter, r *http.Request) { 26 | var indexName = mux.Vars(r)["index_name"] 27 | var err error 28 | var logs [][]byte 29 | 30 | s.m.RLock() 31 | // check if this is a valid index 32 | if _, exists := s.indices[indexName]; exists { 33 | logs = s.indices[indexName].Logs 34 | } 35 | s.m.RUnlock() 36 | 37 | if r.Header.Get("Accept") == "application/json" { 38 | var jsonLogs []map[string]interface{} 39 | 40 | for _, log := range logs { 41 | if log == nil { 42 | break 43 | } 44 | // Unmarshal the struct to json 45 | var raw map[string]interface{} 46 | if err = json.Unmarshal(log, &raw); err != nil { 47 | w.WriteHeader(http.StatusInternalServerError) 48 | } 49 | jsonLogs = append(jsonLogs, raw) 50 | } 51 | 52 | w.Header().Set("Content-Type", "application/json") 53 | _ = json.NewEncoder(w).Encode(jsonLogs) 54 | } else { 55 | // write logs to response 56 | for i := range logs { 57 | if _, err = w.Write(logs[i]); err != nil { 58 | w.WriteHeader(http.StatusInternalServerError) 59 | } 60 | 61 | if _, err = w.Write([]byte("\n")); err != nil { 62 | w.WriteHeader(http.StatusInternalServerError) 63 | } 64 | } 65 | } 66 | } 67 | 68 | func (s Service) WriteLog(w http.ResponseWriter, r *http.Request) { 69 | var index Index 70 | var indexName = mux.Vars(r)["index_name"] 71 | 72 | s.m.Lock() 73 | defer s.m.Unlock() 74 | // check if this index exists 75 | if _, exists := s.indices[indexName]; exists { 76 | index = s.indices[indexName] 77 | } else { 78 | index = NewIndex(indexName) 79 | } 80 | 81 | scanner := bufio.NewScanner(r.Body) 82 | for scanner.Scan() { 83 | WriteLog(&index, []byte(scanner.Text())) 84 | } 85 | 86 | s.indices[indexName] = index 87 | } 88 | 89 | func (s Service) ListIndices(w http.ResponseWriter, r *http.Request) { 90 | var indices []Index 91 | 92 | s.m.RLock() 93 | 94 | for _, index := range s.indices { 95 | indices = append(indices, index) 96 | } 97 | 98 | s.m.RUnlock() 99 | 100 | if r.Header.Get("Accept") == "application/json" { 101 | w.Header().Set("Content-Type", "application/json") 102 | _ = json.NewEncoder(w).Encode(indices) 103 | 104 | } else { 105 | _template, err := template.New("listPage").Funcs(template.FuncMap{"DateFormat": DateFormat}).Parse(listPageTemplate) 106 | _ = err 107 | _ = _template.Execute(w, indices) 108 | w.Header().Add("Content-Type", "text/html") 109 | } 110 | } 111 | --------------------------------------------------------------------------------