├── .github └── workflows │ ├── build-workflow.yaml │ └── release-workflow.yaml ├── .gitignore ├── .go-version ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── charts └── log-router │ ├── .gitignore │ ├── .helmignore │ ├── Chart.yaml │ ├── Makefile │ ├── templates │ ├── _helpers.tpl │ ├── clusterrole.yaml │ ├── daemonset.yaml │ ├── dashboard-cm.yaml │ ├── rolebinding.yaml │ ├── sa.yaml │ ├── secret.yaml │ ├── service.yaml │ └── servicemonitor.yaml │ └── values.yaml ├── config-reloader ├── .dockerignore ├── .gitignore ├── config │ ├── config.go │ └── config_test.go ├── controller │ ├── controller.go │ ├── controller_test.go │ └── updater.go ├── datasource │ ├── datasource.go │ ├── fake.go │ ├── fs.go │ ├── kube_informer.go │ ├── kube_informer_test.go │ └── kubedatasource │ │ ├── configmap.go │ │ ├── fluentdconfig.go │ │ ├── fluentdconfig │ │ ├── apis │ │ │ └── logs.vdp.vmware.com │ │ │ │ └── v1beta1 │ │ │ │ ├── doc.go │ │ │ │ ├── register.go │ │ │ │ ├── types.go │ │ │ │ └── zz_generated.deepcopy.go │ │ ├── client │ │ │ ├── clientset │ │ │ │ └── versioned │ │ │ │ │ ├── clientset.go │ │ │ │ │ ├── doc.go │ │ │ │ │ ├── fake │ │ │ │ │ ├── clientset_generated.go │ │ │ │ │ ├── doc.go │ │ │ │ │ └── register.go │ │ │ │ │ ├── scheme │ │ │ │ │ ├── doc.go │ │ │ │ │ └── register.go │ │ │ │ │ └── typed │ │ │ │ │ └── logs.vdp.vmware.com │ │ │ │ │ └── v1beta1 │ │ │ │ │ ├── doc.go │ │ │ │ │ ├── fake │ │ │ │ │ ├── doc.go │ │ │ │ │ ├── fake_fluentdconfig.go │ │ │ │ │ └── fake_logs.vdp.vmware.com_client.go │ │ │ │ │ ├── fluentdconfig.go │ │ │ │ │ ├── generated_expansion.go │ │ │ │ │ └── logs.vdp.vmware.com_client.go │ │ │ ├── informers │ │ │ │ └── externalversions │ │ │ │ │ ├── factory.go │ │ │ │ │ ├── generic.go │ │ │ │ │ ├── internalinterfaces │ │ │ │ │ └── factory_interfaces.go │ │ │ │ │ └── logs.vdp.vmware.com │ │ │ │ │ ├── interface.go │ │ │ │ │ └── v1beta1 │ │ │ │ │ ├── fluentdconfig.go │ │ │ │ │ └── interface.go │ │ │ └── listers │ │ │ │ └── logs.vdp.vmware.com │ │ │ │ └── v1beta1 │ │ │ │ ├── expansion_generated.go │ │ │ │ └── fluentdconfig.go │ │ └── crd │ │ │ └── crd.go │ │ ├── kubedatasource.go │ │ └── migrationmode.go ├── examples │ ├── demo.conf │ ├── kube-system.conf │ ├── manifests │ │ └── kfo-test.yaml │ └── my-favorite-namespace.conf ├── fluentd │ ├── fake-fluentd.sh │ ├── parser.go │ ├── parser_test.go │ ├── reloader.go │ ├── reloader_test.go │ ├── stack.go │ ├── validator.go │ └── validator_test.go ├── generator │ └── generator.go ├── go.mod ├── go.sum ├── godepgraph.png ├── local-fluent.conf ├── main.go ├── metrics │ └── metrics.go ├── processors │ ├── destinations.go │ ├── destinations_test.go │ ├── detect_exceptions.go │ ├── detect_exceptions_test.go │ ├── expand_tags.go │ ├── expand_tags_test.go │ ├── extract_plugins.go │ ├── extract_plugins_test.go │ ├── labels.go │ ├── labels_test.go │ ├── mounted_file.go │ ├── mounted_file_test.go │ ├── processor.go │ ├── processor_test.go │ ├── relabel.go │ ├── relabel_test.go │ ├── share.go │ ├── share_test.go │ ├── thisns.go │ ├── thisns_test.go │ ├── unique_rewrite_tag.go │ └── unique_rewrite_tag_test.go ├── template │ ├── template.go │ └── template_test.go ├── templates │ ├── fluent.conf │ ├── kubernetes-postprocess.conf │ ├── kubernetes.conf │ ├── prometheus.conf │ └── systemd.conf ├── util │ ├── util.go │ └── util_test.go ├── validate-from-dir.sh └── validate-logging-config.sh ├── image ├── Gemfile ├── Gemfile.lock ├── entrypoint.sh ├── failsafe.conf ├── fluentd_test.go ├── go.mod ├── go.sum ├── plugins │ ├── filter_dedot.rb │ ├── filter_extract.rb │ ├── in_just_exit.rb │ ├── out_truncating_remote_syslog.rb │ ├── parser_kubernetes.rb │ ├── parser_logfmt.rb │ └── parser_multiline_kubernetes.rb └── test │ ├── ci.conf │ ├── containers │ └── my-pod_my-namespace_my-container-df14e0d5ae4c07284fa636d739c8fc2e6b52bc344658de7d3f08c36a2e804115.log │ ├── extract.log │ ├── input.conf │ ├── json.log │ ├── kube-apiserver.log │ ├── kube-controller.log │ ├── local.conf │ ├── logfmt.log │ ├── results │ ├── extract.out │ ├── json.out │ ├── kube.var.log.containers.my-pod_my-namespace_my-container-df14e0d5ae4c07284fa636d739c8fc2e6b52bc344658de7d3f08c36a2e804115.log.out │ ├── kubernetes.out │ └── logfmt.out │ └── truncator.log ├── kubebuilder └── README.md └── kubectl ├── README.md ├── crd.yaml └── manifests.yaml /.github/workflows/build-workflow.yaml: -------------------------------------------------------------------------------- 1 | name: build-workflow 2 | # This workflow is triggered on pushes to the repository. 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | jobs: 11 | build: 12 | name: build-job 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Set up QEMU 17 | uses: docker/setup-qemu-action@v2 18 | - name: Set up Docker Buildx 19 | uses: docker/setup-buildx-action@v2 20 | - uses: engineerd/setup-kind@v0.5.0 21 | with: 22 | version: "v0.9.0" 23 | - uses: FranzDiebold/github-env-vars-action@v1.2.1 24 | - name: buildx-image 25 | run: | 26 | export TESTCONTAINERS_RYUK_DISABLED=true 27 | make build-test-ci 28 | make buildx-image TAG=v$GITHUB_SHA_SHORT 29 | - name: load-image 30 | run: | 31 | kind load docker-image vmware/kube-fluentd-operator:v$GITHUB_SHA_SHORT 32 | - name: install-log-router 33 | run: | 34 | cd charts 35 | PATH=$PATH:/tmp/bin helm upgrade --install --set image.tag=v$GITHUB_SHA_SHORT --set rbac.create=true --wait --timeout 60s log-router ./log-router 36 | -------------------------------------------------------------------------------- /.github/workflows/release-workflow.yaml: -------------------------------------------------------------------------------- 1 | name: release-workflow 2 | # This workflow is triggered on pushes to the repository. 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | publish: 10 | name: publish-job 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Set up QEMU 15 | uses: docker/setup-qemu-action@v2 16 | - name: Set up Docker Buildx 17 | uses: docker/setup-buildx-action@v2 18 | - name: Login to DockerHub 19 | uses: docker/login-action@v1 20 | with: 21 | username: ${{ secrets.DOCKER_USERNAME }} 22 | password: ${{ secrets.DOCKER_PASSWORD }} 23 | - name: push-image 24 | run: | 25 | make pushx-image 26 | env: 27 | TAG: ${{github.ref_name}} 28 | 29 | release: 30 | name: create-release 31 | runs-on: ubuntu-latest 32 | needs: publish 33 | container: 34 | image: alpine/helm 35 | steps: 36 | - uses: actions/checkout@v2 37 | - name: build-chart 38 | working-directory: charts/log-router 39 | run: helm package . 40 | - name: create-release 41 | uses: softprops/action-gh-release@v1 42 | if: startsWith(github.ref, 'refs/tags') 43 | with: 44 | files: charts/log-router/log-router-0.4.0.tgz 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .history/ 3 | -------------------------------------------------------------------------------- /.go-version: -------------------------------------------------------------------------------- 1 | 1.24.1 2 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to kube-fluentd-operator (KFO) 2 | 3 | The kube-fluentd-operator (KFO) project team welcomes contributions from the community. If you wish to contribute code and you have not 4 | signed our contributor license agreement (CLA), our bot will update the issue when you open a Pull Request. For any 5 | questions about the CLA process, please refer to our [FAQ](https://cla.vmware.com/faq). 6 | 7 | ## Community 8 | 9 | ## Getting Started 10 | 11 | ## Contribution Flow 12 | 13 | This is a rough outline of what a contributor's workflow looks like: 14 | 15 | - Create a topic branch from where you want to base your work 16 | - Make commits of logical units 17 | - Make sure your commit messages are in the proper format (see below) 18 | - Push your changes to a topic branch in your fork of the repository 19 | - Submit a pull request 20 | 21 | Example: 22 | 23 | ``` shell 24 | git remote add upstream https://github.com/vmware/kube-fluentd-operator.git 25 | git checkout -b my-new-feature master 26 | git commit -a 27 | git push origin my-new-feature 28 | ``` 29 | 30 | ### Staying In Sync With Upstream 31 | 32 | When your branch gets out of sync with the vmware/master branch, use the following to update: 33 | 34 | ``` shell 35 | git checkout my-new-feature 36 | git fetch -a 37 | git pull --rebase upstream master 38 | git push --force-with-lease origin my-new-feature 39 | ``` 40 | 41 | ### Updating pull requests 42 | 43 | If your PR fails to pass CI or needs changes based on code review, you'll most likely want to squash these changes into 44 | existing commits. 45 | 46 | If your pull request contains a single commit or your changes are related to the most recent commit, you can simply 47 | amend the commit. 48 | 49 | ``` shell 50 | git add . 51 | git commit --amend 52 | git push --force-with-lease origin my-new-feature 53 | ``` 54 | 55 | If you need to squash changes into an earlier commit, you can use: 56 | 57 | ``` shell 58 | git add . 59 | git commit --fixup 60 | git rebase -i --autosquash master 61 | git push --force-with-lease origin my-new-feature 62 | ``` 63 | 64 | Be sure to add a comment to the PR indicating your new changes are ready to review, as GitHub does not generate a 65 | notification when you git push. 66 | 67 | ### Code Style 68 | 69 | ### Formatting Commit Messages 70 | 71 | We follow the conventions on [How to Write a Git Commit Message](http://chris.beams.io/posts/git-commit/). 72 | 73 | Be sure to include any related GitHub issue references in the commit message. See 74 | [GFM syntax](https://guides.github.com/features/mastering-markdown/#GitHub-flavored-markdown) for referencing issues 75 | and commits. 76 | 77 | ## Reporting Bugs and Creating Issues 78 | 79 | When opening a new issue, try to roughly follow the commit message format conventions above. 80 | 81 | ## Repository Structure 82 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright © 2023 VMware, Inc. All Rights Reserved. 2 | # SPDX-License-Identifier: BSD-2-Clause 3 | # Similar to https://github.com/drecom/docker-centos-ruby/blob/2.6.5-slim/Dockerfile 4 | 5 | 6 | FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.24 AS builder 7 | 8 | ARG TARGETPLATFORM 9 | ARG BUILDPLATFORM 10 | ARG TARGETOS 11 | ARG TARGETARCH 12 | 13 | WORKDIR /go/src/github.com/vmware/kube-fluentd-operator/config-reloader 14 | COPY config-reloader . 15 | COPY Makefile . 16 | 17 | # Speed up local builds where vendor is populated 18 | ARG VERSION 19 | RUN make build VERSION=$VERSION TARGETARCH=$TARGETARCH TARGETOS=$TARGETOS 20 | 21 | FROM --platform=${TARGETPLATFORM:-linux/amd64} photon:4.0 22 | 23 | ARG RVM_PATH=/usr/local/rvm 24 | ARG RUBY_VERSION=ruby-3.1.4 25 | ARG RUBY_PATH=/usr/local/rvm/rubies/$RUBY_VERSION 26 | ARG RUBYOPT='-W:no-deprecated -W:no-experimental' 27 | ARG TARGETPLATFORM 28 | ARG BUILDPLATFORM 29 | ARG TARGETOS 30 | ARG TARGETARCH 31 | 32 | ENV PATH=$RUBY_PATH/bin:$PATH 33 | ENV FLUENTD_DISABLE_BUNDLER_INJECTION=1 34 | ENV BUILDDEPS="\ 35 | gmp-devel \ 36 | libffi-devel \ 37 | bzip2 \ 38 | shadow \ 39 | which \ 40 | wget \ 41 | vim \ 42 | git \ 43 | less \ 44 | tar \ 45 | gzip \ 46 | sed \ 47 | gcc \ 48 | build-essential \ 49 | zlib-devel \ 50 | libedit \ 51 | libedit-devel \ 52 | gdbm \ 53 | gdbm-devel \ 54 | openssl-devel \ 55 | gpg" 56 | 57 | RUN tdnf clean all && \ 58 | tdnf upgrade -y && \ 59 | tdnf erase -y toybox && \ 60 | tdnf install -y \ 61 | findutils \ 62 | procps-ng \ 63 | util-linux \ 64 | systemd \ 65 | net-tools && \ 66 | tdnf clean all 67 | 68 | SHELL [ "/bin/bash", "-l", "-c" ] 69 | 70 | COPY image/failsafe.conf image/entrypoint.sh image/Gemfile image/Gemfile.lock /fluentd/ 71 | 72 | # Install the gems with bundler is better practice 73 | # We need to keep this as a single layer because of the builddeps 74 | # if we split between multiple steps, we need up with the lots of extra files between layers 75 | RUN tdnf install -y $BUILDDEPS \ 76 | && curl -sSL https://rvm.io/mpapis.asc | gpg --import \ 77 | && curl -sSL https://rvm.io/pkuczynski.asc | gpg --import \ 78 | && curl -sSL https://get.rvm.io | bash -s stable \ 79 | && source /etc/profile.d/rvm.sh \ 80 | && rvm autolibs disable \ 81 | && rvm requirements \ 82 | && rvm install --disable-binary $RUBY_VERSION --default \ 83 | && gem update --system --no-document \ 84 | && gem install bundler -v '>= 2.4.15' --default --no-document \ 85 | && rm -rf $RVM_PATH/src $RVM_PATH/examples $RVM_PATH/docs $RVM_PATH/archives \ 86 | $RUBY_PATH/lib/ruby/gems/3.*/cache $RUBY_PATH/lib/ruby/gems/3.*/doc/ \ 87 | /usr/share/doc /root/.bundle/cache \ 88 | && rvm cleanup all \ 89 | && gem sources --clear-all \ 90 | && gem cleanup \ 91 | && tdnf remove -y $BUILDDEPS \ 92 | && tdnf clean all 93 | 94 | RUN tdnf install -y $BUILDDEPS \ 95 | && mkdir -p /fluentd/log /fluentd/etc /fluentd/plugins /usr/local/bundle/bin/ \ 96 | && echo 'gem: --no-document' >> /etc/gemrc \ 97 | && bundle config silence_root_warning true \ 98 | && cd /fluentd \ 99 | && bundle install \ 100 | && cd /fluentd \ 101 | && gem specific_install https://github.com/javiercri/fluent-plugin-google-cloud.git \ 102 | && cd /fluentd \ 103 | && gem sources --clear-all \ 104 | && ln -s $(which fluentd) /usr/local/bundle/bin/fluentd \ 105 | && gem cleanup \ 106 | ## Install jemalloc 107 | && curl -sLo /tmp/jemalloc-5.3.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 \ 108 | && tar -C /tmp/ -xjvf /tmp/jemalloc-5.3.0.tar.bz2 \ 109 | && cd /tmp/jemalloc-5.3.0 \ 110 | && ./configure && make \ 111 | && mv -v lib/libjemalloc.so* /usr/lib \ 112 | && rm -rf /tmp/* \ 113 | # cleanup build deps 114 | && tdnf remove -y $BUILDDEPS \ 115 | && tdnf clean all 116 | 117 | COPY image/plugins /fluentd/plugins 118 | 119 | COPY config-reloader/templates /templates 120 | COPY config-reloader/validate-from-dir.sh /bin/validate-from-dir.sh 121 | COPY --from=builder /go/src/github.com/vmware/kube-fluentd-operator/config-reloader/config-reloader /bin/config-reloader 122 | 123 | # Make sure fluentd picks jemalloc 5.3.0 lib as default 124 | ENV LD_PRELOAD="/usr/lib/libjemalloc.so" 125 | 126 | EXPOSE 24444 5140 127 | 128 | USER root 129 | 130 | ENTRYPOINT ["/fluentd/entrypoint.sh"] 131 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | kube-fluentd-operator 2 | 3 | Copyright 2023 VMware, Inc. All rights reserved 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | kube-fluentd-operator 2 | 3 | Copyright 2018 VMware, Inc. All Rights Reserved. 4 | 5 | This product is licensed to you under the BSD-2 license (the "License"). You may not use this product except in compliance with the BSD-2 License. 6 | 7 | This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 8 | 9 | 10 | -------------------------------------------------------------------------------- /charts/log-router/.gitignore: -------------------------------------------------------------------------------- 1 | *.tgz 2 | -------------------------------------------------------------------------------- /charts/log-router/.helmignore: -------------------------------------------------------------------------------- 1 | Makefile 2 | .gitignore 3 | *.tgz 4 | -------------------------------------------------------------------------------- /charts/log-router/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | # SPDX-License-Identifier: BSD-2-Clause 3 | 4 | apiVersion: v1 5 | description: Distribution of Fluentd as K8S daemonset 6 | name: log-router 7 | version: 0.4.1 8 | home: https://github.com/vmware/kube-fluentd-operator 9 | sources: 10 | - https://github.com/vmware/kube-fluentd-operator 11 | -------------------------------------------------------------------------------- /charts/log-router/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | # SPDX-License-Identifier: BSD-2-Clause 3 | 4 | HELM_PACKAGE := log-router 5 | 6 | .DEFAULT_GOAL := helm-package 7 | 8 | helm-package: clean 9 | helm lint . 10 | helm package . 11 | 12 | helm-dry-run: 13 | helm install . --dry-run --debug \ 14 | --set image.pullSecret=test-pull-secret \ 15 | --set namespaces[0]=kfo-test \ 16 | --set namespaces[1]=kfo-consumer \ 17 | --set rbac.create=true \ 18 | --set meta.key=key \ 19 | --set meta.values.env=test \ 20 | --set meta.values.region=eu \ 21 | --set tolerations[0].key=key \ 22 | --set tolerations[0].operator=op \ 23 | --set tolerations[0].value=value \ 24 | --set extraEnv.HELLO=world \ 25 | --set updateStrategy.type=OnDelete 26 | 27 | clean: 28 | rm -fr $(HELM_PACKAGE)-*.tgz || true 29 | -------------------------------------------------------------------------------- /charts/log-router/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "fluentd-router.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | */}} 13 | {{- define "fluentd-router.fullname" -}} 14 | {{- if .Values.fullnameOverride -}} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 16 | {{- else -}} 17 | {{- $name := default .Chart.Name .Values.nameOverride -}} 18 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 19 | {{- end -}} 20 | {{- end -}} 21 | 22 | {{/* 23 | Set apiVersion based on .Capabilities.APIVersions 24 | */}} 25 | {{- define "rbacAPIVersion" -}} 26 | {{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" -}} 27 | rbac.authorization.k8s.io/v1 28 | {{- else if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1beta1" -}} 29 | rbac.authorization.k8s.io/v1beta1 30 | {{- else if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1alpha1" -}} 31 | rbac.authorization.k8s.io/v1alpha1 32 | {{- else -}} 33 | rbac.authorization.k8s.io/v1 34 | {{- end -}} 35 | {{- end -}} 36 | {{- define "APIVersion" -}} 37 | {{- if .Capabilities.APIVersions.Has "apps/v1" -}} 38 | apps/v1 39 | {{- else if .Capabilities.APIVersions.Has "extensions/v1beta1" -}} 40 | extensions/v1beta1 41 | {{- else -}} 42 | apps/v1 43 | {{- end -}} 44 | {{- end -}} 45 | -------------------------------------------------------------------------------- /charts/log-router/templates/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | {{/* 2 | Copyright © 2018 VMware, Inc. All Rights Reserved. 3 | SPDX-License-Identifier: BSD-2-Clause 4 | */}} 5 | {{- if .Values.rbac.create }} 6 | {{- if (.Capabilities.APIVersions.Has (include "rbacAPIVersion" .)) -}} 7 | apiVersion: {{ template "rbacAPIVersion" . }} 8 | kind: ClusterRole 9 | metadata: 10 | labels: 11 | app: {{ template "fluentd-router.name" . }} 12 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 13 | heritage: {{ .Release.Service }} 14 | release: {{ .Release.Name }} 15 | {{- if .Values.extraLabels }} 16 | {{ toYaml .Values.extraLabels | indent 4 }} 17 | {{- end }} 18 | name: {{ template "fluentd-router.fullname" . }} 19 | rules: 20 | {{- with .Values.extraRBAC }} 21 | {{- toYaml . | nindent 2 }} 22 | {{- end }} 23 | - apiGroups: [""] 24 | resources: 25 | - configmaps 26 | - namespaces 27 | - pods 28 | verbs: 29 | - get 30 | - list 31 | - watch 32 | - apiGroups: [""] 33 | resources: 34 | - namespaces 35 | verbs: 36 | - patch 37 | - update 38 | {{- if or (eq .Values.datasource "crd") (eq .Values.crdMigrationMode true) }} 39 | - apiGroups: ["apiextensions.k8s.io"] 40 | resources: 41 | - customresourcedefinitions 42 | verbs: 43 | - create 44 | - get 45 | - watch 46 | - list 47 | - apiGroups: ["logs.vdp.vmware.com"] 48 | resources: 49 | - fluentdconfigs 50 | verbs: 51 | - get 52 | - list 53 | - watch 54 | {{- end }} 55 | {{- end }} 56 | {{- end }} 57 | -------------------------------------------------------------------------------- /charts/log-router/templates/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{/* 2 | Copyright © 2018 VMware, Inc. All Rights Reserved. 3 | SPDX-License-Identifier: BSD-2-Clause 4 | */}} 5 | {{- if .Values.rbac.create }} 6 | {{- if (.Capabilities.APIVersions.Has (include "rbacAPIVersion" .)) -}} 7 | apiVersion: {{ template "rbacAPIVersion" . }} 8 | kind: ClusterRoleBinding 9 | metadata: 10 | labels: 11 | app: {{ template "fluentd-router.name" . }} 12 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 13 | heritage: {{ .Release.Service }} 14 | release: {{ .Release.Name }} 15 | {{- if .Values.extraLabels }} 16 | {{ toYaml .Values.extraLabels | indent 4 }} 17 | {{- end }} 18 | name: {{ template "fluentd-router.fullname" . }} 19 | subjects: 20 | - kind: ServiceAccount 21 | name: {{ template "fluentd-router.fullname" . }} 22 | namespace: {{ .Release.Namespace }} 23 | roleRef: 24 | apiGroup: rbac.authorization.k8s.io 25 | kind: ClusterRole 26 | name: {{ template "fluentd-router.fullname" . }} 27 | {{- end }} 28 | {{- end }} 29 | -------------------------------------------------------------------------------- /charts/log-router/templates/sa.yaml: -------------------------------------------------------------------------------- 1 | {{/* 2 | Copyright © 2018 VMware, Inc. All Rights Reserved. 3 | SPDX-License-Identifier: BSD-2-Clause 4 | */}} 5 | {{- if .Values.rbac.create }} 6 | apiVersion: v1 7 | kind: ServiceAccount 8 | metadata: 9 | labels: 10 | app: {{ template "fluentd-router.name" . }} 11 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 12 | heritage: {{ .Release.Service }} 13 | release: {{ .Release.Name }} 14 | name: {{ template "fluentd-router.fullname" . }} 15 | {{- if .Values.rbac.serviceAccount.annotations }} 16 | annotations: {{ toYaml .Values.rbac.serviceAccount.annotations | nindent 4 }} 17 | {{- end }} 18 | {{ if .Values.imagePullSecret }} 19 | imagePullSecrets: 20 | - name: {{ .Values.imagePullSecret}} 21 | {{end}} 22 | {{- end }} -------------------------------------------------------------------------------- /charts/log-router/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | {{/* 2 | Copyright © 2018 VMware, Inc. All Rights Reserved. 3 | SPDX-License-Identifier: BSD-2-Clause 4 | */}} 5 | {{- if (or (.Values.fluentd.extraEnv) (.Values.reloader.extraEnv)) -}} 6 | apiVersion: v1 7 | kind: Secret 8 | metadata: 9 | name: {{ template "fluentd-router.fullname" . }} 10 | labels: 11 | app: {{ template "fluentd-router.name" . }} 12 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 13 | release: {{ .Release.Name }} 14 | heritage: {{ .Release.Service }} 15 | annotations: 16 | checksum/fluentd-extraenv: {{ toYaml .Values.fluentd.extraEnv | sha256sum }} 17 | checksum/reloader-extraenv: {{ toYaml .Values.reloader.extraEnv | sha256sum }} 18 | type: Opaque 19 | data: 20 | {{- range $key, $value := .Values.fluentd.extraEnv }} 21 | fluentd.{{ $key }}: {{ $value | b64enc | quote }} 22 | {{- end }} 23 | {{- range $key, $value := .Values.reloader.extraEnv }} 24 | reloader.{{ $key }}: {{ $value | b64enc | quote }} 25 | {{- end }} 26 | {{- end }} 27 | -------------------------------------------------------------------------------- /charts/log-router/templates/service.yaml: -------------------------------------------------------------------------------- 1 | {{/* 2 | Copyright © 2018 VMware, Inc. All Rights Reserved. 3 | SPDX-License-Identifier: BSD-2-Clause 4 | */}} 5 | {{- if .Values.prometheusEnabled }} 6 | apiVersion: v1 7 | kind: Service 8 | metadata: 9 | labels: 10 | app: {{ template "fluentd-router.name" . }} 11 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 12 | heritage: {{ .Release.Service }} 13 | release: {{ .Release.Name }} 14 | metrics: fluentd 15 | annotations: 16 | prometheus.io/scrape: "true" 17 | prometheus.io/port: "24231" 18 | name: {{ template "fluentd-router.fullname" . }} 19 | spec: 20 | selector: 21 | app: {{ template "fluentd-router.name" . }} 22 | release: {{ .Release.Name }} 23 | ports: 24 | - port: 24231 25 | name: prometheus 26 | targetPort: prometheus 27 | 28 | --- 29 | 30 | apiVersion: v1 31 | kind: Service 32 | metadata: 33 | labels: 34 | app: {{ template "fluentd-router.name" . }} 35 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 36 | heritage: {{ .Release.Service }} 37 | release: {{ .Release.Name }} 38 | metrics: reloader 39 | annotations: 40 | prometheus.io/scrape: "true" 41 | prometheus.io/port: {{ .Values.metricsPort | quote }} 42 | name: {{ template "fluentd-router.fullname" . }}-reloader 43 | spec: 44 | selector: 45 | app: {{ template "fluentd-router.name" . }} 46 | release: {{ .Release.Name }} 47 | ports: 48 | - port: {{ .Values.metricsPort }} 49 | name: metrics 50 | targetPort: metrics 51 | {{- end }} -------------------------------------------------------------------------------- /charts/log-router/templates/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.monitoring.serviceMonitor.enabled }} 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: ServiceMonitor 4 | metadata: 5 | name: {{ template "fluentd-router.fullname" . }} 6 | labels: 7 | app: {{ template "fluentd-router.name" . }} 8 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 9 | release: {{ .Release.Name }} 10 | heritage: {{ .Release.Service }} 11 | spec: 12 | jobLabel: app 13 | selector: 14 | matchLabels: 15 | app: {{ template "fluentd-router.name" . }} 16 | release: {{ .Release.Name }} 17 | metrics: fluentd 18 | endpoints: 19 | - port: prometheus 20 | {{- if .Values.monitoring.serviceMonitor.interval }} 21 | interval: {{ .Values.monitoring.serviceMonitor.interval }} 22 | {{- end }} 23 | 24 | --- 25 | 26 | apiVersion: monitoring.coreos.com/v1 27 | kind: ServiceMonitor 28 | metadata: 29 | name: {{ template "fluentd-router.fullname" . }}-reloader 30 | labels: 31 | app: {{ template "fluentd-router.name" . }} 32 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 33 | release: {{ .Release.Name }} 34 | heritage: {{ .Release.Service }} 35 | spec: 36 | jobLabel: app 37 | selector: 38 | matchLabels: 39 | app: {{ template "fluentd-router.name" . }} 40 | release: {{ .Release.Name }} 41 | metrics: reloader 42 | endpoints: 43 | - port: metrics 44 | {{- if .Values.monitoring.serviceMonitor.interval }} 45 | interval: {{ .Values.monitoring.serviceMonitor.interval }} 46 | {{- end }} 47 | {{- end }} 48 | -------------------------------------------------------------------------------- /charts/log-router/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | # SPDX-License-Identifier: BSD-2-Clause 3 | 4 | # Default values for fluentd-router. 5 | # This is a YAML-formatted file. 6 | # Declare variables to be passed into your templates. 7 | 8 | podLabels: [] 9 | 10 | rbac: 11 | # rbac.create -- If `true`, create and use RBAC resources. 12 | create: false 13 | serviceAccount: 14 | # rbac.serviceAccount.annotations -- Additional Service Account annotations. 15 | annotations: {} 16 | 17 | serviceAccountName: "default" 18 | 19 | extraRBAC: {} 20 | # - apiGroups: [""] 21 | # resources: ["secrets"] 22 | # verbs: ["get", "list", "watch"] 23 | 24 | # Possible values: default|fake|fs|multimap|crd 25 | datasource: default 26 | 27 | # Use with datasource: default or datasource: multimap, crdMigrationMode enables also the crd datasource 28 | # together with the specified legacy datasource to facilitate the migration process to CRDs. 29 | crdMigrationMode: false 30 | 31 | defaultConfigmap: "fluentd-config" 32 | 33 | # Use with datasource: fs, the fsDataSourceDir will be used for finding config files 34 | fsDatasourceDir: "" 35 | 36 | image: 37 | repository: vmware/kube-fluentd-operator 38 | pullPolicy: IfNotPresent 39 | tag: latest 40 | pullSecret: "" 41 | 42 | logLevel: debug 43 | fluentdLogLevel: debug 44 | interval: 45 45 | kubeletRoot: /var/lib/kubelet 46 | # bufferMountFolder -- a folder inside /var/log to write all fluentd buffers to 47 | bufferMountFolder: "" 48 | 49 | # fullnameOverride -- String to fully override `fluentd-router.fullname` template. 50 | fullnameOverride: "" 51 | 52 | meta: 53 | key: "" 54 | values: {} 55 | 56 | # Use with datasource: multimap, the label selector will be used for finding ConfigMaps inside 57 | # the Namespaces in order to compile the Namespace fluentd configuration. The simple concatenation 58 | # is used and the ConfigMap is then processed for macros. 59 | labelSelector: 60 | matchLabels: {} 61 | 62 | #extraVolumes: 63 | # - name: es-certs 64 | # secret: 65 | # defaultMode: 420 66 | # secretName: es-certs 67 | 68 | # - name: es-certs 69 | # mountPath: /certs 70 | # readOnly: true 71 | 72 | 73 | fluentd: 74 | extraEnv: {} 75 | resources: {} 76 | # extraVolumeMounts: 77 | # - name: es-certs 78 | # mountPath: /certs 79 | # readOnly: true 80 | 81 | reloader: 82 | extraEnv: {} 83 | resources: {} 84 | # extraVolumeMounts: 85 | # - name: es-certs 86 | # mountPath: /certs 87 | # readOnly: true 88 | 89 | tolerations: [] 90 | # We usually recommend not to specify default resources and to leave this as a conscious 91 | # choice for the user. This also increases chances charts run on environments with little 92 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 93 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 94 | # limits: 95 | # cpu: 100m 96 | # memory: 128Mi 97 | # requests: 98 | # cpu: 100m 99 | # memory: 128Mi 100 | 101 | priorityClassName: "" 102 | 103 | updateStrategy: {} 104 | # type: RollingUpdate 105 | 106 | ## Annotations to add to the DaemonSet's Pods 107 | #podAnnotations: 108 | # scheduler.alpha.kubernetes.io/tolerations: '[{"key": "example", "value": "foo"}]' 109 | 110 | prometheusEnabled: false 111 | metricsPort: 9000 112 | 113 | ### Disable privileged access and running service as root 114 | #securityContext: 115 | # runAsUser: 0 116 | # runAsGroup: 0 117 | # fsGroup: 0 118 | 119 | ### By default, Fluentd does not rotate log files 120 | logRotate: 121 | enabled: false 122 | # logRotateAge: 5 123 | # logRotateSize: 1048576000 124 | 125 | monitoring: 126 | serviceMonitor: 127 | # enable support for Prometheus Operator 128 | enabled: false 129 | 130 | # Job label for scrape target 131 | jobLabel: app 132 | 133 | # Scrape interval. If not set, the Prometheus default scrape interval is used. 134 | interval: "" 135 | # Create grafana dashboard with the chart 136 | grafanaDashboard: 137 | enabled: true 138 | 139 | allowTagExpansion: false 140 | 141 | # Change the following value to define a different namespace that is treated as admin 142 | # namespace, i.e. its configs are not validated or processed and virtual plugins can be 143 | # defined to be used in all other namespaces. 144 | adminNamespace: "kube-system" 145 | -------------------------------------------------------------------------------- /config-reloader/.dockerignore: -------------------------------------------------------------------------------- 1 | tmp/ 2 | config-reloader 3 | 4 | -------------------------------------------------------------------------------- /config-reloader/.gitignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | .idea/ 3 | *.iml 4 | tmp/ 5 | bin/ 6 | pkg/ 7 | config-reloader 8 | -------------------------------------------------------------------------------- /config-reloader/config/config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package config 5 | 6 | import ( 7 | "fmt" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestBadConfigs(t *testing.T) { 14 | inputs := [][]string{ 15 | {"--datasource", "fs"}, 16 | {"--id", "???"}, 17 | {"--log-level", "hobbit"}, 18 | {"--fluentd-loglevel", "hobbit"}, 19 | {"--buffer-mount-folder", "../"}, 20 | {"--buffer-mount-folder", " "}, 21 | {"--buffer-mount-folder", "\""}, 22 | {"--buffer-mount-folder", "\"\""}, 23 | {"--annotation", "|kl"}, 24 | {"--status-annotation", "/hello"}, 25 | {"--meta-key=test"}, 26 | {"--meta-key=test", "--meta-values="}, 27 | {"--meta-key=test", "--meta-values=."}, 28 | {"--meta-key=test", "--meta-values='"}, 29 | {"--meta-key=t''t", "--meta-values="}, 30 | {"--meta-key=test", "--meta-values=a"}, 31 | {"--meta-key=test", "--meta-values=a="}, 32 | {"--meta-key=test", "--meta-values=a=="}, 33 | } 34 | 35 | for _, args := range inputs { 36 | cfg := &Config{} 37 | err := cfg.ParseFlags(args) 38 | assert.Nil(t, err) 39 | 40 | err = cfg.Validate() 41 | assert.NotNil(t, err, "'%v' must fail validation", args) 42 | fmt.Printf("error %s\n", err) 43 | } 44 | } 45 | 46 | func TestNormalization(t *testing.T) { 47 | cfg := &Config{} 48 | err := cfg.ParseFlags([]string{"--interval=-1"}) 49 | assert.Nil(t, err) 50 | err = cfg.Validate() 51 | assert.Nil(t, err) 52 | 53 | assert.Equal(t, 60, cfg.IntervalSeconds) 54 | assert.Equal(t, "info", cfg.LogLevel) 55 | } 56 | -------------------------------------------------------------------------------- /config-reloader/controller/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package controller 5 | 6 | import ( 7 | "context" 8 | 9 | "github.com/vmware/kube-fluentd-operator/config-reloader/config" 10 | "github.com/vmware/kube-fluentd-operator/config-reloader/datasource" 11 | "github.com/vmware/kube-fluentd-operator/config-reloader/fluentd" 12 | "github.com/vmware/kube-fluentd-operator/config-reloader/generator" 13 | 14 | "github.com/sirupsen/logrus" 15 | ) 16 | 17 | type Controller interface { 18 | Run(ctx context.Context, stop <-chan struct{}) 19 | RunOnce(ctx context.Context) error 20 | GetTotalConfigNS() int 21 | } 22 | 23 | type controllerInstance struct { 24 | Updater Updater 25 | Reloader *fluentd.Reloader 26 | Datasource datasource.Datasource 27 | Generator generator.Generator 28 | outputDir string 29 | numTotalConfigNS int 30 | } 31 | 32 | var _ Controller = &controllerInstance{} 33 | 34 | // New creates new controller 35 | func New(ctx context.Context, cfg *config.Config, ds datasource.Datasource, up Updater) (Controller, error) { 36 | var reloader *fluentd.Reloader 37 | gen := generator.New(ctx, cfg) 38 | gen.SetStatusUpdater(ctx, ds) 39 | 40 | switch cfg.Datasource { 41 | case "fake", "fs": 42 | logrus.Infof("Setting reloader to null because is running locally") 43 | default: 44 | reloader = fluentd.NewReloader(ctx, cfg.FluentdRPCPort) 45 | } 46 | 47 | return &controllerInstance{ 48 | Updater: up, 49 | Reloader: reloader, 50 | Datasource: ds, 51 | Generator: gen, 52 | outputDir: cfg.OutputDir, 53 | }, nil 54 | } 55 | 56 | func (c *controllerInstance) RunOnce(ctx context.Context) error { 57 | logrus.Infof("Running main control loop") 58 | 59 | allConfigNamespaces, err := c.Datasource.GetNamespaces(ctx) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | c.Generator.SetModel(allConfigNamespaces) 65 | configHashes, err := c.Generator.RenderToDisk(ctx, c.outputDir) 66 | if err != nil { 67 | return nil 68 | } 69 | 70 | needsReload := false 71 | 72 | logrus.Infof("Config hashes returned in RunOnce loop: %v", configHashes) 73 | 74 | for _, nsConfig := range allConfigNamespaces { 75 | logrus.Debugf("Comparing hash with previous one for namespace: %v", nsConfig.Name) 76 | newHash, found := configHashes[nsConfig.Name] 77 | if !found { 78 | logrus.Infof("No config updates for namespace %s", nsConfig.Name) 79 | // error rendering config for the namespace, skip 80 | continue 81 | } 82 | 83 | if newHash != nsConfig.PreviousConfigHash { 84 | logrus.Infof("Detecting updates for namespace %s", nsConfig.Name) 85 | needsReload = true 86 | c.Datasource.WriteCurrentConfigHash(nsConfig.Name, newHash) 87 | } 88 | } 89 | 90 | // lastly, if number of configs has changed, then need to reload configurations obviously! 91 | // this means a crd was deleted or reapplied, and GetNamespaces does not return it anymore 92 | if c.numTotalConfigNS != len(allConfigNamespaces) { 93 | logrus.Infof("New namespaces found. Reloading fluentd...") 94 | needsReload = true 95 | c.numTotalConfigNS = len(allConfigNamespaces) 96 | } 97 | 98 | if needsReload { 99 | c.Reloader.ReloadConfiguration() 100 | } 101 | 102 | c.Generator.CleanupUnusedFiles(c.outputDir, configHashes) 103 | 104 | return nil 105 | } 106 | 107 | func (c *controllerInstance) Run(ctx context.Context, stop <-chan struct{}) { 108 | for { 109 | err := c.RunOnce(ctx) 110 | if err != nil { 111 | logrus.Error(err) 112 | } 113 | 114 | select { 115 | case <-c.Updater.GetUpdateChannel(): 116 | case <-stop: 117 | logrus.Info("Terminating main controller loop") 118 | return 119 | } 120 | } 121 | } 122 | 123 | func (c *controllerInstance) GetTotalConfigNS() int { 124 | return c.numTotalConfigNS 125 | } 126 | -------------------------------------------------------------------------------- /config-reloader/controller/controller_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "github.com/sirupsen/logrus" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/vmware/kube-fluentd-operator/config-reloader/config" 11 | "github.com/vmware/kube-fluentd-operator/config-reloader/datasource" 12 | ) 13 | 14 | // UnitTest for RunOnce 15 | func TestRunOnceController(t *testing.T) { 16 | // 1. Create new controller 17 | // 2. RunOnce controller 18 | // 3. Create a new namespace in the folder 19 | // 4. runOnce controller 20 | assert := assert.New(t) 21 | config := config.Config{ 22 | Datasource: "fs", 23 | FsDatasourceDir: "../examples", 24 | TemplatesDir: "../templates", 25 | ID: "default", 26 | OutputDir: "../tmp", 27 | LogLevel: "debug", 28 | } 29 | expectedResult := 3 30 | // Prepare TestCase 31 | ctx := context.Background() 32 | ds := datasource.NewFileSystemDatasource(ctx, config.FsDatasourceDir, config.OutputDir) 33 | 34 | // Create controller 35 | up := NewFixedTimeUpdater(ctx, config.IntervalSeconds) 36 | // 1. Create new controller 37 | ctrl, err := New(ctx, &config, ds, up) 38 | if err != nil { 39 | logrus.Fatalf("Unable to create new controller: %+v", err.Error()) 40 | } 41 | 42 | // 2. RunOnce controller 43 | err = ctrl.RunOnce(ctx) 44 | if err != nil { 45 | logrus.Fatalf("Unable to trigger RunOnce: %+v",err.Error()) 46 | } 47 | assert.Equal(expectedResult, ctrl.GetTotalConfigNS()) 48 | 49 | // 3. Create a new namespace in the folder 50 | newNamespaceFile := config.FsDatasourceDir + "/new-namespace.conf" 51 | configData := []byte("\n@type stdout\n") 52 | defer os.Remove(newNamespaceFile) 53 | err = os.WriteFile(newNamespaceFile, configData, 0644) 54 | if err != nil { 55 | logrus.Fatalf("Unable to create new Namespace: %+v", err.Error()) 56 | } 57 | 58 | // 4. RunOnce controller 59 | err = ctrl.RunOnce(ctx) 60 | if err != nil { 61 | logrus.Fatalf("Unable to trigger runOnce: %+v",err.Error()) 62 | } 63 | assert.Equal(expectedResult+1, ctrl.GetTotalConfigNS()) 64 | } 65 | -------------------------------------------------------------------------------- /config-reloader/controller/updater.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | // Updater provides a means to notify through a channel that an update is necessary 9 | type Updater interface { 10 | GetUpdateChannel() <-chan time.Time 11 | } 12 | 13 | // FixedTimeUpdater is an Updater that delivers a notification after a fixed amount of time 14 | type FixedTimeUpdater struct { 15 | interval time.Duration 16 | } 17 | 18 | var _ Updater = &FixedTimeUpdater{} 19 | 20 | func NewFixedTimeUpdater(ctx context.Context, seconds int) *FixedTimeUpdater { 21 | return &FixedTimeUpdater{interval: time.Duration(seconds) * time.Second} 22 | } 23 | 24 | func (f *FixedTimeUpdater) GetUpdateChannel() <-chan time.Time { 25 | return time.After(f.interval) 26 | } 27 | 28 | // OnDemandUpdater is an Updater that delivers notifications on demand through a shared channel 29 | type OnDemandUpdater struct { 30 | channel chan time.Time 31 | } 32 | 33 | var _ Updater = &OnDemandUpdater{} 34 | 35 | func NewOnDemandUpdater(ctx context.Context, channel chan time.Time) *OnDemandUpdater { 36 | return &OnDemandUpdater{channel: channel} 37 | } 38 | 39 | func (o *OnDemandUpdater) GetUpdateChannel() <-chan time.Time { 40 | return o.channel 41 | } 42 | -------------------------------------------------------------------------------- /config-reloader/datasource/datasource.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package datasource 5 | 6 | import ( 7 | "context" 8 | "sort" 9 | 10 | core "k8s.io/api/core/v1" 11 | ) 12 | 13 | type Mount struct { 14 | Path string 15 | VolumeName string 16 | SubPath string 17 | } 18 | 19 | // MiniContainer container subset with the parent pod's metadata 20 | type MiniContainer struct { 21 | // the pod id 22 | PodID string 23 | PodName string 24 | 25 | Image string 26 | ContainerID string 27 | 28 | // pod labels 29 | Labels map[string]string 30 | 31 | // container name 32 | Name string 33 | // only the emptyDir mounts, never empty, sorted by len(Path), descending 34 | HostMounts []*Mount 35 | 36 | NodeName string 37 | } 38 | 39 | // NamespaceConfig holds all relevant data for a namespace 40 | type NamespaceConfig struct { 41 | Name string 42 | FluentdConfig string 43 | PreviousConfigHash string 44 | MiniContainers []*MiniContainer 45 | Labels map[string]string 46 | } 47 | 48 | // StatusUpdater sets an error description on the namespace 49 | // in case configuration cannot be applied or an empty string otherwise 50 | type StatusUpdater interface { 51 | UpdateStatus(ctx context.Context, namespace string, status string) 52 | } 53 | 54 | // Datasource reads data from k8s 55 | type Datasource interface { 56 | StatusUpdater 57 | GetNamespaces(ctx context.Context) ([]*NamespaceConfig, error) 58 | WriteCurrentConfigHash(namespace string, hash string) 59 | } 60 | 61 | type byLength []*Mount 62 | 63 | func (s byLength) Len() int { 64 | return len(s) 65 | } 66 | 67 | func (s byLength) Swap(i, j int) { 68 | s[i], s[j] = s[j], s[i] 69 | } 70 | 71 | func (s byLength) Less(i, j int) bool { 72 | return len(s[i].Path) > len(s[j].Path) 73 | } 74 | 75 | func findContainerStatus(statuses []core.ContainerStatus, name string) *core.ContainerStatus { 76 | for _, st := range statuses { 77 | if st.Name == name { 78 | return &st 79 | } 80 | } 81 | return nil 82 | } 83 | 84 | func convertPodToMinis(resp *core.PodList) []*MiniContainer { 85 | var res []*MiniContainer 86 | 87 | for _, pod := range resp.Items { 88 | for _, cont := range pod.Spec.Containers { 89 | contStatus := findContainerStatus(pod.Status.ContainerStatuses, cont.Name) 90 | cid := "" 91 | if contStatus != nil { 92 | cid = contStatus.ContainerID 93 | } 94 | 95 | mini := &MiniContainer{ 96 | PodID: string(pod.UID), 97 | PodName: pod.Name, 98 | Labels: pod.Labels, 99 | Name: cont.Name, 100 | NodeName: pod.Spec.NodeName, 101 | Image: cont.Image, 102 | ContainerID: cid, 103 | } 104 | 105 | for i := range cont.VolumeMounts { 106 | m := makeVolume(pod.Spec.Volumes, &cont.VolumeMounts[i]) 107 | if m != nil { 108 | mini.HostMounts = append(mini.HostMounts, m) 109 | } 110 | } 111 | 112 | if len(mini.HostMounts) > 0 { 113 | sort.Sort(byLength(mini.HostMounts)) 114 | res = append(res, mini) 115 | } 116 | } 117 | } 118 | return res 119 | } 120 | 121 | func makeVolume(volumes []core.Volume, volumeMount *core.VolumeMount) *Mount { 122 | for _, v := range volumes { 123 | if v.Name == volumeMount.Name && v.EmptyDir != nil { 124 | return &Mount{ 125 | VolumeName: v.Name, 126 | Path: volumeMount.MountPath, 127 | SubPath: volumeMount.SubPath, 128 | } 129 | } 130 | } 131 | return nil 132 | } 133 | -------------------------------------------------------------------------------- /config-reloader/datasource/fake.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package datasource 5 | 6 | import ( 7 | "context" 8 | "strings" 9 | "time" 10 | 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | var logzTemplate = ` 15 | 16 | @type logzio_buffered 17 | endpoint_url https://listener.logz.io:8071?token=secret 18 | output_include_time true 19 | output_include_tags true 20 | buffer_type file 21 | buffer_path /var/log/logzio-$my_ns.buffer 22 | flush_interval 10s 23 | buffer_chunk_limit 1m 24 | 25 | ` 26 | 27 | type fakeDatasource struct { 28 | hashes map[string]string 29 | } 30 | 31 | // NewFakeDatasource returns a predefined set of namespaces + configs 32 | func NewFakeDatasource(ctx context.Context) Datasource { 33 | return &fakeDatasource{ 34 | hashes: make(map[string]string), 35 | } 36 | } 37 | 38 | func makeFakeConfig(namespace string) string { 39 | contents := logzTemplate 40 | contents = strings.ReplaceAll(contents, "$ns$", namespace) 41 | contents = strings.ReplaceAll(contents, "$ts$", time.Now().String()) 42 | 43 | return contents 44 | } 45 | 46 | func (d *fakeDatasource) GetNamespaces(ctx context.Context) ([]*NamespaceConfig, error) { 47 | res := []*NamespaceConfig{} 48 | 49 | for _, ns := range []string{"kube-system", "monitoring", "csp-main"} { 50 | res = append(res, &NamespaceConfig{ 51 | Name: ns, 52 | FluentdConfig: makeFakeConfig(ns), 53 | }) 54 | } 55 | 56 | // unconfigured namespace 57 | res = append(res, &NamespaceConfig{ 58 | Name: "not-configured", 59 | FluentdConfig: ` 60 | 61 | @type null 62 | 63 | `, 64 | }) 65 | return res, nil 66 | } 67 | 68 | func (d *fakeDatasource) WriteCurrentConfigHash(namespace string, hash string) { 69 | d.hashes[namespace] = hash 70 | } 71 | 72 | func (d *fakeDatasource) UpdateStatus(ctx context.Context, namespace string, status string) { 73 | logrus.Infof("Setting status of namespace %s to %s", namespace, status) 74 | } 75 | -------------------------------------------------------------------------------- /config-reloader/datasource/fs.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package datasource 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "os" 10 | "path/filepath" 11 | 12 | "github.com/vmware/kube-fluentd-operator/config-reloader/util" 13 | 14 | "github.com/sirupsen/logrus" 15 | ) 16 | 17 | type fsDatasource struct { 18 | hashes map[string]string 19 | rootDir string 20 | statusOutputDir string 21 | } 22 | 23 | // NewFileSystemDatasource turns all files matching *.conf patter in the given dir into namespace configs 24 | func NewFileSystemDatasource(ctx context.Context, rootDir string, statusOutputDir string) Datasource { 25 | return &fsDatasource{ 26 | hashes: make(map[string]string), 27 | rootDir: rootDir, 28 | statusOutputDir: statusOutputDir, 29 | } 30 | } 31 | 32 | func (d *fsDatasource) GetNamespaces(ctx context.Context) ([]*NamespaceConfig, error) { 33 | res := []*NamespaceConfig{} 34 | 35 | files, err := filepath.Glob(fmt.Sprintf("%s/*.conf", d.rootDir)) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | for _, f := range files { 41 | base := filepath.Base(f) 42 | ns := base[0 : len(base)-5] 43 | contents, err := os.ReadFile(f) 44 | if err != nil { 45 | logrus.Infof("Cannot read file %s: %+v", f, err) 46 | continue 47 | } 48 | 49 | cfg := &NamespaceConfig{ 50 | Name: ns, 51 | FluentdConfig: string(contents), 52 | PreviousConfigHash: d.hashes[ns], 53 | } 54 | 55 | logrus.Infof("Loading namespace %s from file %s", ns, f) 56 | res = append(res, cfg) 57 | } 58 | 59 | return res, nil 60 | } 61 | 62 | func (d *fsDatasource) WriteCurrentConfigHash(namespace string, hash string) { 63 | d.hashes[namespace] = hash 64 | } 65 | 66 | func (d *fsDatasource) UpdateStatus(ctx context.Context, namespace string, status string) { 67 | fname := filepath.Join(d.statusOutputDir, fmt.Sprintf("ns-%s.status", namespace)) 68 | if status != "" { 69 | util.WriteStringToFile(fname, status) 70 | } else { 71 | os.Remove(fname) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/apis/logs.vdp.vmware.com/v1beta1/doc.go: -------------------------------------------------------------------------------- 1 | // +k8s:deepcopy-gen=package 2 | // +k8s:defaulter-gen=TypeMeta 3 | // +groupName=logs.vdp.vmware.com 4 | 5 | package v1beta1 6 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/apis/logs.vdp.vmware.com/v1beta1/register.go: -------------------------------------------------------------------------------- 1 | package v1beta1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | "k8s.io/apimachinery/pkg/runtime" 6 | "k8s.io/apimachinery/pkg/runtime/schema" 7 | ) 8 | 9 | var SchemeGroupVersion = schema.GroupVersion{Group: "logs.vdp.vmware.com", Version: "v1beta1"} 10 | 11 | var ( 12 | SchemeBuilder runtime.SchemeBuilder 13 | localSchemeBuilder = &SchemeBuilder 14 | AddToScheme = localSchemeBuilder.AddToScheme 15 | ) 16 | 17 | func init() { 18 | // We only register manually written functions here. The registration of the 19 | // generated functions takes place in the generated files. The separation 20 | // makes the code compile even when the generated files are missing. 21 | localSchemeBuilder.Register(addKnownTypes) 22 | } 23 | 24 | // Resource takes an unqualified resource and returns a Group qualified GroupResource 25 | func Resource(resource string) schema.GroupResource { 26 | return SchemeGroupVersion.WithResource(resource).GroupResource() 27 | } 28 | 29 | // Adds the list of known types to the given scheme. 30 | func addKnownTypes(scheme *runtime.Scheme) error { 31 | scheme.AddKnownTypes(SchemeGroupVersion, 32 | &FluentdConfig{}, 33 | &FluentdConfigList{}, 34 | ) 35 | 36 | scheme.AddKnownTypes(SchemeGroupVersion, 37 | &metav1.Status{}, 38 | ) 39 | metav1.AddToGroupVersion(scheme, SchemeGroupVersion) 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/apis/logs.vdp.vmware.com/v1beta1/types.go: -------------------------------------------------------------------------------- 1 | package v1beta1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // +genclient 8 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 9 | 10 | // FluentdConfig defines the CRD 11 | type FluentdConfig struct { 12 | metav1.TypeMeta `json:",inline"` 13 | // +optional 14 | metav1.ObjectMeta `json:"metadata,omitempty"` 15 | 16 | Spec FluentdConfigSpec `json:"spec,omitempty"` 17 | } 18 | 19 | // FluentdConfigSpec implements the fluent.conf file as CRD 20 | type FluentdConfigSpec struct { 21 | FluentConf string `json:"fluentconf,omitempty"` 22 | } 23 | 24 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 25 | 26 | // FluentdConfigList is the mandatory plural type 27 | type FluentdConfigList struct { 28 | metav1.TypeMeta `json:",inline"` 29 | // +optional 30 | metav1.ListMeta `json:"metadata,omitempty"` 31 | 32 | Items []FluentdConfig `json:"items"` 33 | } 34 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/apis/logs.vdp.vmware.com/v1beta1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | Copyright The Kubernetes Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Code generated by deepcopy-gen. DO NOT EDIT. 20 | 21 | package v1beta1 22 | 23 | import ( 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | ) 26 | 27 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 28 | func (in *FluentdConfig) DeepCopyInto(out *FluentdConfig) { 29 | *out = *in 30 | out.TypeMeta = in.TypeMeta 31 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 32 | out.Spec = in.Spec 33 | return 34 | } 35 | 36 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FluentdConfig. 37 | func (in *FluentdConfig) DeepCopy() *FluentdConfig { 38 | if in == nil { 39 | return nil 40 | } 41 | out := new(FluentdConfig) 42 | in.DeepCopyInto(out) 43 | return out 44 | } 45 | 46 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 47 | func (in *FluentdConfig) DeepCopyObject() runtime.Object { 48 | if c := in.DeepCopy(); c != nil { 49 | return c 50 | } 51 | return nil 52 | } 53 | 54 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 55 | func (in *FluentdConfigList) DeepCopyInto(out *FluentdConfigList) { 56 | *out = *in 57 | out.TypeMeta = in.TypeMeta 58 | in.ListMeta.DeepCopyInto(&out.ListMeta) 59 | if in.Items != nil { 60 | in, out := &in.Items, &out.Items 61 | *out = make([]FluentdConfig, len(*in)) 62 | for i := range *in { 63 | (*in)[i].DeepCopyInto(&(*out)[i]) 64 | } 65 | } 66 | return 67 | } 68 | 69 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FluentdConfigList. 70 | func (in *FluentdConfigList) DeepCopy() *FluentdConfigList { 71 | if in == nil { 72 | return nil 73 | } 74 | out := new(FluentdConfigList) 75 | in.DeepCopyInto(out) 76 | return out 77 | } 78 | 79 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 80 | func (in *FluentdConfigList) DeepCopyObject() runtime.Object { 81 | if c := in.DeepCopy(); c != nil { 82 | return c 83 | } 84 | return nil 85 | } 86 | 87 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 88 | func (in *FluentdConfigSpec) DeepCopyInto(out *FluentdConfigSpec) { 89 | *out = *in 90 | return 91 | } 92 | 93 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FluentdConfigSpec. 94 | func (in *FluentdConfigSpec) DeepCopy() *FluentdConfigSpec { 95 | if in == nil { 96 | return nil 97 | } 98 | out := new(FluentdConfigSpec) 99 | in.DeepCopyInto(out) 100 | return out 101 | } 102 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/clientset.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | package versioned 19 | 20 | import ( 21 | "fmt" 22 | 23 | logsv1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/typed/logs.vdp.vmware.com/v1beta1" 24 | discovery "k8s.io/client-go/discovery" 25 | rest "k8s.io/client-go/rest" 26 | flowcontrol "k8s.io/client-go/util/flowcontrol" 27 | ) 28 | 29 | type Interface interface { 30 | Discovery() discovery.DiscoveryInterface 31 | LogsV1beta1() logsv1beta1.LogsV1beta1Interface 32 | } 33 | 34 | // Clientset contains the clients for groups. Each group has exactly one 35 | // version included in a Clientset. 36 | type Clientset struct { 37 | *discovery.DiscoveryClient 38 | logsV1beta1 *logsv1beta1.LogsV1beta1Client 39 | } 40 | 41 | // LogsV1beta1 retrieves the LogsV1beta1Client 42 | func (c *Clientset) LogsV1beta1() logsv1beta1.LogsV1beta1Interface { 43 | return c.logsV1beta1 44 | } 45 | 46 | // Discovery retrieves the DiscoveryClient 47 | func (c *Clientset) Discovery() discovery.DiscoveryInterface { 48 | if c == nil { 49 | return nil 50 | } 51 | return c.DiscoveryClient 52 | } 53 | 54 | // NewForConfig creates a new Clientset for the given config. 55 | // If config's RateLimiter is not set and QPS and Burst are acceptable, 56 | // NewForConfig will generate a rate-limiter in configShallowCopy. 57 | func NewForConfig(c *rest.Config) (*Clientset, error) { 58 | configShallowCopy := *c 59 | if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { 60 | if configShallowCopy.Burst <= 0 { 61 | return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") 62 | } 63 | configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) 64 | } 65 | var cs Clientset 66 | var err error 67 | cs.logsV1beta1, err = logsv1beta1.NewForConfig(&configShallowCopy) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy) 73 | if err != nil { 74 | return nil, err 75 | } 76 | return &cs, nil 77 | } 78 | 79 | // NewForConfigOrDie creates a new Clientset for the given config and 80 | // panics if there is an error in the config. 81 | func NewForConfigOrDie(c *rest.Config) *Clientset { 82 | var cs Clientset 83 | cs.logsV1beta1 = logsv1beta1.NewForConfigOrDie(c) 84 | 85 | cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) 86 | return &cs 87 | } 88 | 89 | // New creates a new Clientset for the given RESTClient. 90 | func New(c rest.Interface) *Clientset { 91 | var cs Clientset 92 | cs.logsV1beta1 = logsv1beta1.New(c) 93 | 94 | cs.DiscoveryClient = discovery.NewDiscoveryClient(c) 95 | return &cs 96 | } 97 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | // This package has the automatically generated clientset. 19 | package versioned 20 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/fake/clientset_generated.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | package fake 19 | 20 | import ( 21 | clientset "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned" 22 | logsv1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/typed/logs.vdp.vmware.com/v1beta1" 23 | fakelogsv1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/typed/logs.vdp.vmware.com/v1beta1/fake" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | "k8s.io/apimachinery/pkg/watch" 26 | "k8s.io/client-go/discovery" 27 | fakediscovery "k8s.io/client-go/discovery/fake" 28 | "k8s.io/client-go/testing" 29 | ) 30 | 31 | // NewSimpleClientset returns a clientset that will respond with the provided objects. 32 | // It's backed by a very simple object tracker that processes creates, updates and deletions as-is, 33 | // without applying any validations and/or defaults. It shouldn't be considered a replacement 34 | // for a real clientset and is mostly useful in simple unit tests. 35 | func NewSimpleClientset(objects ...runtime.Object) *Clientset { 36 | o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) 37 | for _, obj := range objects { 38 | if err := o.Add(obj); err != nil { 39 | panic(err) 40 | } 41 | } 42 | 43 | cs := &Clientset{tracker: o} 44 | cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} 45 | cs.AddReactor("*", "*", testing.ObjectReaction(o)) 46 | cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { 47 | gvr := action.GetResource() 48 | ns := action.GetNamespace() 49 | watch, err := o.Watch(gvr, ns) 50 | if err != nil { 51 | return false, nil, err 52 | } 53 | return true, watch, nil 54 | }) 55 | 56 | return cs 57 | } 58 | 59 | // Clientset implements clientset.Interface. Meant to be embedded into a 60 | // struct to get a default implementation. This makes faking out just the method 61 | // you want to test easier. 62 | type Clientset struct { 63 | testing.Fake 64 | discovery *fakediscovery.FakeDiscovery 65 | tracker testing.ObjectTracker 66 | } 67 | 68 | func (c *Clientset) Discovery() discovery.DiscoveryInterface { 69 | return c.discovery 70 | } 71 | 72 | func (c *Clientset) Tracker() testing.ObjectTracker { 73 | return c.tracker 74 | } 75 | 76 | var _ clientset.Interface = &Clientset{} 77 | 78 | // LogsV1beta1 retrieves the LogsV1beta1Client 79 | func (c *Clientset) LogsV1beta1() logsv1beta1.LogsV1beta1Interface { 80 | return &fakelogsv1beta1.FakeLogsV1beta1{Fake: &c.Fake} 81 | } 82 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | // This package has the automatically generated fake clientset. 19 | package fake 20 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/fake/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | package fake 19 | 20 | import ( 21 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | runtime "k8s.io/apimachinery/pkg/runtime" 23 | schema "k8s.io/apimachinery/pkg/runtime/schema" 24 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 25 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 26 | logsv1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/apis/logs.vdp.vmware.com/v1beta1" 27 | ) 28 | 29 | var scheme = runtime.NewScheme() 30 | var codecs = serializer.NewCodecFactory(scheme) 31 | 32 | var localSchemeBuilder = runtime.SchemeBuilder{ 33 | logsv1beta1.AddToScheme, 34 | } 35 | 36 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 37 | // of clientsets, like in: 38 | // 39 | // import ( 40 | // "k8s.io/client-go/kubernetes" 41 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 42 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 43 | // ) 44 | // 45 | // kclientset, _ := kubernetes.NewForConfig(c) 46 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 47 | // 48 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 49 | // correctly. 50 | var AddToScheme = localSchemeBuilder.AddToScheme 51 | 52 | func init() { 53 | v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) 54 | utilruntime.Must(AddToScheme(scheme)) 55 | } 56 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/scheme/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | // This package contains the scheme of the automatically generated clientset. 19 | package scheme 20 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/scheme/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | package scheme 19 | 20 | import ( 21 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | runtime "k8s.io/apimachinery/pkg/runtime" 23 | schema "k8s.io/apimachinery/pkg/runtime/schema" 24 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 25 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 26 | logsv1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/apis/logs.vdp.vmware.com/v1beta1" 27 | ) 28 | 29 | var Scheme = runtime.NewScheme() 30 | var Codecs = serializer.NewCodecFactory(Scheme) 31 | var ParameterCodec = runtime.NewParameterCodec(Scheme) 32 | var localSchemeBuilder = runtime.SchemeBuilder{ 33 | logsv1beta1.AddToScheme, 34 | } 35 | 36 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 37 | // of clientsets, like in: 38 | // 39 | // import ( 40 | // "k8s.io/client-go/kubernetes" 41 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 42 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 43 | // ) 44 | // 45 | // kclientset, _ := kubernetes.NewForConfig(c) 46 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 47 | // 48 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 49 | // correctly. 50 | var AddToScheme = localSchemeBuilder.AddToScheme 51 | 52 | func init() { 53 | v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) 54 | utilruntime.Must(AddToScheme(Scheme)) 55 | } 56 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/typed/logs.vdp.vmware.com/v1beta1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | // This package has the automatically generated typed clients. 19 | package v1beta1 20 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/typed/logs.vdp.vmware.com/v1beta1/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | // Package fake has the automatically generated clients. 19 | package fake 20 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/typed/logs.vdp.vmware.com/v1beta1/fake/fake_logs.vdp.vmware.com_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | package fake 19 | 20 | import ( 21 | rest "k8s.io/client-go/rest" 22 | testing "k8s.io/client-go/testing" 23 | v1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/typed/logs.vdp.vmware.com/v1beta1" 24 | ) 25 | 26 | type FakeLogsV1beta1 struct { 27 | *testing.Fake 28 | } 29 | 30 | func (c *FakeLogsV1beta1) FluentdConfigs(namespace string) v1beta1.FluentdConfigInterface { 31 | return &FakeFluentdConfigs{c, namespace} 32 | } 33 | 34 | // RESTClient returns a RESTClient that is used to communicate 35 | // with API server by this client implementation. 36 | func (c *FakeLogsV1beta1) RESTClient() rest.Interface { 37 | var ret *rest.RESTClient 38 | return ret 39 | } 40 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/typed/logs.vdp.vmware.com/v1beta1/generated_expansion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | package v1beta1 19 | 20 | type FluentdConfigExpansion interface{} 21 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/typed/logs.vdp.vmware.com/v1beta1/logs.vdp.vmware.com_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | package v1beta1 19 | 20 | import ( 21 | rest "k8s.io/client-go/rest" 22 | v1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/apis/logs.vdp.vmware.com/v1beta1" 23 | "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned/scheme" 24 | ) 25 | 26 | type LogsV1beta1Interface interface { 27 | RESTClient() rest.Interface 28 | FluentdConfigsGetter 29 | } 30 | 31 | // LogsV1beta1Client is used to interact with features provided by the logs.vdp.vmware.com group. 32 | type LogsV1beta1Client struct { 33 | restClient rest.Interface 34 | } 35 | 36 | func (c *LogsV1beta1Client) FluentdConfigs(namespace string) FluentdConfigInterface { 37 | return newFluentdConfigs(c, namespace) 38 | } 39 | 40 | // NewForConfig creates a new LogsV1beta1Client for the given config. 41 | func NewForConfig(c *rest.Config) (*LogsV1beta1Client, error) { 42 | config := *c 43 | if err := setConfigDefaults(&config); err != nil { 44 | return nil, err 45 | } 46 | client, err := rest.RESTClientFor(&config) 47 | if err != nil { 48 | return nil, err 49 | } 50 | return &LogsV1beta1Client{client}, nil 51 | } 52 | 53 | // NewForConfigOrDie creates a new LogsV1beta1Client for the given config and 54 | // panics if there is an error in the config. 55 | func NewForConfigOrDie(c *rest.Config) *LogsV1beta1Client { 56 | client, err := NewForConfig(c) 57 | if err != nil { 58 | panic(err) 59 | } 60 | return client 61 | } 62 | 63 | // New creates a new LogsV1beta1Client for the given RESTClient. 64 | func New(c rest.Interface) *LogsV1beta1Client { 65 | return &LogsV1beta1Client{c} 66 | } 67 | 68 | func setConfigDefaults(config *rest.Config) error { 69 | gv := v1beta1.SchemeGroupVersion 70 | config.GroupVersion = &gv 71 | config.APIPath = "/apis" 72 | config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() 73 | 74 | if config.UserAgent == "" { 75 | config.UserAgent = rest.DefaultKubernetesUserAgent() 76 | } 77 | 78 | return nil 79 | } 80 | 81 | // RESTClient returns a RESTClient that is used to communicate 82 | // with API server by this client implementation. 83 | func (c *LogsV1beta1Client) RESTClient() rest.Interface { 84 | if c == nil { 85 | return nil 86 | } 87 | return c.restClient 88 | } 89 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/informers/externalversions/generic.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by informer-gen. DO NOT EDIT. 17 | 18 | package externalversions 19 | 20 | import ( 21 | "fmt" 22 | 23 | schema "k8s.io/apimachinery/pkg/runtime/schema" 24 | cache "k8s.io/client-go/tools/cache" 25 | v1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/apis/logs.vdp.vmware.com/v1beta1" 26 | ) 27 | 28 | // GenericInformer is type of SharedIndexInformer which will locate and delegate to other 29 | // sharedInformers based on type 30 | type GenericInformer interface { 31 | Informer() cache.SharedIndexInformer 32 | Lister() cache.GenericLister 33 | } 34 | 35 | type genericInformer struct { 36 | informer cache.SharedIndexInformer 37 | resource schema.GroupResource 38 | } 39 | 40 | // Informer returns the SharedIndexInformer. 41 | func (f *genericInformer) Informer() cache.SharedIndexInformer { 42 | return f.informer 43 | } 44 | 45 | // Lister returns the GenericLister. 46 | func (f *genericInformer) Lister() cache.GenericLister { 47 | return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) 48 | } 49 | 50 | // ForResource gives generic access to a shared informer of the matching type 51 | // TODO extend this to unknown resources with a client pool 52 | func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { 53 | switch resource { 54 | // Group=logs.vdp.vmware.com, Version=v1beta1 55 | case v1beta1.SchemeGroupVersion.WithResource("fluentdconfigs"): 56 | return &genericInformer{resource: resource.GroupResource(), informer: f.Logs().V1beta1().FluentdConfigs().Informer()}, nil 57 | 58 | } 59 | 60 | return nil, fmt.Errorf("no informer found for %v", resource) 61 | } 62 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/informers/externalversions/internalinterfaces/factory_interfaces.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by informer-gen. DO NOT EDIT. 17 | 18 | package internalinterfaces 19 | 20 | import ( 21 | time "time" 22 | 23 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | cache "k8s.io/client-go/tools/cache" 26 | versioned "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned" 27 | ) 28 | 29 | // NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. 30 | type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer 31 | 32 | // SharedInformerFactory a small interface to allow for adding an informer without an import cycle 33 | type SharedInformerFactory interface { 34 | Start(stopCh <-chan struct{}) 35 | InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer 36 | } 37 | 38 | // TweakListOptionsFunc is a function that transforms a v1.ListOptions. 39 | type TweakListOptionsFunc func(*v1.ListOptions) 40 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/informers/externalversions/logs.vdp.vmware.com/interface.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by informer-gen. DO NOT EDIT. 17 | 18 | package logs 19 | 20 | import ( 21 | internalinterfaces "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/informers/externalversions/internalinterfaces" 22 | v1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/informers/externalversions/logs.vdp.vmware.com/v1beta1" 23 | ) 24 | 25 | // Interface provides access to each of this group's versions. 26 | type Interface interface { 27 | // V1beta1 provides access to shared informers for resources in V1beta1. 28 | V1beta1() v1beta1.Interface 29 | } 30 | 31 | type group struct { 32 | factory internalinterfaces.SharedInformerFactory 33 | namespace string 34 | tweakListOptions internalinterfaces.TweakListOptionsFunc 35 | } 36 | 37 | // New returns a new Interface. 38 | func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { 39 | return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} 40 | } 41 | 42 | // V1beta1 returns a new v1beta1.Interface. 43 | func (g *group) V1beta1() v1beta1.Interface { 44 | return v1beta1.New(g.factory, g.namespace, g.tweakListOptions) 45 | } 46 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/informers/externalversions/logs.vdp.vmware.com/v1beta1/fluentdconfig.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by informer-gen. DO NOT EDIT. 17 | 18 | package v1beta1 19 | 20 | import ( 21 | "context" 22 | time "time" 23 | 24 | logsv1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/apis/logs.vdp.vmware.com/v1beta1" 25 | versioned "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/clientset/versioned" 26 | internalinterfaces "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/informers/externalversions/internalinterfaces" 27 | v1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/listers/logs.vdp.vmware.com/v1beta1" 28 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 | runtime "k8s.io/apimachinery/pkg/runtime" 30 | watch "k8s.io/apimachinery/pkg/watch" 31 | cache "k8s.io/client-go/tools/cache" 32 | ) 33 | 34 | // FluentdConfigInformer provides access to a shared informer and lister for 35 | // FluentdConfigs. 36 | type FluentdConfigInformer interface { 37 | Informer() cache.SharedIndexInformer 38 | Lister() v1beta1.FluentdConfigLister 39 | } 40 | 41 | type fluentdConfigInformer struct { 42 | factory internalinterfaces.SharedInformerFactory 43 | tweakListOptions internalinterfaces.TweakListOptionsFunc 44 | namespace string 45 | } 46 | 47 | // NewFluentdConfigInformer constructs a new informer for FluentdConfig type. 48 | // Always prefer using an informer factory to get a shared informer instead of getting an independent 49 | // one. This reduces memory footprint and number of connections to the server. 50 | func NewFluentdConfigInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { 51 | return NewFilteredFluentdConfigInformer(client, namespace, resyncPeriod, indexers, nil) 52 | } 53 | 54 | // NewFilteredFluentdConfigInformer constructs a new informer for FluentdConfig type. 55 | // Always prefer using an informer factory to get a shared informer instead of getting an independent 56 | // one. This reduces memory footprint and number of connections to the server. 57 | func NewFilteredFluentdConfigInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { 58 | return cache.NewSharedIndexInformer( 59 | &cache.ListWatch{ 60 | ListFunc: func(options v1.ListOptions) (runtime.Object, error) { 61 | if tweakListOptions != nil { 62 | tweakListOptions(&options) 63 | } 64 | return client.LogsV1beta1().FluentdConfigs(namespace).List(context.TODO(), options) 65 | }, 66 | WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { 67 | if tweakListOptions != nil { 68 | tweakListOptions(&options) 69 | } 70 | return client.LogsV1beta1().FluentdConfigs(namespace).Watch(context.TODO(), options) 71 | }, 72 | }, 73 | &logsv1beta1.FluentdConfig{}, 74 | resyncPeriod, 75 | indexers, 76 | ) 77 | } 78 | 79 | func (f *fluentdConfigInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { 80 | return NewFilteredFluentdConfigInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) 81 | } 82 | 83 | func (f *fluentdConfigInformer) Informer() cache.SharedIndexInformer { 84 | return f.factory.InformerFor(&logsv1beta1.FluentdConfig{}, f.defaultInformer) 85 | } 86 | 87 | func (f *fluentdConfigInformer) Lister() v1beta1.FluentdConfigLister { 88 | return v1beta1.NewFluentdConfigLister(f.Informer().GetIndexer()) 89 | } 90 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/informers/externalversions/logs.vdp.vmware.com/v1beta1/interface.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by informer-gen. DO NOT EDIT. 17 | 18 | package v1beta1 19 | 20 | import ( 21 | internalinterfaces "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/informers/externalversions/internalinterfaces" 22 | ) 23 | 24 | // Interface provides access to all the informers in this group version. 25 | type Interface interface { 26 | // FluentdConfigs returns a FluentdConfigInformer. 27 | FluentdConfigs() FluentdConfigInformer 28 | } 29 | 30 | type version struct { 31 | factory internalinterfaces.SharedInformerFactory 32 | namespace string 33 | tweakListOptions internalinterfaces.TweakListOptionsFunc 34 | } 35 | 36 | // New returns a new Interface. 37 | func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { 38 | return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} 39 | } 40 | 41 | // FluentdConfigs returns a FluentdConfigInformer. 42 | func (v *version) FluentdConfigs() FluentdConfigInformer { 43 | return &fluentdConfigInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} 44 | } 45 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/listers/logs.vdp.vmware.com/v1beta1/expansion_generated.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by lister-gen. DO NOT EDIT. 17 | 18 | package v1beta1 19 | 20 | // FluentdConfigListerExpansion allows custom methods to be added to 21 | // FluentdConfigLister. 22 | type FluentdConfigListerExpansion interface{} 23 | 24 | // FluentdConfigNamespaceListerExpansion allows custom methods to be added to 25 | // FluentdConfigNamespaceLister. 26 | type FluentdConfigNamespaceListerExpansion interface{} 27 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/fluentdconfig/client/listers/logs.vdp.vmware.com/v1beta1/fluentdconfig.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 VMware VDP Team. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by lister-gen. DO NOT EDIT. 17 | 18 | package v1beta1 19 | 20 | import ( 21 | v1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/apis/logs.vdp.vmware.com/v1beta1" 22 | "k8s.io/apimachinery/pkg/api/errors" 23 | "k8s.io/apimachinery/pkg/labels" 24 | "k8s.io/client-go/tools/cache" 25 | ) 26 | 27 | // FluentdConfigLister helps list FluentdConfigs. 28 | // All objects returned here must be treated as read-only. 29 | type FluentdConfigLister interface { 30 | // List lists all FluentdConfigs in the indexer. 31 | // Objects returned here must be treated as read-only. 32 | List(selector labels.Selector) (ret []*v1beta1.FluentdConfig, err error) 33 | // FluentdConfigs returns an object that can list and get FluentdConfigs. 34 | FluentdConfigs(namespace string) FluentdConfigNamespaceLister 35 | FluentdConfigListerExpansion 36 | } 37 | 38 | // fluentdConfigLister implements the FluentdConfigLister interface. 39 | type fluentdConfigLister struct { 40 | indexer cache.Indexer 41 | } 42 | 43 | // NewFluentdConfigLister returns a new FluentdConfigLister. 44 | func NewFluentdConfigLister(indexer cache.Indexer) FluentdConfigLister { 45 | return &fluentdConfigLister{indexer: indexer} 46 | } 47 | 48 | // List lists all FluentdConfigs in the indexer. 49 | func (s *fluentdConfigLister) List(selector labels.Selector) (ret []*v1beta1.FluentdConfig, err error) { 50 | err = cache.ListAll(s.indexer, selector, func(m interface{}) { 51 | ret = append(ret, m.(*v1beta1.FluentdConfig)) 52 | }) 53 | return ret, err 54 | } 55 | 56 | // FluentdConfigs returns an object that can list and get FluentdConfigs. 57 | func (s *fluentdConfigLister) FluentdConfigs(namespace string) FluentdConfigNamespaceLister { 58 | return fluentdConfigNamespaceLister{indexer: s.indexer, namespace: namespace} 59 | } 60 | 61 | // FluentdConfigNamespaceLister helps list and get FluentdConfigs. 62 | // All objects returned here must be treated as read-only. 63 | type FluentdConfigNamespaceLister interface { 64 | // List lists all FluentdConfigs in the indexer for a given namespace. 65 | // Objects returned here must be treated as read-only. 66 | List(selector labels.Selector) (ret []*v1beta1.FluentdConfig, err error) 67 | // Get retrieves the FluentdConfig from the indexer for a given namespace and name. 68 | // Objects returned here must be treated as read-only. 69 | Get(name string) (*v1beta1.FluentdConfig, error) 70 | FluentdConfigNamespaceListerExpansion 71 | } 72 | 73 | // fluentdConfigNamespaceLister implements the FluentdConfigNamespaceLister 74 | // interface. 75 | type fluentdConfigNamespaceLister struct { 76 | indexer cache.Indexer 77 | namespace string 78 | } 79 | 80 | // List lists all FluentdConfigs in the indexer for a given namespace. 81 | func (s fluentdConfigNamespaceLister) List(selector labels.Selector) (ret []*v1beta1.FluentdConfig, err error) { 82 | err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { 83 | ret = append(ret, m.(*v1beta1.FluentdConfig)) 84 | }) 85 | return ret, err 86 | } 87 | 88 | // Get retrieves the FluentdConfig from the indexer for a given namespace and name. 89 | func (s fluentdConfigNamespaceLister) Get(name string) (*v1beta1.FluentdConfig, error) { 90 | obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) 91 | if err != nil { 92 | return nil, err 93 | } 94 | if !exists { 95 | return nil, errors.NewNotFound(v1beta1.Resource("fluentdconfig"), name) 96 | } 97 | return obj.(*v1beta1.FluentdConfig), nil 98 | } 99 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/kubedatasource.go: -------------------------------------------------------------------------------- 1 | package kubedatasource 2 | 3 | import ( 4 | "context" 5 | kfoListersV1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/listers/logs.vdp.vmware.com/v1beta1" 6 | ) 7 | 8 | // KubeDS is an interface defining behavor for the Kubernetes Resources 9 | // containing FluentD configurations 10 | type KubeDS interface { 11 | GetFluentdConfig(ctx context.Context, namespace string) (string, error) 12 | IsReady() bool 13 | GetFdlist() kfoListersV1beta1.FluentdConfigLister 14 | } 15 | -------------------------------------------------------------------------------- /config-reloader/datasource/kubedatasource/migrationmode.go: -------------------------------------------------------------------------------- 1 | package kubedatasource 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/vmware/kube-fluentd-operator/config-reloader/config" 8 | kfoListersV1beta1 "github.com/vmware/kube-fluentd-operator/config-reloader/datasource/kubedatasource/fluentdconfig/client/listers/logs.vdp.vmware.com/v1beta1" 9 | 10 | "k8s.io/client-go/informers" 11 | "k8s.io/client-go/rest" 12 | ) 13 | 14 | // MigrationModeDS is an abstraction providing a virtual KubeDS instance 15 | // that internally uses both the fluentdConfig KubeDS and the configMap KubeDS 16 | // This abstraction allows both KubeDS to be used together 17 | type MigrationModeDS struct { 18 | fdKubeDS KubeDS 19 | cmKubeDS KubeDS 20 | } 21 | 22 | func NewMigrationModeDS(ctx context.Context, cfg *config.Config, kubeCfg *rest.Config, factory informers.SharedInformerFactory, updateChan chan time.Time) (*MigrationModeDS, error) { 23 | fdKubeDS, err := NewFluentdConfigDS(ctx, cfg, kubeCfg, updateChan) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | cmKubeDS, err := NewConfigMapDS(ctx, cfg, factory, updateChan) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | return &MigrationModeDS{ 34 | fdKubeDS: fdKubeDS, 35 | cmKubeDS: cmKubeDS, 36 | }, nil 37 | } 38 | 39 | // IsReady returns a boolean specifying whether the MigrationModeDS is ready 40 | func (m *MigrationModeDS) IsReady() bool { 41 | return m.fdKubeDS.IsReady() && m.cmKubeDS.IsReady() 42 | } 43 | 44 | // GetFluentdConfig returns the fluentd configs for the given ns extracted 45 | // by the two KubeDS and concatenated together 46 | func (m *MigrationModeDS) GetFluentdConfig(ctx context.Context, namespace string) (string, error) { 47 | fdConfigs, err := m.fdKubeDS.GetFluentdConfig(ctx, namespace) 48 | if err != nil { 49 | return "", err 50 | } 51 | 52 | cmConfigs, err := m.cmKubeDS.GetFluentdConfig(ctx, namespace) 53 | if err != nil { 54 | return "", err 55 | } 56 | 57 | return cmConfigs + "\n" + fdConfigs, nil 58 | } 59 | 60 | // GetFdlist return nil for this mode because it does not use CRDs: 61 | func (m *MigrationModeDS) GetFdlist() kfoListersV1beta1.FluentdConfigLister { 62 | return m.fdKubeDS.GetFdlist() 63 | } 64 | -------------------------------------------------------------------------------- /config-reloader/examples/demo.conf: -------------------------------------------------------------------------------- 1 | # this file is invalid on purpose to show that an error status is persisted 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /config-reloader/examples/kube-system.conf: -------------------------------------------------------------------------------- 1 | # kube-system namespace is never processed nor validated so be careful 2 | 3 | 4 | @type logzio_buffered 5 | endpoint_url https://listener.logz.io:8071?token=TOKEN&type=log-router 6 | output_include_time true 7 | output_include_tags true 8 | buffer_type file 9 | buffer_path /var/log/kube-system-logz.buf 10 | flush_interval 10s 11 | buffer_chunk_limit 1m 12 | 13 | 14 | # ... or ignore all system-level logs 15 | # 16 | # ## Use your destination for the logs in the kube-system namespace as well as 17 | # # the systemd logs and the Kubernetes internal components 18 | # @type null 19 | # 20 | 21 | 22 | 23 | @type logzio_buffered 24 | endpoint_url https://listener.logz.io:8071?token=TOKEN&type=log-router 25 | buffer_path /some/path 26 | 27 | 28 | 29 | @type logzio_buffered 30 | endpoint_url https://listener.logz.io:8071?token=TOKEN&type=log-router 31 | buffer_path /some/path456 32 | buffer_size 1m 33 | 34 | -------------------------------------------------------------------------------- /config-reloader/examples/my-favorite-namespace.conf: -------------------------------------------------------------------------------- 1 | # ignore all logs from the containers called "fluentd" of the pods labeled with app=fluentd-router 2 | 3 | @type null 4 | 5 | 6 | 7 | # this is a virtual plugin defined in the kube-system namespace 8 | @type test456 9 | extra_param which is just passed to the body of test456 plugin... whatever it may be 10 | 11 | 12 | 13 | # this is a virtual plugin defined in the kube-system namespace 14 | @type test456 15 | buffer_size 7m 16 | 17 | 18 | # all the rest goes to logz. ** gets expanded to kube.{namespace}.** 19 | 20 | @type copy 21 | 22 | @type logzio_buffered 23 | endpoint_url https://listener.logz.io:8071?token=$LOGZ_TOKEN 24 | buffer_path /test 25 | 26 | 27 | @type share 28 | with_namespace my-other-namespace 29 | 30 | 31 | 32 | 33 | # $thisns expands to the current namespace. So, $thisns.** matches all logs for a namespace 34 | # Also, you can use ** which will be expanded to the full namespace syntax. 35 | # 36 | # @type null 37 | # 38 | 39 | # use logz.io destination 40 | # 41 | # @type logzio_buffered 42 | # endpoint_url https://listener.logz.io:8071?token=SECRET 43 | # output_include_time true 44 | # output_include_tags true 45 | # buffer_type file 46 | # buffer_path /var/log/mybuffer.pos 47 | # flush_interval 10s 48 | # buffer_chunk_limit 1m 49 | # 50 | 51 | # use papertrail 52 | # 53 | # @type papertrail 54 | # papertrail_host YOUR_HOST.papertrailapp.com 55 | # papertrail_port YOUR_PORT 56 | # flush_interval 30 57 | # 58 | -------------------------------------------------------------------------------- /config-reloader/fluentd/fake-fluentd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # this is used for local test as fluentd may not be installed 4 | # no validation is performed just the invocation parameters are printed 5 | 6 | if [[ $1 == "--version" ]]; then 7 | echo "fake-fluentd 1.0" 8 | exit 0 9 | fi 10 | 11 | file="${@: -1}" 12 | echo 13 | echo Invoked with "$@" 14 | echo __________________ 15 | 16 | cat "$file" 17 | 18 | echo __________________ 19 | 20 | # for unit tests: if the input contains #ERROR, exit with 1 21 | if grep 'ERROR' < "$file" ; then 22 | exit 1 23 | fi 24 | 25 | exit 0 26 | -------------------------------------------------------------------------------- /config-reloader/fluentd/reloader.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package fluentd 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "io" 10 | "net/http" 11 | 12 | "github.com/sirupsen/logrus" 13 | ) 14 | 15 | type RPCMethod string 16 | 17 | const ( 18 | gracefulReloadConf RPCMethod = "config.gracefulReload" 19 | reloadConf RPCMethod = "config.reload" 20 | ) 21 | 22 | // Reloader sends a reload signal to fluentd 23 | type Reloader struct { 24 | port int 25 | } 26 | 27 | // NewReloader will notify on the given rpc port 28 | func NewReloader(ctx context.Context, port int) *Reloader { 29 | return &Reloader{ 30 | port: port, 31 | } 32 | } 33 | 34 | // ReloadConfiguration talks to fluentd's RPC endpoint. If r is nil does nothing 35 | func (r *Reloader) ReloadConfiguration() { 36 | if r == nil { 37 | logrus.Infof("Not reloading fluentd (fake or filesystem datasource used)") 38 | return 39 | } 40 | 41 | logrus.Infof("Reloading fluentd configuration via /api/%s", gracefulReloadConf) 42 | if err := r.rpc(gracefulReloadConf); err != nil { 43 | logrus.Warnf("graceful reload failed: %+v", err) 44 | logrus.Infof("Reloading fluentd configuration via /api/%s", reloadConf) 45 | if err := r.rpc(reloadConf); err != nil { 46 | logrus.Error(err.Error()) 47 | } 48 | } 49 | } 50 | 51 | // rpc calls the given fluentd HTTP RPC endpoint 52 | // for more details see: https://docs.fluentd.org/deployment/rpc 53 | func (r *Reloader) rpc(method RPCMethod) error { 54 | resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/api/%s", r.port, method)) 55 | if err != nil { 56 | return fmt.Errorf("fluentd %s request failed: %w", method, err) 57 | } 58 | 59 | if resp.StatusCode != 200 { 60 | body, _ := io.ReadAll(resp.Body) 61 | resp.Body.Close() 62 | return fmt.Errorf("fluentd %s endpoint returned statuscode %v; response: %v", method, resp.StatusCode, string(body)) 63 | } 64 | 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /config-reloader/fluentd/reloader_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package fluentd 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "net/http" 10 | "testing" 11 | "time" 12 | 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestNullReloader(t *testing.T) { 17 | var r *Reloader 18 | r.ReloadConfiguration() 19 | } 20 | func TestReloaderCalls(t *testing.T) { 21 | ctx := context.Background() 22 | port := 24444 23 | 24 | counter := 0 25 | 26 | handler := func(w http.ResponseWriter, r *http.Request) { 27 | fmt.Printf("req %+v", r) 28 | if r.Method == "GET" && r.RequestURI == "/api/config.gracefulReload" { 29 | counter++ 30 | } 31 | } 32 | 33 | server := http.Server{ 34 | Addr: fmt.Sprintf(":%d", port), 35 | Handler: http.HandlerFunc(handler), 36 | } 37 | 38 | go server.ListenAndServe() 39 | defer server.Close() 40 | 41 | r := NewReloader(ctx, port) 42 | 43 | var err error 44 | for i := 10; i >= 0; i-- { 45 | _, err = http.Get(fmt.Sprintf("http://127.0.0.1:%d", port)) 46 | if err != nil { 47 | time.Sleep(200 * time.Millisecond) 48 | } 49 | } 50 | 51 | if err != nil { 52 | t.Fatalf("Mock server not up within reasonable time") 53 | } 54 | 55 | r.ReloadConfiguration() 56 | r.ReloadConfiguration() 57 | r.ReloadConfiguration() 58 | 59 | assert.Equal(t, 3, counter) 60 | } 61 | -------------------------------------------------------------------------------- /config-reloader/fluentd/stack.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package fluentd 5 | 6 | type Stack struct { 7 | top *node 8 | length int 9 | } 10 | 11 | type node struct { 12 | value interface{} 13 | prev *node 14 | } 15 | 16 | func NewStack() *Stack { 17 | return &Stack{nil, 0} 18 | } 19 | 20 | func (s *Stack) Len() int { 21 | return s.length 22 | } 23 | 24 | func (s *Stack) Peek() interface{} { 25 | if s.length == 0 { 26 | return nil 27 | } 28 | return s.top.value 29 | } 30 | 31 | func (s *Stack) Pop() interface{} { 32 | if s.length == 0 { 33 | return nil 34 | } 35 | 36 | top := s.top 37 | s.top = top.prev 38 | s.length-- 39 | return top.value 40 | } 41 | 42 | func (s *Stack) Push(value interface{}) { 43 | n := &node{ 44 | value: value, 45 | prev: s.top, 46 | } 47 | s.top = n 48 | s.length++ 49 | } 50 | -------------------------------------------------------------------------------- /config-reloader/fluentd/validator.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package fluentd 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "fmt" 10 | "os" 11 | "strings" 12 | "time" 13 | "unicode" 14 | 15 | "github.com/vmware/kube-fluentd-operator/config-reloader/util" 16 | 17 | "github.com/sirupsen/logrus" 18 | ) 19 | 20 | // Validator validates a generated config using fluentd's --dry-run command 21 | type Validator interface { 22 | ValidateConfig(config string, namespace string) error 23 | ValidateConfigExtremely(config string, namespace string) error 24 | EnsureUsable() error 25 | } 26 | 27 | type validatorState struct { 28 | command string 29 | args []string 30 | timeout time.Duration 31 | } 32 | 33 | var justExitPluginDirective = ` 34 | # extreme validation 35 | 36 | @type just_exit 37 | 38 | ` 39 | 40 | // NewValidator creates a Validator using the given command 41 | func NewValidator(ctx context.Context, command string, timeout time.Duration) Validator { 42 | parts := strings.Split(util.Trim(command), " ") 43 | 44 | return &validatorState{ 45 | command: parts[0], 46 | args: parts[1:], 47 | timeout: timeout, 48 | } 49 | } 50 | 51 | func (v *validatorState) ValidateConfigExtremely(config string, namespace string) error { 52 | if v == nil { 53 | return nil 54 | } 55 | 56 | tmpfile, err := os.CreateTemp("", "validate-ext-"+namespace) 57 | if err != nil { 58 | logrus.Errorf("error creating temporary file for namespace %s: %s", namespace, err.Error()) 59 | return err 60 | } 61 | defer os.Remove(tmpfile.Name()) 62 | 63 | config += justExitPluginDirective 64 | if _, err = tmpfile.WriteString(config); err != nil { 65 | logrus.Errorf("error writing config to temp file for namespace %s: %s", namespace, err.Error()) 66 | return err 67 | } 68 | 69 | if err := tmpfile.Close(); err != nil { 70 | logrus.Errorf("error closing temp file for namespace %s: %s", namespace, err.Error()) 71 | return err 72 | } 73 | 74 | args := make([]string, len(v.args)) 75 | copy(args, v.args) 76 | 77 | args = append(args, "-q", "--no-supervisor", "-c", tmpfile.Name()) 78 | 79 | out, err := util.ExecAndGetOutput(v.command, v.timeout, args...) 80 | 81 | // strip color stuff from fluentd output 82 | out = strings.TrimFunc(out, func(r rune) bool { 83 | return !unicode.IsPrint(r) 84 | }) 85 | 86 | logrus.Debugf("Checked config for namespace %s with fluentd and got: %s", namespace, out) 87 | if err != nil { 88 | logrus.Errorf("error running validation command for namespace %s: %s", namespace, err.Error()) 89 | return errors.New(out) 90 | } 91 | 92 | return nil 93 | } 94 | 95 | func (v *validatorState) ValidateConfig(config string, namespace string) error { 96 | if v == nil { 97 | return nil 98 | } 99 | 100 | tmpfile, err := os.CreateTemp("", "validate-"+namespace) 101 | if err != nil { 102 | logrus.Errorf("error creating temporary file for namespace %s: %s", namespace, err.Error()) 103 | return err 104 | } 105 | defer os.Remove(tmpfile.Name()) 106 | 107 | if _, err = tmpfile.WriteString(config); err != nil { 108 | logrus.Errorf("error writing config to temp file for namespace %s: %s", namespace, err.Error()) 109 | return err 110 | } 111 | 112 | if err := tmpfile.Close(); err != nil { 113 | logrus.Errorf("error closing temp file for namespace %s: %s", namespace, err.Error()) 114 | return err 115 | } 116 | 117 | args := make([]string, len(v.args)) 118 | copy(args, v.args) 119 | 120 | args = append(args, "--dry-run", "-c", tmpfile.Name()) 121 | 122 | out, err := util.ExecAndGetOutput(v.command, v.timeout, args...) 123 | 124 | // strip color stuf from fluentd output 125 | out = strings.TrimFunc(out, func(r rune) bool { 126 | return !unicode.IsPrint(r) 127 | }) 128 | 129 | logrus.Debugf("Checked config for namespace %s with fluentd and got: %s", namespace, out) 130 | if err != nil { 131 | logrus.Errorf("error running command in namespace %s: %s", namespace, err.Error()) 132 | return errors.New(out) 133 | } 134 | 135 | return nil 136 | } 137 | 138 | func (v *validatorState) EnsureUsable() error { 139 | if v == nil { 140 | return nil 141 | } 142 | out, err := util.ExecAndGetOutput(v.command, v.timeout, "--version") 143 | if err != nil { 144 | return fmt.Errorf("invalid fluentd binary used %s: %+v", v.command, err) 145 | } 146 | 147 | logrus.Infof("Validator using %s at version %s", v.command, util.Trim(out)) 148 | return nil 149 | } 150 | -------------------------------------------------------------------------------- /config-reloader/fluentd/validator_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package fluentd 5 | 6 | import ( 7 | "context" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | const validateCommand = "./fake-fluentd.sh -p plugins" 15 | 16 | func TestValidConfigString(t *testing.T) { 17 | ctx := context.Background() 18 | 19 | s := ` 20 | 21 | @type null 22 | 23 | ` 24 | 25 | validator := NewValidator(ctx, validateCommand, 30*time.Second) 26 | 27 | err := validator.EnsureUsable() 28 | assert.Nil(t, err, "Must succeed but failed with: %+v", err) 29 | 30 | err = validator.ValidateConfigExtremely(s, "namespace-1") 31 | assert.Nil(t, err, "Must succeed but failed with %+v", err) 32 | } 33 | 34 | func TestUnusable(t *testing.T) { 35 | ctx := context.Background() 36 | 37 | validator := NewValidator(ctx, "./no-such command", 30*time.Second) 38 | 39 | err := validator.EnsureUsable() 40 | assert.NotNil(t, err, "Must have failed") 41 | } 42 | 43 | func TestBadConfigString(t *testing.T) { 44 | ctx := context.Background() 45 | 46 | s := ` 47 | # ERROR <- this is a marker to cause failure 48 | 49 | @type null 50 | 51 | ` 52 | 53 | validator := NewValidator(ctx, validateCommand, 30*time.Second) 54 | 55 | err := validator.EnsureUsable() 56 | assert.Nil(t, err, "Must succeed but failed with: %+v", err) 57 | 58 | err = validator.ValidateConfigExtremely(s, "namespace-1") 59 | assert.NotNil(t, err) 60 | } 61 | -------------------------------------------------------------------------------- /config-reloader/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vmware/kube-fluentd-operator/config-reloader 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/Masterminds/sprig/v3 v3.2.3 7 | github.com/alecthomas/kingpin v2.2.6+incompatible 8 | github.com/prometheus/client_golang v1.15.1 9 | github.com/sirupsen/logrus v1.9.0 10 | github.com/stretchr/testify v1.8.2 11 | k8s.io/api v0.27.0 12 | k8s.io/apiextensions-apiserver v0.27.0 13 | k8s.io/apimachinery v0.27.0 14 | k8s.io/client-go v0.27.0 15 | sigs.k8s.io/controller-runtime v0.14.2 16 | sigs.k8s.io/yaml v1.3.0 17 | ) 18 | 19 | require ( 20 | github.com/Masterminds/goutils v1.1.1 // indirect 21 | github.com/Masterminds/semver/v3 v3.2.0 // indirect 22 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect 23 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect 24 | github.com/beorn7/perks v1.0.1 // indirect 25 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 26 | github.com/davecgh/go-spew v1.1.1 // indirect 27 | github.com/emicklei/go-restful/v3 v3.9.0 // indirect 28 | github.com/evanphx/json-patch v4.12.0+incompatible // indirect 29 | github.com/evanphx/json-patch/v5 v5.6.0 // indirect 30 | github.com/go-logr/logr v1.2.3 // indirect 31 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 32 | github.com/go-openapi/jsonreference v0.20.1 // indirect 33 | github.com/go-openapi/swag v0.22.3 // indirect 34 | github.com/gogo/protobuf v1.3.2 // indirect 35 | github.com/golang/protobuf v1.5.3 // indirect 36 | github.com/google/gnostic v0.5.7-v3refs // indirect 37 | github.com/google/go-cmp v0.5.9 // indirect 38 | github.com/google/gofuzz v1.1.0 // indirect 39 | github.com/google/uuid v1.3.0 // indirect 40 | github.com/huandu/xstrings v1.3.3 // indirect 41 | github.com/imdario/mergo v0.3.11 // indirect 42 | github.com/josharian/intern v1.0.0 // indirect 43 | github.com/json-iterator/go v1.1.12 // indirect 44 | github.com/mailru/easyjson v0.7.7 // indirect 45 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 46 | github.com/mitchellh/copystructure v1.0.0 // indirect 47 | github.com/mitchellh/reflectwalk v1.0.0 // indirect 48 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 49 | github.com/modern-go/reflect2 v1.0.2 // indirect 50 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 51 | github.com/pkg/errors v0.9.1 // indirect 52 | github.com/pmezard/go-difflib v1.0.0 // indirect 53 | github.com/prometheus/client_model v0.4.0 // indirect 54 | github.com/prometheus/common v0.44.0 // indirect 55 | github.com/prometheus/procfs v0.9.0 // indirect 56 | github.com/shopspring/decimal v1.2.0 // indirect 57 | github.com/spf13/cast v1.3.1 // indirect 58 | github.com/spf13/pflag v1.0.5 // indirect 59 | golang.org/x/crypto v0.31.0 // indirect 60 | golang.org/x/net v0.33.0 // indirect 61 | golang.org/x/oauth2 v0.8.0 // indirect 62 | golang.org/x/sys v0.28.0 // indirect 63 | golang.org/x/term v0.27.0 // indirect 64 | golang.org/x/text v0.21.0 // indirect 65 | golang.org/x/time v0.3.0 // indirect 66 | google.golang.org/appengine v1.6.7 // indirect 67 | google.golang.org/protobuf v1.33.0 // indirect 68 | gopkg.in/inf.v0 v0.9.1 // indirect 69 | gopkg.in/yaml.v2 v2.4.0 // indirect 70 | gopkg.in/yaml.v3 v3.0.1 // indirect 71 | k8s.io/klog/v2 v2.90.1 // indirect 72 | k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect 73 | k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect 74 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 75 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 76 | ) 77 | -------------------------------------------------------------------------------- /config-reloader/godepgraph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/kube-fluentd-operator/cf511bbe1ae44955d518849823372e07e8edbe25/config-reloader/godepgraph.png -------------------------------------------------------------------------------- /config-reloader/local-fluent.conf: -------------------------------------------------------------------------------- 1 | # Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | # SPDX-License-Identifier: BSD-2-Clause 3 | 4 | 5 | # use this with `make run-fluentd`. this way you can router logs to a local 6 | # server using a "@type forward" destination 7 | 8 | 9 | log_level debug 10 | 11 | 12 | 13 | @type http 14 | port 9880 15 | bind 0.0.0.0 16 | body_size_limit 32m 17 | keepalive_timeout 10s 18 | 19 | 20 | # simple test for the tail plugin 21 | 22 | @type tail 23 | path /workspace/Dockerfile 24 | pos_file /tmp/dockerfile.pos 25 | read_from_head true 26 | 27 | @type none 28 | 29 | tag files.dockerfile 30 | 31 | 32 | 33 | @type tail 34 | path /workspace/Makefile 35 | pos_file /tmp/makefile.pos 36 | read_from_head true 37 | 38 | @type none 39 | 40 | tag files.makefile 41 | 42 | 43 | 44 | @type copy 45 | 46 | @type relabel 47 | @label @hello 48 | 49 | 50 | @type relabel 51 | @label @hello 52 | 53 | 54 | 55 | 60 | 61 | 62 | 67 | 68 | 69 | 70 | @type stdout 71 | -------------------------------------------------------------------------------- /config-reloader/main.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | "time" 12 | 13 | "github.com/vmware/kube-fluentd-operator/config-reloader/config" 14 | "github.com/vmware/kube-fluentd-operator/config-reloader/controller" 15 | "github.com/vmware/kube-fluentd-operator/config-reloader/datasource" 16 | "github.com/vmware/kube-fluentd-operator/config-reloader/fluentd" 17 | "github.com/vmware/kube-fluentd-operator/config-reloader/metrics" 18 | 19 | "github.com/sirupsen/logrus" 20 | ) 21 | 22 | func main() { 23 | ctx := context.Background() 24 | cfg := &config.Config{} 25 | 26 | if err := cfg.ParseFlags(os.Args[1:]); err != nil { 27 | logrus.Fatalf("flag parsing error: %v", err) 28 | } 29 | logrus.Infof("Version: %s", config.Version) 30 | 31 | logrus.Infof("Config: %+v", cfg) 32 | 33 | if err := cfg.Validate(); err != nil { 34 | logrus.Fatalf("Config validation failed: %+v", err) 35 | } 36 | 37 | if cfg.FluentdValidateCommand != "" { 38 | validator := fluentd.NewValidator(ctx, cfg.FluentdValidateCommand, time.Second*time.Duration(cfg.ExecTimeoutSeconds)) 39 | if err := validator.EnsureUsable(); err != nil { 40 | logrus.Fatalf("Bad validate command used: '%s', either use correct one or none at all: %+v", 41 | cfg.FluentdValidateCommand, err) 42 | } 43 | } 44 | 45 | logrus.SetLevel(cfg.GetLogLevel()) 46 | 47 | // Create datasource and updater base in config 48 | var ds datasource.Datasource 49 | var up controller.Updater 50 | 51 | var err error 52 | 53 | switch cfg.Datasource { 54 | case "fake": 55 | ds = datasource.NewFakeDatasource(ctx) 56 | up = controller.NewFixedTimeUpdater(ctx, cfg.IntervalSeconds) 57 | case "fs": 58 | ds = datasource.NewFileSystemDatasource(ctx, cfg.FsDatasourceDir, cfg.OutputDir) 59 | up = controller.NewFixedTimeUpdater(ctx, cfg.IntervalSeconds) 60 | default: 61 | updateChan := make(chan time.Time, 1) 62 | ds, err = datasource.NewKubernetesInformerDatasource(ctx, cfg, updateChan) 63 | if err != nil { 64 | logrus.Fatalf("Cannot start informer %+v", err) 65 | } 66 | up = controller.NewOnDemandUpdater(ctx, updateChan) 67 | } 68 | 69 | ctrl, err := controller.New(ctx, cfg, ds, up) 70 | if err != nil { 71 | logrus.Fatalf("Cannot start control loop %+v", err) 72 | } 73 | 74 | // Add this for a timeout between 0-120 seconds (default: 30 (ExecTimeoutSeconds)) 75 | // This is for golang/fluentd race condition when KFO starts/restarts: 76 | if cfg.ExecTimeoutSeconds > 0 && cfg.ExecTimeoutSeconds <= 120 { 77 | logrus.Infof("Sleeping for %v seconds in order for fluentd to be ready.", cfg.ExecTimeoutSeconds) 78 | time.Sleep(time.Second * time.Duration(cfg.ExecTimeoutSeconds)) 79 | } 80 | 81 | if cfg.IntervalSeconds == 0 { 82 | ctrl.RunOnce(ctx) 83 | return 84 | } 85 | 86 | stopChan := make(chan struct{}, 1) 87 | go handleSigterm(stopChan) 88 | 89 | if cfg.PrometheusEnabled { 90 | metrics.InitMetrics(cfg.MetricsPort) 91 | } 92 | 93 | ctrl.Run(ctx, stopChan) 94 | } 95 | 96 | func handleSigterm(stopChan chan struct{}) { 97 | signals := make(chan os.Signal, 1) 98 | signal.Notify(signals, os.Interrupt, syscall.SIGTERM) 99 | sig := <-signals 100 | logrus.Infof("Received %v. Terminating...", sig) 101 | close(stopChan) 102 | } 103 | -------------------------------------------------------------------------------- /config-reloader/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/http" 7 | "strconv" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/prometheus/client_golang/prometheus/promhttp" 11 | ) 12 | 13 | const ( 14 | LabelTargetNamespace = "target_namespace" 15 | ) 16 | 17 | var namespaceConfigStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 18 | Namespace: "kube_fluentd_operator", 19 | Name: "namespace_config_status", 20 | Help: "Current validation status of fluentd configs in the namespace. Values are 0 (validation error) or 1 (validation successful)", 21 | }, []string{LabelTargetNamespace}) 22 | 23 | // SetNamespaceConfigStatusMetric sets the current metric value for a given namespace 24 | func SetNamespaceConfigStatusMetric(namespace string, valid bool) { 25 | var value float64 26 | if valid { 27 | value = 1 28 | } 29 | 30 | namespaceConfigStatus.With(prometheus.Labels{LabelTargetNamespace: namespace}).Set(value) 31 | } 32 | 33 | // DeleteNamespaceConfigStatusMetric deletes the metric value for a given namespace 34 | func DeleteNamespaceConfigStatusMetric(namespace string) { 35 | namespaceConfigStatus.Delete(prometheus.Labels{LabelTargetNamespace: namespace}) 36 | } 37 | 38 | // InitMetrics should be called to initialize metrics and start the HTTP handler 39 | func InitMetrics(port int) error { 40 | if err := serveMetrics(port); err != nil { 41 | return fmt.Errorf("Failed to start metrics handler: %s", err) 42 | } 43 | 44 | registerMetrics() 45 | return nil 46 | } 47 | 48 | func registerMetrics() { 49 | prometheus.MustRegister(namespaceConfigStatus) 50 | } 51 | 52 | func serveMetrics(port int) error { 53 | ln, err := net.Listen("tcp", ":"+strconv.Itoa(port)) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | mux := http.NewServeMux() 59 | mux.Handle("/metrics", promhttp.Handler()) 60 | srv := &http.Server{Handler: mux} 61 | go func() { 62 | srv.Serve(ln) 63 | }() 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /config-reloader/processors/destinations.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package processors 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | 10 | "github.com/vmware/kube-fluentd-operator/config-reloader/fluentd" 11 | "github.com/vmware/kube-fluentd-operator/config-reloader/util" 12 | ) 13 | 14 | const ( 15 | paramBufferPath = "buffer_path" 16 | ) 17 | 18 | type fixDestinations struct { 19 | BaseProcessorState 20 | } 21 | 22 | func makeSafeBufferPath(ctx *ProcessorContext, origBufPath string) string { 23 | // make a custom buffer path directory if BufferMountFolder is set: 24 | if ctx.BufferMountFolder != "" { 25 | return fmt.Sprintf("/var/log/%s/kfo-%s-%s-%s.buf", ctx.BufferMountFolder, util.MakeFluentdSafeName(ctx.DeploymentID), ctx.Namespace, util.Hash("", origBufPath)) 26 | } 27 | return fmt.Sprintf("/var/log/kfo-%s-%s-%s.buf", util.MakeFluentdSafeName(ctx.DeploymentID), ctx.Namespace, util.Hash("", origBufPath)) 28 | } 29 | 30 | func prohibitSources(d *fluentd.Directive, ctx *ProcessorContext) error { 31 | if d.Name == "source" { 32 | if d.Type() != mountedFileSourceType { 33 | return errors.New("cannot use directive") 34 | } 35 | } 36 | 37 | return nil 38 | } 39 | 40 | func prohibitTypes(d *fluentd.Directive, ctx *ProcessorContext) error { 41 | if d.Name != "match" && 42 | d.Name != "store" && 43 | d.Name != "filter" { 44 | return nil 45 | } 46 | 47 | switch d.Type() { 48 | case "exec", "exec_filter", 49 | "stdout", "rewrite_tag_filter": 50 | return fmt.Errorf("cannot use '@type %s' in <%s>", d.Type(), d.Name) 51 | case "detect_exceptions": 52 | if d.Name == "match" { 53 | return fmt.Errorf("cannot use '@type %s' in <%s>", d.Type(), d.Name) 54 | } 55 | case "file": 56 | if !ctx.AllowFile { 57 | return fmt.Errorf("cannot use '@type %s' in <%s>", d.Type(), d.Name) 58 | } 59 | case "fields_parser": 60 | if d.Param("remove_tag_prefix") != "" || 61 | d.Param("add_tag_prefix") != "" { 62 | return fmt.Errorf("cannot modify tags using the plugin %s", d.Type()) 63 | } 64 | } 65 | 66 | return nil 67 | } 68 | 69 | func rewriteBufferPath(d *fluentd.Directive, ctx *ProcessorContext) error { 70 | if d.Name == "match" || d.Name == "store" { 71 | origBufPath := d.Param(paramBufferPath) 72 | if origBufPath != "" { 73 | d.SetParam(paramBufferPath, makeSafeBufferPath(ctx, origBufPath)) 74 | } 75 | return nil 76 | } 77 | 78 | if d.Name == "buffer" && d.Type() == "file" { 79 | path := d.Param("path") 80 | if path != "" { 81 | d.SetParam("path", makeSafeBufferPath(ctx, path)) 82 | } 83 | } 84 | 85 | return nil 86 | } 87 | 88 | func (p *fixDestinations) Process(input fluentd.Fragment) (fluentd.Fragment, error) { 89 | funcs := []func(*fluentd.Directive, *ProcessorContext) error{ 90 | prohibitTypes, 91 | rewriteBufferPath, 92 | prohibitSources, 93 | } 94 | 95 | for _, f := range funcs { 96 | err := applyRecursivelyInPlace(input, p.Context, f) 97 | if err != nil { 98 | return nil, err 99 | } 100 | } 101 | 102 | return input, nil 103 | } 104 | -------------------------------------------------------------------------------- /config-reloader/processors/destinations_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package processors 5 | 6 | import ( 7 | "fmt" 8 | "testing" 9 | 10 | "github.com/vmware/kube-fluentd-operator/config-reloader/fluentd" 11 | 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func TestDestinationsRewriteBufferPath(t *testing.T) { 16 | var s = ` 17 | 18 | @type logzio 19 | 20 | @type file 21 | path /etc/passwd 22 | 23 | 24 | 25 | 26 | @type kafka 27 | buffer_path /etc/hosts 28 | 29 | 30 | 31 | @type record_transformer 32 | enable_ruby true 33 | 34 | hostname ${record["kubernetes"]["namespace_name"]}-${record["kubernetes"]["pod_name"]} 35 | program ${record["kubernetes"]["container_name"]} 36 | severity info 37 | facility local0 38 | message ${record['log']} 39 | 40 | 41 | 42 | 43 | @type whatever 44 | 45 | ` 46 | fragment, err := fluentd.ParseString(s) 47 | assert.Nil(t, err) 48 | 49 | fmt.Printf("Original:\n%s", fragment) 50 | 51 | ctx := &ProcessorContext{ 52 | Namespace: "monitoring", 53 | } 54 | fragment, err = Process(fragment, ctx, &fixDestinations{}) 55 | assert.Nil(t, err) 56 | fmt.Printf("Processed: %s", fragment) 57 | 58 | logzio := fragment[0] 59 | assert.NotEqual(t, "/etc/passwd", logzio.Nested[0].Param("path")) 60 | 61 | kafka := fragment[1] 62 | assert.NotEqual(t, "/etc/hosts", kafka.Param(paramBufferPath)) 63 | 64 | whatever := fragment[2] 65 | assert.Equal(t, "", whatever.Param(paramBufferPath)) 66 | } 67 | 68 | func TestExpandBadConfig(t *testing.T) { 69 | ctx := &ProcessorContext{ 70 | Namespace: "monitoring", 71 | } 72 | 73 | list := []string{ 74 | ` 75 | @type file 76 | `, 77 | ` 78 | @type stdout 79 | `, 80 | ` 81 | @type rewrite_tag_filter 82 | 83 | `, 84 | ` 85 | 86 | @type syslog 87 | 88 | `, 89 | } 90 | 91 | for _, s := range list { 92 | fragment, err := fluentd.ParseString(s) 93 | assert.Nil(t, err) 94 | 95 | _, err = Process(fragment, ctx, &fixDestinations{}) 96 | assert.NotNil(t, err) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /config-reloader/processors/detect_exceptions.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package processors 5 | 6 | import ( 7 | "fmt" 8 | "strings" 9 | 10 | "github.com/vmware/kube-fluentd-operator/config-reloader/fluentd" 11 | "github.com/vmware/kube-fluentd-operator/config-reloader/util" 12 | ) 13 | 14 | const ( 15 | keyDetExc = "detexc" 16 | ) 17 | 18 | type detectExceptionsState struct { 19 | BaseProcessorState 20 | } 21 | 22 | func (state *detectExceptionsState) Prepare(input fluentd.Fragment) (fluentd.Fragment, error) { 23 | cb := func(d *fluentd.Directive, ctx *ProcessorContext) error { 24 | if d.Name == "filter" && d.Type() == "detect_exceptions" { 25 | ctx.GenerationContext.NeedsProcessing = true 26 | } 27 | 28 | return nil 29 | } 30 | 31 | applyRecursivelyInPlace(input, state.Context, cb) 32 | return nil, nil 33 | } 34 | 35 | func (state *detectExceptionsState) Process(input fluentd.Fragment) (fluentd.Fragment, error) { 36 | rewrite := func(dir *fluentd.Directive, parent *fluentd.Fragment) *fluentd.Directive { 37 | if dir.Name != "filter" || dir.Type() != "detect_exceptions" { 38 | c := dir.Clone() 39 | *parent = append(*parent, c) 40 | return c 41 | } 42 | 43 | unprocessedSelector := extractSelector(dir.Tag) 44 | tagPrefix := makeTagPrefix(unprocessedSelector) 45 | 46 | rule := &fluentd.Directive{ 47 | Name: "rule", 48 | Params: fluentd.Params{}, 49 | } 50 | rule.SetParam("key", "_dummy") 51 | rule.SetParam("pattern", "/ZZ/") 52 | rule.SetParam("invert", "true") 53 | rule.SetParam("tag", fmt.Sprintf("%s.%s.${tag}", tagPrefix, prefixProcessed)) 54 | 55 | rewriteTag := &fluentd.Directive{ 56 | Name: "match", 57 | Tag: unprocessedSelector, 58 | Params: fluentd.ParamsFromKV("@type", "rewrite_tag_filter"), 59 | Nested: fluentd.Fragment{rule}, 60 | } 61 | 62 | detectExceptions := &fluentd.Directive{ 63 | Name: "match", 64 | Tag: fmt.Sprintf("%s.%s.%s", tagPrefix, prefixProcessed, unprocessedSelector), 65 | Params: fluentd.ParamsFromKV("@type", "detect_exceptions"), 66 | } 67 | detectExceptions.SetParam("stream", "container_info") 68 | detectExceptions.SetParam("remove_tag_prefix", tagPrefix) 69 | 70 | // copy all relevant params from the original directive 71 | // https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions 72 | copyParam("languages", dir, detectExceptions) 73 | copyParam("multiline_flush_interval", dir, detectExceptions) 74 | copyParam("max_lines", dir, detectExceptions) 75 | copyParam("max_bytes", dir, detectExceptions) 76 | copyParam("message", dir, detectExceptions) 77 | 78 | *parent = append(*parent, rewriteTag, detectExceptions) 79 | 80 | return nil 81 | } 82 | 83 | res := transform(input, rewrite) 84 | return res, nil 85 | } 86 | 87 | func makeTagPrefix(selector string) string { 88 | return util.Hash(keyDetExc, selector) 89 | } 90 | 91 | func extractSelector(tag string) string { 92 | parts := strings.Split(tag, " ") 93 | // abstraction leak: the labels processor has produced a tag in the form "xxx _proc.xxx" 94 | // the auto-generated directives need only the first one 95 | return parts[0] 96 | } 97 | 98 | func copyParam(name string, src, dest *fluentd.Directive) { 99 | val := src.Param(name) 100 | if val != "" { 101 | dest.SetParam(name, val) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /config-reloader/processors/detect_exceptions_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package processors 5 | 6 | import ( 7 | "fmt" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/vmware/kube-fluentd-operator/config-reloader/fluentd" 12 | ) 13 | 14 | func TestRewrite(t *testing.T) { 15 | ctx := &ProcessorContext{ 16 | Namespace: "monitoring", 17 | GenerationContext: &GenerationContext{ 18 | ReferencedBridges: map[string]bool{}, 19 | }, 20 | } 21 | 22 | s := ` 23 | 24 | @type detect_exceptions 25 | languages java, python 26 | 27 | 28 | 29 | @type parse 30 | format apache2 31 | 32 | 33 | 34 | @type null 35 | 36 | ` 37 | 38 | fragment, err := fluentd.ParseString(s) 39 | assert.Nil(t, err) 40 | 41 | detExc := &detectExceptionsState{} 42 | labelProc := &expandLabelsMacroState{} 43 | expandThis := &expandThisnsMacroState{} 44 | 45 | _, err = Prepare(fragment, ctx, expandThis, labelProc, detExc) 46 | assert.Nil(t, err) 47 | 48 | processed, err := Process(fragment, ctx, expandThis, labelProc, detExc) 49 | assert.Nil(t, err) 50 | assert.Equal(t, 7, len(processed)) 51 | 52 | fmt.Printf("Processed:\n%s\n", processed) 53 | } 54 | 55 | func TestWithoutExceptions(t *testing.T) { 56 | ctx := &ProcessorContext{ 57 | Namespace: "monitoring", 58 | GenerationContext: &GenerationContext{ 59 | ReferencedBridges: map[string]bool{}, 60 | }, 61 | AllowTagExpansion: true, 62 | } 63 | 64 | s := ` 65 | 66 | @type parse 67 | format apache2 68 | 69 | 70 | 71 | @type null 72 | 73 | ` 74 | 75 | fragment, err := fluentd.ParseString(s) 76 | assert.Nil(t, err) 77 | 78 | procs := DefaultProcessors() 79 | _, err = Prepare(fragment, ctx, procs...) 80 | assert.Nil(t, err) 81 | 82 | processed, err := Process(fragment, ctx, procs...) 83 | assert.Nil(t, err) 84 | assert.Equal(t, 5, len(processed)) 85 | 86 | fmt.Printf("Processed:\n%s\n", processed) 87 | 88 | last := processed[len(processed)-1] 89 | assert.Equal(t, "kube.monitoring.**", last.Tag) 90 | } 91 | 92 | func TestWithExceptions(t *testing.T) { 93 | ctx := &ProcessorContext{ 94 | Namespace: "monitoring", 95 | GenerationContext: &GenerationContext{ 96 | ReferencedBridges: map[string]bool{}, 97 | }, 98 | AllowTagExpansion: true, 99 | } 100 | 101 | s := ` 102 | 103 | @type parse 104 | format apache2 105 | 106 | 107 | 108 | @type detect_exceptions 109 | language python 110 | 111 | 112 | 113 | @type null 114 | 115 | ` 116 | 117 | fragment, err := fluentd.ParseString(s) 118 | assert.Nil(t, err) 119 | 120 | procs := DefaultProcessors() 121 | _, err = Prepare(fragment, ctx, procs...) 122 | assert.Nil(t, err) 123 | 124 | processed, err := Process(fragment, ctx, procs...) 125 | assert.Nil(t, err) 126 | assert.Equal(t, 7, len(processed)) 127 | 128 | fmt.Printf("Processed:\n%s\n", processed) 129 | 130 | last := processed[len(processed)-1] 131 | assert.Equal(t, "kube.monitoring.** _proc.kube.monitoring.**", last.Tag) 132 | 133 | detExc := processed[len(processed)-2] 134 | assert.Equal(t, "container_info", detExc.Param("stream")) 135 | assert.Equal(t, "detect_exceptions", detExc.Type()) 136 | } 137 | 138 | func TestBuild(t *testing.T) { 139 | var s = ` 140 | 141 | @type logzio 142 | 143 | @type file 144 | path /etc/passwd 145 | 146 | 147 | 148 | 149 | 150 | 151 | @type logzio 152 | 153 | @type file 154 | path /etc/passwd 155 | 156 | 157 | ` 158 | 159 | fragment, err := fluentd.ParseString(s) 160 | assert.Nil(t, err) 161 | 162 | clone := transform(fragment, copy) 163 | 164 | assert.Equal(t, fragment.String(), clone.String()) 165 | } 166 | 167 | func TestExtractSelector(t *testing.T) { 168 | assert.Equal(t, "xxx", extractSelector("xxx")) 169 | assert.Equal(t, "xxx", extractSelector("xxx _proc.xxx")) 170 | assert.Equal(t, "xxx", extractSelector("xxx what ever man")) 171 | } 172 | -------------------------------------------------------------------------------- /config-reloader/processors/expand_tags.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "regexp" 7 | "strings" 8 | 9 | "github.com/vmware/kube-fluentd-operator/config-reloader/fluentd" 10 | ) 11 | 12 | const ( 13 | tagRegex = `(?:[^\s{}()]*(?:(?:(?:{.*?})|(?:\(.*?\)))[^\s{}()]*)+)|(?:[^\s{}()]+(?:(?:(?:{.*?})|(?:\(.*?\)))[^\s{}()]*)*)` 14 | ) 15 | 16 | type expandTagsState struct { 17 | BaseProcessorState 18 | tagMatcher *regexp.Regexp 19 | } 20 | 21 | func (p *expandTagsState) Process(input fluentd.Fragment) (fluentd.Fragment, error) { 22 | if p.Context.AllowTagExpansion { 23 | return p.ProcessExpandingTags(input) 24 | } 25 | 26 | return p.ProcessNotExpandingTags(input) 27 | } 28 | 29 | func (p *expandTagsState) ProcessExpandingTags(input fluentd.Fragment) (fluentd.Fragment, error) { 30 | f := func(d *fluentd.Directive, ctx *ProcessorContext) ([]*fluentd.Directive, error) { 31 | if d.Name != "match" && d.Name != "filter" { 32 | return []*fluentd.Directive{d}, nil 33 | } 34 | 35 | if p.tagMatcher == nil { 36 | p.tagMatcher = regexp.MustCompile(tagRegex) 37 | } 38 | 39 | expandingTags := p.tagMatcher.FindAllString(d.Tag, -1) 40 | remainders := p.tagMatcher.Split(d.Tag, -1) 41 | 42 | if len(strings.TrimSpace(strings.Join(remainders, ""))) > 0 { 43 | return nil, fmt.Errorf("Malformed tag %s. Cannot parse it", d.Tag) 44 | } 45 | 46 | var processingTags []string 47 | 48 | for len(expandingTags) > len(processingTags) { 49 | processingTags = expandingTags 50 | expandingTags = []string{} 51 | for _, t := range processingTags { 52 | expandedTags, err := expandFirstCurlyBraces(t) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | expandingTags = append(expandingTags, expandedTags...) 58 | } 59 | } 60 | 61 | if len(expandingTags) == 1 { 62 | return []*fluentd.Directive{d}, nil 63 | } 64 | 65 | expandedDirectives := make([]*fluentd.Directive, len(expandingTags)) 66 | for i, t := range expandingTags { 67 | expandedDirectives[i] = d.Clone() 68 | expandedDirectives[i].Tag = t 69 | } 70 | 71 | return expandedDirectives, nil 72 | } 73 | 74 | output, err := applyRecursivelyWithState(input, p.Context, f) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | return output, nil 80 | } 81 | 82 | func expandFirstCurlyBraces(tag string) ([]string, error) { 83 | resultingTags := []string{} 84 | if open := strings.Index(tag, "{"); open >= 0 { 85 | if open > 0 && tag[open-1:open] == "#" { 86 | return nil, errors.New("Pattern #{...} is not yet supported in tag definition") 87 | } 88 | if close := strings.Index(tag, "}"); close > open+1 { 89 | expansionTerm := tag[open+1 : close] 90 | expansionTerms := strings.Split(expansionTerm, ",") 91 | 92 | for _, t := range expansionTerms { 93 | resultingTags = append(resultingTags, tag[:open]+strings.TrimSpace(t)+tag[close+1:]) 94 | } 95 | } else { 96 | return nil, errors.New("Invalid {...} pattern in tag definition") 97 | } 98 | } else { 99 | resultingTags = append(resultingTags, tag) 100 | } 101 | 102 | return resultingTags, nil 103 | } 104 | 105 | func applyRecursivelyWithState(directives fluentd.Fragment, ctx *ProcessorContext, callback func(*fluentd.Directive, *ProcessorContext) ([]*fluentd.Directive, error)) (fluentd.Fragment, error) { 106 | if directives == nil { 107 | return nil, nil 108 | } 109 | 110 | for _, d := range directives { 111 | output, err := applyRecursivelyWithState(d.Nested, ctx, callback) 112 | if err != nil { 113 | return nil, err 114 | } 115 | d.Nested = output 116 | } 117 | 118 | newDirectives := []*fluentd.Directive{} 119 | for _, d := range directives { 120 | output, err := callback(d, ctx) 121 | if err != nil { 122 | return nil, err 123 | } 124 | newDirectives = append(newDirectives, output...) 125 | } 126 | 127 | return newDirectives, nil 128 | } 129 | 130 | func (p *expandTagsState) ProcessNotExpandingTags(input fluentd.Fragment) (fluentd.Fragment, error) { 131 | f := func(d *fluentd.Directive, ctx *ProcessorContext) error { 132 | if d.Name != "match" && d.Name != "filter" { 133 | return nil 134 | } 135 | 136 | if strings.Contains(d.Tag, "{") { 137 | return fmt.Errorf("Processing of {...} pattern in tags is disabled") 138 | } 139 | 140 | return nil 141 | } 142 | 143 | err := applyRecursivelyInPlace(input, p.Context, f) 144 | 145 | if err != nil { 146 | return nil, err 147 | } 148 | 149 | return input, nil 150 | } 151 | -------------------------------------------------------------------------------- /config-reloader/processors/expand_tags_test.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/vmware/kube-fluentd-operator/config-reloader/fluentd" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestTagsExpandOk(t *testing.T) { 14 | var input = []string{ 15 | ` 16 | @type null 17 | `, 18 | ` 19 | @type null 20 | `, 21 | } 22 | 23 | for _, s := range input { 24 | fragment, err := fluentd.ParseString(s) 25 | assert.Nil(t, err) 26 | 27 | fmt.Printf("Original:\n%s", fragment) 28 | 29 | ctx := &ProcessorContext{ 30 | Namespace: "monitoring", 31 | GenerationContext: &GenerationContext{}, 32 | AllowTagExpansion: true, 33 | } 34 | fragment, err = Process(fragment, ctx, &expandTagsState{}) 35 | assert.Nil(t, err) 36 | assert.Equal(t, len(fragment), 3) 37 | fmt.Printf("Processed:\n%s", fragment) 38 | 39 | app1 := fragment[0] 40 | assert.Equal(t, "kube.monitoring.app1.**", app1.Tag) 41 | 42 | app2 := fragment[1] 43 | assert.Equal(t, "kube.monitoring.app2.**", app2.Tag) 44 | 45 | app3 := fragment[2] 46 | assert.Equal(t, "kube.monitoring.app3.**", app3.Tag) 47 | 48 | assert.True(t, !strings.Contains(fragment.String(), "{")) 49 | assert.True(t, !strings.Contains(fragment.String(), "}")) 50 | } 51 | } 52 | 53 | func TestNestedTagsExpandOk(t *testing.T) { 54 | var s = ` 55 | 56 | @type relabel 57 | @label @test 58 | 59 | 60 | 65 | ` 66 | 67 | fragment, err := fluentd.ParseString(s) 68 | assert.Nil(t, err) 69 | 70 | fmt.Printf("Original:\n%s", fragment) 71 | 72 | ctx := &ProcessorContext{ 73 | Namespace: "monitoring", 74 | GenerationContext: &GenerationContext{}, 75 | AllowTagExpansion: true, 76 | } 77 | fragment, err = Process(fragment, ctx, DefaultProcessors()...) 78 | assert.Nil(t, err) 79 | assert.Equal(t, len(fragment), 2) 80 | assert.Equal(t, len(fragment[1].Nested), 3) 81 | fmt.Printf("Processed:\n%s", fragment) 82 | 83 | app1 := fragment[1].Nested[0] 84 | assert.Equal(t, "kube.monitoring.app1.**", app1.Tag) 85 | 86 | app2 := fragment[1].Nested[1] 87 | assert.Equal(t, "kube.monitoring.app2.**", app2.Tag) 88 | 89 | app3 := fragment[1].Nested[2] 90 | assert.Equal(t, "kube.monitoring.app3.**", app3.Tag) 91 | 92 | assert.True(t, !strings.Contains(fragment.String(), "{")) 93 | assert.True(t, !strings.Contains(fragment.String(), "}")) 94 | } 95 | 96 | func TestTagsExpandBadConfig(t *testing.T) { 97 | ctx := &ProcessorContext{ 98 | Namespace: "monitoring", 99 | AllowTagExpansion: true, 100 | } 101 | 102 | list := []string{ 103 | ` 104 | @type null 105 | `, 106 | ` 107 | @type null 108 | `, 109 | ` 110 | @type null 111 | `, 112 | } 113 | 114 | for _, s := range list { 115 | fragment, err := fluentd.ParseString(s) 116 | assert.Nil(t, err) 117 | 118 | _, err = Process(fragment, ctx, &expandTagsState{}) 119 | assert.NotNil(t, err) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /config-reloader/processors/extract_plugins.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "github.com/vmware/kube-fluentd-operator/config-reloader/fluentd" 5 | ) 6 | 7 | const ( 8 | dirPlugin = "plugin" 9 | ) 10 | 11 | // ExtractPlugins looks at the top-level directives in the admin namespace, deletes all 12 | // and stores the found plugin definitions under GenerationContext.Plugins map keyed by the plugin directive's path 13 | func ExtractPlugins(g *GenerationContext, input fluentd.Fragment) fluentd.Fragment { 14 | plugins := map[string]*fluentd.Directive{} 15 | res := fluentd.Fragment{} 16 | 17 | // process only top-level plugin directives 18 | for _, dir := range input { 19 | if dir.Name == dirPlugin { 20 | plugins[dir.Tag] = dir 21 | } else { 22 | res = append(res, dir) 23 | } 24 | } 25 | 26 | g.Plugins = plugins 27 | 28 | return res 29 | } 30 | 31 | type expandPluginsState struct { 32 | BaseProcessorState 33 | } 34 | 35 | func (p *expandPluginsState) Process(input fluentd.Fragment) (fluentd.Fragment, error) { 36 | if len(p.Context.GenerationContext.Plugins) == 0 { 37 | // nothing to expand 38 | return input, nil 39 | } 40 | 41 | f := func(d *fluentd.Directive, ctx *ProcessorContext) error { 42 | if d.Name != "match" && d.Name != "store" { 43 | // only output plugins supported 44 | return nil 45 | } 46 | 47 | replacement, ok := p.Context.GenerationContext.Plugins[d.Type()] 48 | if !ok { 49 | return nil 50 | } 51 | 52 | // replace any nested content (buffers etc) 53 | // there is no option to redefine nested content 54 | d.Nested = replacement.Nested.Clone() 55 | 56 | // replace the params 57 | for k, v := range replacement.Params { 58 | // prefer the params defined at the call site 59 | if _, ok := d.Params[k]; !ok { 60 | d.Params[k] = v.Clone() 61 | } 62 | } 63 | 64 | // always change the type 65 | d.SetParam("@type", replacement.Type()) 66 | 67 | // delete type parameter (someone is using the old syntax without @) 68 | delete(d.Params, "type") 69 | 70 | return nil 71 | } 72 | 73 | err := applyRecursivelyInPlace(input, p.Context, f) 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | return input, nil 79 | } 80 | -------------------------------------------------------------------------------- /config-reloader/processors/labels.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package processors 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "regexp" 10 | "strings" 11 | 12 | "github.com/vmware/kube-fluentd-operator/config-reloader/fluentd" 13 | "github.com/vmware/kube-fluentd-operator/config-reloader/template" 14 | "github.com/vmware/kube-fluentd-operator/config-reloader/util" 15 | ) 16 | 17 | type expandLabelsMacroState struct { 18 | BaseProcessorState 19 | } 20 | 21 | var reSafe = regexp.MustCompile(`[.-]|^$`) 22 | 23 | // got this value from running kubectl with bad args 24 | // error: invalid label value: "test=-asdf": a valid label must be an empty string 25 | // or consist of alphanumeric characters, '-', '_' or '.', and must start and end with 26 | // an alphanumeric character (e.g. 'MyValue', or 'my_value', or '12345', regex used for validation is 27 | // '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?' 28 | 29 | var retagTemplate = template.Must(template.New("retagTemplate").Parse( 30 | ` 31 | 32 | @type record_transformer 33 | enable_ruby true 34 | 35 | kubernetes_pod_label_values {{range $i, $e := .Labels -}}${record.dig('kubernetes','labels','{{$e}}')&.gsub(/[.-]/, '_') || '_'}{{if isLast $i $.Labels }}{{else}}.{{end}}{{- end}} 36 | 37 | 38 | 39 | 40 | @type rewrite_tag_filter 41 | 42 | key kubernetes_pod_label_values 43 | pattern ^(.+)$ 44 | tag ${tag}._labels.$1 45 | 46 | 47 | 48 | 49 | @type record_transformer 50 | remove_keys kubernetes_pod_label_values 51 | 52 | `)) 53 | 54 | func makeTagFromFilter(ns string, sortedLabelNames []string, labelNames map[string]string) string { 55 | buf := &bytes.Buffer{} 56 | 57 | if cont, ok := labelNames[util.ContainerLabel]; ok { 58 | // if the special label _container is used then its name goes to the 59 | // part of the tag that denotes the container 60 | buf.WriteString(fmt.Sprintf("kube.%s.*.%s._labels.", ns, cont)) 61 | } else { 62 | buf.WriteString(fmt.Sprintf("kube.%s.*.*._labels.", ns)) 63 | } 64 | 65 | for i, lb := range sortedLabelNames { 66 | if lb == util.ContainerLabel { 67 | continue 68 | } 69 | 70 | val, ok := labelNames[lb] 71 | if ok { 72 | buf.WriteString(safeLabelValue(val)) 73 | } else { 74 | buf.WriteString("*") 75 | } 76 | 77 | if i < len(sortedLabelNames)-1 { 78 | buf.WriteString(".") 79 | } 80 | } 81 | 82 | return buf.String() 83 | } 84 | 85 | // replaces the empty string and all . with _ 86 | // as they have special meaning to fluentd 87 | func safeLabelValue(s string) string { 88 | return reSafe.ReplaceAllString(s, "_") 89 | } 90 | 91 | func (p *expandLabelsMacroState) Process(input fluentd.Fragment) (fluentd.Fragment, error) { 92 | allReferencedLabels := map[string]string{} 93 | collectLabels := func(d *fluentd.Directive, ctx *ProcessorContext) error { 94 | if d.Name != "filter" && d.Name != "match" { 95 | return nil 96 | } 97 | 98 | if !strings.HasPrefix(d.Tag, util.MacroLabels) { 99 | return nil 100 | } 101 | 102 | labelNames, err := util.ParseTagToLabels(d.Tag) 103 | if err != nil { 104 | return err 105 | } 106 | 107 | for lb := range labelNames { 108 | allReferencedLabels[lb] = "" 109 | } 110 | 111 | return nil 112 | } 113 | e := applyRecursivelyInPlace(input, p.Context, collectLabels) 114 | if e != nil { 115 | return nil, e 116 | } 117 | if len(allReferencedLabels) == 0 { 118 | return input, nil 119 | } 120 | 121 | delete(allReferencedLabels, util.ContainerLabel) 122 | sortedLabelNames := util.SortedKeys(allReferencedLabels) 123 | 124 | replaceLabels := func(d *fluentd.Directive, ctx *ProcessorContext) error { 125 | if d.Name != "filter" && d.Name != "match" { 126 | return nil 127 | } 128 | 129 | if !strings.HasPrefix(d.Tag, util.MacroLabels) { 130 | return nil 131 | } 132 | 133 | labelNames, err := util.ParseTagToLabels(d.Tag) 134 | if err != nil { 135 | // should never happen as the error should be caught beforehand 136 | return nil 137 | } 138 | 139 | d.Tag = makeTagFromFilter(ctx.Namespace, sortedLabelNames, labelNames) 140 | ctx.GenerationContext.augmentTag(d) 141 | return nil 142 | } 143 | applyRecursivelyInPlace(input, p.Context, replaceLabels) 144 | 145 | // prepare extra directives 146 | model := struct { 147 | Pattern string 148 | Labels []string 149 | }{ 150 | fmt.Sprintf("kube.%s.*.*", p.Context.Namespace), 151 | sortedLabelNames, 152 | } 153 | writer := &bytes.Buffer{} 154 | retagTemplate.Execute(writer, model) 155 | 156 | extraDirectives, err := fluentd.ParseString(writer.String()) 157 | if err != nil { 158 | return nil, err 159 | } 160 | 161 | extraDirectives = append(extraDirectives, input...) 162 | 163 | return extraDirectives, nil 164 | } 165 | -------------------------------------------------------------------------------- /config-reloader/processors/processor_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package processors 5 | 6 | import ( 7 | "fmt" 8 | "testing" 9 | 10 | "github.com/vmware/kube-fluentd-operator/config-reloader/fluentd" 11 | 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func Test_callForEveryDirective(t *testing.T) { 16 | var s = ` 17 | 18 | @type logzio 19 | 20 | @type file 21 | path /etc/passwd 22 | 23 | 24 | 25 | 26 | 27 | 28 | @type logzio 29 | 30 | @type file 31 | path /etc/passwd 32 | 33 | 34 | ` 35 | fragment, err := fluentd.ParseString(s) 36 | assert.Nil(t, err) 37 | 38 | fmt.Printf("Original:\n%s", fragment) 39 | 40 | ctx := &ProcessorContext{ 41 | Namespace: "test", 42 | } 43 | count := 0 44 | inc := func(*fluentd.Directive, *ProcessorContext) error { 45 | count++ 46 | return nil 47 | } 48 | 49 | err = applyRecursivelyInPlace(fragment, ctx, inc) 50 | assert.Nil(t, err, "was error instead %+v", err) 51 | assert.Equal(t, 5, count) 52 | } 53 | 54 | func TestTransformAndCopy(t *testing.T) { 55 | s := ` 56 | 57 | @type logzio 58 | 59 | @type file 60 | path /etc/passwd 61 | 62 | 63 | 64 | 65 | 66 | 67 | @type logzio 68 | 69 | @type file 70 | path /etc/passwd 71 | 72 | 73 | ` 74 | fragment, err := fluentd.ParseString(s) 75 | assert.Nil(t, err) 76 | 77 | clone := transform(fragment, copy) 78 | 79 | assert.Equal(t, fragment.String(), clone.String()) 80 | } 81 | 82 | func TestAugmentTag(t *testing.T) { 83 | s := augmentTag("hello") 84 | assert.Equal(t, "hello _proc.hello", s) 85 | } 86 | 87 | func TestCloneFragment(t *testing.T) { 88 | s := ` 89 | 90 | @type logzio 91 | 92 | @type file 93 | path /etc/passwd 94 | 95 | 96 | ` 97 | fragment, err := fluentd.ParseString(s) 98 | assert.Nil(t, err) 99 | 100 | clone := fragment.Clone() 101 | assert.Equal(t, fragment.String(), clone.String()) 102 | assert.NotEqual(t, fragment[0].Nested[0].Params, &clone[0].Nested[0].Params) 103 | } 104 | -------------------------------------------------------------------------------- /config-reloader/processors/relabel.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package processors 5 | 6 | import ( 7 | "fmt" 8 | "strings" 9 | 10 | "github.com/vmware/kube-fluentd-operator/config-reloader/fluentd" 11 | "github.com/vmware/kube-fluentd-operator/config-reloader/util" 12 | ) 13 | 14 | type rewriteLabelsState struct { 15 | BaseProcessorState 16 | } 17 | 18 | var validLabelDirectives = []string{"match", "store", "filter", "parse", "source"} 19 | var validLabelTypes = []string{"relabel", "null", "forward", "stdout", "copy", "kafka", "elasticsearch"} 20 | 21 | func contains(slice []string, item string) bool { 22 | set := make(map[string]struct{}, len(slice)) 23 | for _, s := range slice { 24 | set[s] = struct{}{} 25 | } 26 | 27 | _, ok := set[item] 28 | return ok 29 | } 30 | 31 | func normalizeLabelName(ctx *ProcessorContext, label string) string { 32 | if strings.HasPrefix(label, "@$") { 33 | // cross dependency to the share.go processor 34 | return label 35 | } 36 | 37 | return fmt.Sprintf("@%s-%s", 38 | util.MakeFluentdSafeName(label), 39 | util.Hash(ctx.Namespace, label)) 40 | } 41 | 42 | func (p *rewriteLabelsState) Process(input fluentd.Fragment) (fluentd.Fragment, error) { 43 | normalizeAllLabels := func(d *fluentd.Directive, ctx *ProcessorContext) error { 44 | if !contains(validLabelDirectives, d.Name) { 45 | return nil 46 | } 47 | 48 | // Process any timeout_labels since they are valid labels: 49 | timeoutLabel := d.Param("timeout_label") 50 | if timeoutLabel != "" { 51 | if !strings.HasPrefix(timeoutLabel, "@") { 52 | return fmt.Errorf("bad label name %s for timeout_label, must start with @", timeoutLabel) 53 | } 54 | 55 | d.SetParam("timeout_label", normalizeLabelName(ctx, timeoutLabel)) 56 | } 57 | 58 | // Continue parsing normal @labels: 59 | if !contains(validLabelTypes, d.Type()) { 60 | return nil 61 | } 62 | 63 | labelName := d.Param("@label") 64 | if labelName != "" { 65 | if !strings.HasPrefix(labelName, "@") { 66 | return fmt.Errorf("bad label name %s for @label, must start with @", labelName) 67 | } 68 | 69 | d.SetParam("@label", normalizeLabelName(ctx, labelName)) 70 | } 71 | 72 | return nil 73 | } 74 | 75 | rewriteLabelTag := func(d *fluentd.Directive, ctx *ProcessorContext) error { 76 | if d.Name != "label" { 77 | return nil 78 | } 79 | 80 | labelName := d.Tag 81 | if !strings.HasPrefix(labelName, "@") { 82 | return fmt.Errorf("bad label name %s for