├── config ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── network-policy │ ├── kustomization.yaml │ └── allow-metrics-traffic.yaml ├── default │ ├── manager_metrics_patch.yaml │ └── metrics_service.yaml ├── rbac │ ├── metrics_reader_role.yaml │ ├── service_account.yaml │ ├── metrics_auth_role_binding.yaml │ ├── metrics_auth_role.yaml │ ├── role_binding.yaml │ ├── leader_election_role_binding.yaml │ ├── vector_viewer_role.yaml │ ├── vectorpipeline_viewer_role.yaml │ ├── vectoraggregator_viewer_role.yaml │ ├── vector_editor_role.yaml │ ├── clustervectorpipeline_viewer_role.yaml │ ├── clustervectoraggregator_viewer_role.yaml │ ├── vectorpipeline_editor_role.yaml │ ├── vectoraggregator_editor_role.yaml │ ├── clustervectorpipeline_editor_role.yaml │ ├── clustervectoraggregator_editor_role.yaml │ ├── leader_election_role.yaml │ ├── kustomization.yaml │ └── role.yaml ├── manager │ ├── kustomization.yaml │ └── manager.yaml ├── samples │ ├── observability_v1alpha1_vectoraggregator.yaml │ ├── observability_v1alpha1_clustervectoraggregator.yaml │ ├── kustomization.yaml │ ├── observability_v1alpha1_clustervectorpipeline.yaml │ ├── observability_v1alpha1_vector.yaml │ └── observability_v1alpha1_vectorpipeline.yaml └── crd │ ├── kustomizeconfig.yaml │ ├── kustomization.yaml │ └── bases │ ├── observability.kaasops.io_vectorpipelines.yaml │ └── observability.kaasops.io_clustervectorpipelines.yaml ├── internal ├── buildinfo │ └── info.go ├── config │ ├── configcheck │ │ ├── const.go │ │ ├── configcheck_error.go │ │ ├── configcheck_rbac.go │ │ └── configcheck_config.go │ ├── utils.go │ ├── default.go │ ├── agent.go │ └── vector_source_types.go ├── common │ └── annotations.go ├── evcollector │ ├── config.go │ ├── metrics.go │ └── event.go ├── utils │ ├── compression │ │ └── compression.go │ ├── hash │ │ ├── hash.go │ │ └── hash_test.go │ └── k8s │ │ └── label.go ├── vector │ ├── vectoragent │ │ ├── vectoragent_podmonitor.go │ │ ├── vectoragent_service.go │ │ ├── vectoragent_config.go │ │ ├── vectoragent.go │ │ └── vectoragent_rbac.go │ └── aggregator │ │ ├── config.go │ │ ├── podmonitor.go │ │ ├── service.go │ │ └── rbac.go ├── pipeline │ └── hash.go └── controller │ ├── pipeline_controller_test.go │ ├── clustervectoraggregator_controller_test.go │ ├── vector_controller_test.go │ └── vectoraggregator_controller_test.go ├── docs ├── images │ ├── logo.png │ └── telegram-logo.png ├── README.md ├── logs-from-file.md ├── journald-logs.md ├── kubernetes-events.md ├── secure-credential.md └── aggregator.md ├── helm ├── packages │ ├── vector-operator-0.2.tgz │ ├── vector-operator-0.3.tgz │ ├── vector-operator-0.4.tgz │ ├── vector-operator-0.5.tgz │ ├── vector-operator-0.6.tgz │ ├── vector-operator-0.7.tgz │ ├── vector-operator-0.0.1.tgz │ ├── vector-operator-0.0.6.tgz │ ├── vector-operator-0.0.7.tgz │ ├── vector-operator-0.0.8.tgz │ ├── vector-operator-0.0.9.tgz │ ├── vector-operator-0.1.0.tgz │ ├── vector-operator-0.1.1.tgz │ ├── vector-operator-0.7.1.tgz │ ├── vector-operator-0.7.2.tgz │ ├── vector-operator-0.8.0.tgz │ ├── vector-operator-0.0.10.tgz │ ├── vector-operator-0.0.11.tgz │ ├── vector-operator-0.0.12.tgz │ ├── vector-operator-0.0.13.tgz │ ├── vector-operator-0.0.14.tgz │ ├── vector-operator-0.0.15.tgz │ ├── vector-operator-0.0.16.tgz │ ├── vector-operator-0.0.17.tgz │ ├── vector-operator-0.0.18.tgz │ ├── vector-operator-0.0.19.tgz │ ├── vector-operator-0.0.20.tgz │ ├── vector-operator-0.0.21.tgz │ ├── vector-operator-0.0.22.tgz │ ├── vector-operator-0.0.23.tgz │ ├── vector-operator-0.0.24.tgz │ ├── vector-operator-0.0.25.tgz │ ├── vector-operator-0.0.26.tgz │ ├── vector-operator-0.0.27.tgz │ ├── vector-operator-0.0.28.tgz │ ├── vector-operator-0.0.29.tgz │ ├── vector-operator-0.0.30.tgz │ ├── vector-operator-0.0.31.tgz │ ├── vector-operator-0.0.32.tgz │ ├── vector-operator-0.0.33.tgz │ ├── vector-operator-0.0.34.tgz │ ├── vector-operator-0.0.35.tgz │ ├── vector-operator-0.0.36.tgz │ ├── vector-operator-0.0.37.tgz │ ├── vector-operator-0.0.38.tgz │ ├── vector-operator-0.0.39.tgz │ ├── vector-operator-0.0.40.tgz │ └── vector-operator-0.1.0-rc1.tgz ├── charts │ └── vector-operator │ │ ├── templates │ │ ├── vector.yaml │ │ ├── clustervectorpipeline.yaml │ │ ├── secrets.yaml │ │ ├── clusterrolebinding.yaml │ │ ├── serviceaccount.yaml │ │ ├── deployment.yaml │ │ ├── _helpers.tpl │ │ ├── openshift.yaml │ │ └── clusterrole.yaml │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── values.yaml │ │ └── crds │ │ ├── observability.kaasops.io_vectorpipelines.yaml │ │ └── observability.kaasops.io_clustervectorpipelines.yaml └── Makefile ├── test └── e2e │ ├── testdata │ ├── normal-mode │ │ ├── namespace-isolation-ns.yaml │ │ ├── agent.yaml │ │ ├── aggregator.yaml │ │ ├── pipeline-basic.yaml │ │ ├── pipeline-template.yaml │ │ ├── pipeline-deletable.yaml │ │ ├── namespace-isolation-pod-main.yaml │ │ ├── namespace-isolation-pipeline.yaml │ │ ├── cluster-pipeline-pod-ns1.yaml │ │ ├── namespace-isolation-pod-isolated.yaml │ │ ├── cluster-pipeline-pod-ns2.yaml │ │ ├── test-app-pod.yaml │ │ ├── pipeline-kubernetes-logs.yaml │ │ ├── cluster-pipeline.yaml │ │ ├── pipeline-aggregator-role.yaml │ │ └── pipeline-complex.yaml │ ├── selector-matching │ │ ├── agent.yaml │ │ ├── cva-no-selector.yaml │ │ ├── cva-matching.yaml │ │ ├── cva-non-matching.yaml │ │ ├── cvp-no-labels.yaml │ │ └── cvp-with-labels.yaml │ └── podmonitor │ │ ├── agent-no-metrics.yaml │ │ ├── agent-with-defaults.yaml │ │ ├── agent-with-scrape-config.yaml │ │ ├── aggregator-with-defaults.yaml │ │ ├── agent-with-updated-interval.yaml │ │ ├── aggregator-with-scrape-config.yaml │ │ ├── test-pod.yaml │ │ ├── cluster-aggregator-with-defaults.yaml │ │ ├── pipeline-aggregator-role.yaml │ │ ├── pipeline-without-exporter.yaml │ │ ├── cluster-aggregator-with-scrape-config.yaml │ │ └── pipeline-with-custom-exporter.yaml │ ├── testdata_helper.go │ ├── scripts │ └── README.md │ ├── framework │ ├── resources.go │ ├── config │ │ ├── constants.go │ │ └── timeouts.go │ ├── artifacts │ │ └── storage_test.go │ ├── lifecycle.go │ └── errors │ │ └── errors.go │ └── e2e_test.go ├── buf.yaml ├── .dockerignore ├── buf.gen.yaml ├── .golangci.yml ├── scripts └── kind-config-ci.yaml ├── proto └── vector │ └── vector.proto ├── .gitignore ├── hack └── boilerplate.go.txt ├── .github └── workflows │ ├── unit-tests.yaml │ ├── lint.yaml │ ├── build-image.yaml │ └── e2e-tests.yaml ├── api └── v1alpha1 │ ├── groupversion_info.go │ ├── clustervectorpipeline.go │ ├── vectorpipeline.go │ ├── clustervectorpipeline_types.go │ ├── vector_types.go │ ├── clustervectoraggregator_types.go │ ├── vectoraggregator_types.go │ └── vectorpipeline_types.go ├── Dockerfile ├── event_collector.Dockerfile ├── PROJECT └── cmd └── evgen └── main.go /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /internal/buildinfo/info.go: -------------------------------------------------------------------------------- 1 | package buildinfo 2 | 3 | var Version string 4 | -------------------------------------------------------------------------------- /config/network-policy/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - allow-metrics-traffic.yaml 3 | -------------------------------------------------------------------------------- /docs/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/docs/images/logo.png -------------------------------------------------------------------------------- /docs/images/telegram-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/docs/images/telegram-logo.png -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.2.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.2.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.3.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.3.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.4.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.4.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.5.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.5.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.6.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.6.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.7.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.7.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.1.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.1.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.6.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.6.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.7.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.7.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.8.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.8.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.9.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.9.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.1.0.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.1.0.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.1.1.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.1.1.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.7.1.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.7.1.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.7.2.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.7.2.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.8.0.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.8.0.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.10.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.10.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.11.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.11.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.12.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.12.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.13.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.13.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.14.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.14.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.15.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.15.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.16.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.16.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.17.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.17.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.18.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.18.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.19.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.19.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.20.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.20.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.21.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.21.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.22.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.22.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.23.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.23.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.24.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.24.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.25.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.25.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.26.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.26.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.27.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.27.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.28.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.28.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.29.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.29.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.30.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.30.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.31.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.31.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.32.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.32.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.33.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.33.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.34.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.34.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.35.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.35.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.36.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.36.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.37.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.37.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.38.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.38.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.39.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.39.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.0.40.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.0.40.tgz -------------------------------------------------------------------------------- /helm/packages/vector-operator-0.1.0-rc1.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaasops/vector-operator/HEAD/helm/packages/vector-operator-0.1.0-rc1.tgz -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/namespace-isolation-ns.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: test-normal-mode-isolated 5 | -------------------------------------------------------------------------------- /buf.yaml: -------------------------------------------------------------------------------- 1 | version: v2 2 | modules: 3 | - path: proto/vector 4 | lint: 5 | use: 6 | - DEFAULT 7 | breaking: 8 | use: 9 | - FILE 10 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file 2 | # Ignore build and test binaries. 3 | bin/ 4 | testbin/ 5 | -------------------------------------------------------------------------------- /internal/config/configcheck/const.go: -------------------------------------------------------------------------------- 1 | package configcheck 2 | 3 | const ( 4 | ConfigCheckInitiatorVector = "VectorInitiator" 5 | ConfigCheckInitiatorPipieline = "PipelineInitiator" 6 | ) 7 | -------------------------------------------------------------------------------- /internal/common/annotations.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | const ( 4 | AnnotationServiceName = "observability.kaasops.io/service-name" 5 | AnnotationRestartedAt = "vector-operator.kaasops.io/restartedAt" 6 | ) 7 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/agent.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: Vector 3 | metadata: 4 | name: normal-agent 5 | spec: 6 | agent: 7 | image: timberio/vector:0.40.0-alpine 8 | -------------------------------------------------------------------------------- /test/e2e/testdata/selector-matching/agent.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: Vector 3 | metadata: 4 | name: test-agent 5 | spec: 6 | agent: 7 | image: timberio/vector:0.40.0-alpine 8 | -------------------------------------------------------------------------------- /config/default/manager_metrics_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch adds the args to allow exposing the metrics endpoint using HTTPS 2 | - op: add 3 | path: /spec/template/spec/containers/0/args/0 4 | value: --metrics-bind-address=:8443 5 | -------------------------------------------------------------------------------- /config/rbac/metrics_reader_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: 7 | - "/metrics" 8 | verbs: 9 | - get 10 | -------------------------------------------------------------------------------- /config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manager.yaml 3 | apiVersion: kustomize.config.k8s.io/v1beta1 4 | kind: Kustomization 5 | images: 6 | - name: controller 7 | newName: example.com/vector-operator 8 | newTag: v0.0.1 -------------------------------------------------------------------------------- /config/rbac/service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: vector-operator 6 | app.kubernetes.io/managed-by: kustomize 7 | name: controller-manager 8 | namespace: system 9 | -------------------------------------------------------------------------------- /test/e2e/testdata/podmonitor/agent-no-metrics.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: Vector 3 | metadata: 4 | name: podmonitor-agent-no-metrics 5 | spec: 6 | agent: 7 | image: timberio/vector:0.40.0-alpine 8 | internalMetrics: false 9 | -------------------------------------------------------------------------------- /test/e2e/testdata/podmonitor/agent-with-defaults.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: Vector 3 | metadata: 4 | name: podmonitor-agent-defaults 5 | spec: 6 | agent: 7 | image: timberio/vector:0.40.0-alpine 8 | internalMetrics: true 9 | -------------------------------------------------------------------------------- /internal/evcollector/config.go: -------------------------------------------------------------------------------- 1 | package evcollector 2 | 3 | type ReceiverParams struct { 4 | ServiceName string 5 | ServiceNamespace string 6 | Port string 7 | WatchedNamespace string 8 | } 9 | 10 | type Config struct { 11 | MaxBatchSize int32 12 | Receivers []*ReceiverParams 13 | } 14 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/aggregator.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorAggregator 3 | metadata: 4 | name: normal-aggregator 5 | spec: 6 | image: timberio/vector:0.40.0-alpine 7 | replicas: 1 8 | selector: 9 | matchLabels: 10 | app: test 11 | role: aggregator 12 | -------------------------------------------------------------------------------- /test/e2e/testdata/podmonitor/agent-with-scrape-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: Vector 3 | metadata: 4 | name: podmonitor-agent 5 | spec: 6 | agent: 7 | image: timberio/vector:0.40.0-alpine 8 | internalMetrics: true 9 | scrapeInterval: "45s" 10 | scrapeTimeout: "15s" 11 | -------------------------------------------------------------------------------- /config/samples/observability_v1alpha1_vectoraggregator.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorAggregator 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: vector-operator 6 | app.kubernetes.io/managed-by: kustomize 7 | name: vectoraggregator-sample 8 | spec: 9 | # TODO(user): Add fields here 10 | -------------------------------------------------------------------------------- /internal/config/utils.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | func addPrefix(Namespace, Name, componentName string) string { 4 | return generateName(Namespace, Name) + "-" + componentName 5 | } 6 | 7 | func generateName(Namespace, Name string) string { 8 | if Namespace != "" { 9 | return Namespace + "-" + Name 10 | } 11 | return Name 12 | } 13 | -------------------------------------------------------------------------------- /test/e2e/testdata/podmonitor/aggregator-with-defaults.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorAggregator 3 | metadata: 4 | name: podmonitor-aggregator-defaults 5 | spec: 6 | image: timberio/vector:0.40.0-alpine 7 | replicas: 1 8 | internalMetrics: true 9 | selector: 10 | matchLabels: 11 | app: test 12 | -------------------------------------------------------------------------------- /config/samples/observability_v1alpha1_clustervectoraggregator.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: ClusterVectorAggregator 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: vector-operator 6 | app.kubernetes.io/managed-by: kustomize 7 | name: clustervectoraggregator-sample 8 | spec: 9 | # TODO(user): Add fields here 10 | -------------------------------------------------------------------------------- /test/e2e/testdata/selector-matching/cva-no-selector.yaml: -------------------------------------------------------------------------------- 1 | # ClusterVectorAggregator without selector (matches all pipelines) 2 | apiVersion: observability.kaasops.io/v1alpha1 3 | kind: ClusterVectorAggregator 4 | metadata: 5 | name: no-selector-aggregator 6 | spec: 7 | resourceNamespace: NAMESPACE 8 | image: timberio/vector:0.40.0-alpine 9 | replicas: 1 10 | -------------------------------------------------------------------------------- /config/rbac/metrics_auth_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: metrics-auth-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: metrics-auth-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/samples/kustomization.yaml: -------------------------------------------------------------------------------- 1 | ## Append samples of your project ## 2 | resources: 3 | - observability_v1alpha1_vector.yaml 4 | - observability_v1alpha1_vectorpipeline.yaml 5 | - observability_v1alpha1_clustervectorpipeline.yaml 6 | - observability_v1alpha1_vectoraggregator.yaml 7 | - observability_v1alpha1_clustervectoraggregator.yaml 8 | # +kubebuilder:scaffold:manifestskustomizesamples 9 | -------------------------------------------------------------------------------- /test/e2e/testdata/podmonitor/agent-with-updated-interval.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: Vector 3 | metadata: 4 | name: podmonitor-agent 5 | spec: 6 | agent: 7 | image: timberio/vector:0.40.0-alpine 8 | internalMetrics: true 9 | scrapeInterval: "90s" # Updated from 45s to 90s 10 | scrapeTimeout: "30s" # Updated from 15s to 30s 11 | -------------------------------------------------------------------------------- /config/rbac/metrics_auth_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-auth-role 5 | rules: 6 | - apiGroups: 7 | - authentication.k8s.io 8 | resources: 9 | - tokenreviews 10 | verbs: 11 | - create 12 | - apiGroups: 13 | - authorization.k8s.io 14 | resources: 15 | - subjectaccessreviews 16 | verbs: 17 | - create 18 | -------------------------------------------------------------------------------- /test/e2e/testdata/podmonitor/aggregator-with-scrape-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorAggregator 3 | metadata: 4 | name: podmonitor-aggregator 5 | spec: 6 | image: timberio/vector:0.40.0-alpine 7 | replicas: 1 8 | internalMetrics: true 9 | scrapeInterval: "60s" 10 | scrapeTimeout: "20s" 11 | selector: 12 | matchLabels: 13 | app: test 14 | -------------------------------------------------------------------------------- /test/e2e/testdata/podmonitor/test-pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test-app 5 | labels: 6 | app: test 7 | spec: 8 | containers: 9 | - name: nginx 10 | image: nginx:alpine 11 | command: ["/bin/sh", "-c"] 12 | args: 13 | - | 14 | while true; do 15 | echo "Test log message from test-app" 16 | sleep 5 17 | done 18 | -------------------------------------------------------------------------------- /test/e2e/testdata/podmonitor/cluster-aggregator-with-defaults.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: ClusterVectorAggregator 3 | metadata: 4 | name: podmonitor-cluster-agg-defaults 5 | spec: 6 | resourceNamespace: NAMESPACE 7 | image: timberio/vector:0.40.0-alpine 8 | replicas: 1 9 | internalMetrics: true 10 | selector: 11 | matchLabels: 12 | app: cluster-test 13 | -------------------------------------------------------------------------------- /test/e2e/testdata/selector-matching/cva-matching.yaml: -------------------------------------------------------------------------------- 1 | # ClusterVectorAggregator with selector that MATCHES pipeline labels 2 | apiVersion: observability.kaasops.io/v1alpha1 3 | kind: ClusterVectorAggregator 4 | metadata: 5 | name: matching-aggregator 6 | spec: 7 | resourceNamespace: NAMESPACE 8 | image: timberio/vector:0.40.0-alpine 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | team: platform 13 | -------------------------------------------------------------------------------- /helm/charts/vector-operator/templates/vector.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.vector.enable -}} 2 | apiVersion: observability.kaasops.io/v1alpha1 3 | kind: Vector 4 | metadata: 5 | name: {{ .Values.vector.name }} 6 | namespace: {{ .Release.Namespace }} 7 | spec: 8 | useApiServerCache: {{ .Values.vector.useApiServerCache }} 9 | {{- with .Values.vector.agent }} 10 | agent: 11 | {{ toYaml . | indent 4 }} 12 | {{- end }} 13 | {{- end }} -------------------------------------------------------------------------------- /test/e2e/testdata/podmonitor/pipeline-aggregator-role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorPipeline 3 | metadata: 4 | name: aggregator-test-pipeline 5 | labels: 6 | app: test 7 | spec: 8 | sources: 9 | http_source: 10 | type: http_server 11 | address: "0.0.0.0:8080" 12 | sinks: 13 | blackhole: 14 | type: blackhole 15 | inputs: 16 | - http_source 17 | -------------------------------------------------------------------------------- /test/e2e/testdata/podmonitor/pipeline-without-exporter.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorPipeline 3 | metadata: 4 | name: no-exporter-pipeline 5 | labels: 6 | app: test 7 | spec: 8 | sources: 9 | k8s_logs: 10 | type: kubernetes_logs 11 | extra_label_selector: "app=test" 12 | sinks: 13 | blackhole: 14 | type: blackhole 15 | inputs: 16 | - k8s_logs 17 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/pipeline-basic.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorPipeline 3 | metadata: 4 | name: basic-pipeline 5 | spec: 6 | sources: 7 | kubernetes_logs: 8 | type: kubernetes_logs 9 | extra_label_selector: "app=test-app" 10 | sinks: 11 | console: 12 | type: console 13 | inputs: 14 | - kubernetes_logs 15 | encoding: 16 | codec: json 17 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/pipeline-template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorPipeline 3 | metadata: 4 | name: {{INDEX}} 5 | spec: 6 | sources: 7 | kubernetes_logs: 8 | type: kubernetes_logs 9 | extra_label_selector: "app=test-app" 10 | sinks: 11 | console: 12 | type: console 13 | inputs: 14 | - kubernetes_logs 15 | encoding: 16 | codec: json 17 | -------------------------------------------------------------------------------- /buf.gen.yaml: -------------------------------------------------------------------------------- 1 | version: v2 2 | managed: 3 | enabled: true 4 | override: 5 | - file_option: go_package_prefix 6 | value: github.com/kaasops/vector-operator/internal/vector/gen 7 | plugins: 8 | - remote: buf.build/protocolbuffers/go 9 | out: internal/vector/gen 10 | opt: paths=source_relative 11 | - remote: buf.build/grpc/go 12 | out: internal/vector/gen 13 | opt: paths=source_relative 14 | inputs: 15 | - directory: proto -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/pipeline-deletable.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorPipeline 3 | metadata: 4 | name: deletable-pipeline 5 | spec: 6 | sources: 7 | kubernetes_logs: 8 | type: kubernetes_logs 9 | extra_label_selector: "app=test-app" 10 | sinks: 11 | console: 12 | type: console 13 | inputs: 14 | - kubernetes_logs 15 | encoding: 16 | codec: json 17 | -------------------------------------------------------------------------------- /test/e2e/testdata/podmonitor/cluster-aggregator-with-scrape-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: ClusterVectorAggregator 3 | metadata: 4 | name: podmonitor-cluster-agg 5 | spec: 6 | resourceNamespace: NAMESPACE 7 | image: timberio/vector:0.40.0-alpine 8 | replicas: 1 9 | internalMetrics: true 10 | scrapeInterval: "90s" 11 | scrapeTimeout: "25s" 12 | selector: 13 | matchLabels: 14 | app: cluster-test 15 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: vector-operator 6 | app.kubernetes.io/managed-by: kustomize 7 | name: manager-rolebinding 8 | roleRef: 9 | apiGroup: rbac.authorization.k8s.io 10 | kind: ClusterRole 11 | name: manager-role 12 | subjects: 13 | - kind: ServiceAccount 14 | name: controller-manager 15 | namespace: system 16 | -------------------------------------------------------------------------------- /config/samples/observability_v1alpha1_clustervectorpipeline.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: ClusterVectorPipeline 3 | metadata: 4 | name: clustervectorpipeline-sample 5 | spec: 6 | sources: 7 | test1: 8 | type: "kubernetes_logs" 9 | extra_label_selector: "app!=testdeployment3" 10 | sinks: 11 | test2: 12 | type: "console" 13 | encoding: 14 | codec: "json" 15 | inputs: 16 | - test1 -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: vector-operator 6 | app.kubernetes.io/managed-by: kustomize 7 | name: leader-election-rolebinding 8 | roleRef: 9 | apiGroup: rbac.authorization.k8s.io 10 | kind: Role 11 | name: leader-election-role 12 | subjects: 13 | - kind: ServiceAccount 14 | name: controller-manager 15 | namespace: system 16 | -------------------------------------------------------------------------------- /config/default/metrics_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | app.kubernetes.io/name: vector-operator 7 | app.kubernetes.io/managed-by: kustomize 8 | name: controller-manager-metrics-service 9 | namespace: system 10 | spec: 11 | ports: 12 | - name: https 13 | port: 8443 14 | protocol: TCP 15 | targetPort: 8443 16 | selector: 17 | control-plane: controller-manager 18 | -------------------------------------------------------------------------------- /helm/charts/vector-operator/templates/clustervectorpipeline.yaml: -------------------------------------------------------------------------------- 1 | {{- range $pipeline := .Values.clustervectorpipeline }} 2 | apiVersion: observability.kaasops.io/v1alpha1 3 | kind: ClusterVectorPipeline 4 | metadata: 5 | name: {{ $pipeline.name }} 6 | spec: 7 | sources: 8 | {{- toYaml $pipeline.sources | nindent 4 }} 9 | transforms: 10 | {{- toYaml $pipeline.transforms | nindent 4 }} 11 | sinks: 12 | {{- toYaml $pipeline.sinks | nindent 4 }} 13 | --- 14 | {{- end }} 15 | -------------------------------------------------------------------------------- /helm/charts/vector-operator/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/namespace-isolation-pod-main.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: main-namespace-pod 5 | labels: 6 | app: main-app 7 | spec: 8 | containers: 9 | - name: log-generator 10 | image: busybox:1.36 11 | command: 12 | - sh 13 | - -c 14 | - | 15 | while true; do 16 | echo '{"marker":"MAIN_NAMESPACE","message":"This is from main namespace"}' 17 | sleep 5 18 | done 19 | restartPolicy: Always 20 | -------------------------------------------------------------------------------- /helm/Makefile: -------------------------------------------------------------------------------- 1 | URL=https://kaasops.github.io/vector-operator/helm 2 | HELM_IMAGE = alpine/helm:3.9.1 3 | HELM_DOCS_IMAGE = jnorwood/helm-docs:v1.11.0 4 | KNOWN_TARGETS=helm 5 | 6 | # Run linter for helm chart 7 | lint: 8 | helm lint charts/vector-operator 9 | 10 | # Package chart into zip file 11 | package: 12 | cp ../config/crd/bases/*.yaml charts/vector-operator/crds/ 13 | helm package charts/* -d packages 14 | 15 | # Create index file (use only for initial setup) 16 | index: 17 | helm repo index --url ${URL} . 18 | -------------------------------------------------------------------------------- /test/e2e/testdata/selector-matching/cva-non-matching.yaml: -------------------------------------------------------------------------------- 1 | # ClusterVectorAggregator with selector that does NOT match pipeline labels 2 | # This aggregator requires "team: backend" but our pipeline has "team: platform" 3 | apiVersion: observability.kaasops.io/v1alpha1 4 | kind: ClusterVectorAggregator 5 | metadata: 6 | name: non-matching-aggregator 7 | spec: 8 | resourceNamespace: NAMESPACE 9 | image: timberio/vector:0.40.0-alpine 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | team: backend 14 | -------------------------------------------------------------------------------- /internal/utils/compression/compression.go: -------------------------------------------------------------------------------- 1 | package compression 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | 7 | "github.com/go-logr/logr" 8 | ) 9 | 10 | func Compress(data []byte, log logr.Logger) []byte { 11 | var b bytes.Buffer 12 | gz := gzip.NewWriter(&b) 13 | if _, err := gz.Write(data); err != nil { 14 | log.Error(err, "Failed to compress") 15 | } 16 | 17 | if err := gz.Close(); err != nil { 18 | log.Error(err, "Failed to close writer for compress") 19 | } 20 | 21 | return b.Bytes() 22 | } 23 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/namespace-isolation-pipeline.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorPipeline 3 | metadata: 4 | name: isolated-pipeline 5 | namespace: test-normal-mode-isolated 6 | spec: 7 | sources: 8 | my_namespace_logs: 9 | type: kubernetes_logs 10 | # Should only see logs from test-normal-mode-isolated namespace 11 | sinks: 12 | console: 13 | type: console 14 | inputs: 15 | - my_namespace_logs 16 | encoding: 17 | codec: json 18 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 5m 3 | modules-download-mode: readonly 4 | 5 | linters: 6 | enable: 7 | - gosimple 8 | - govet 9 | - ineffassign 10 | - staticcheck 11 | - unused 12 | - errcheck 13 | - gofmt 14 | - goimports 15 | 16 | linters-settings: 17 | goimports: 18 | local-prefixes: github.com/kaasops/vector-operator 19 | 20 | issues: 21 | exclude-rules: 22 | # Exclude some linters from running on tests files 23 | - path: _test\.go 24 | linters: 25 | - errcheck 26 | -------------------------------------------------------------------------------- /config/samples/observability_v1alpha1_vector.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: Vector 3 | metadata: 4 | name: vector-sample 5 | namespace: vector 6 | spec: 7 | agent: 8 | image: "timberio/vector:0.48.0-debian" 9 | internalMetrics: false 10 | api: 11 | enabled: true 12 | healthcheck: true 13 | configCheck: 14 | disabled: true 15 | resources: 16 | limits: 17 | cpu: 500m 18 | memory: 800Mi 19 | requests: 20 | cpu: 10m 21 | memory: 20Mi 22 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/cluster-pipeline-pod-ns1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: cluster-monitored-pod-1 5 | labels: 6 | app: cluster-test 7 | cluster-monitor: enabled 8 | spec: 9 | containers: 10 | - name: log-generator 11 | image: busybox:1.36 12 | command: 13 | - sh 14 | - -c 15 | - | 16 | while true; do 17 | echo '{"marker":"CLUSTER_MONITORED_NS1","message":"Cluster pipeline test from ns1"}' 18 | sleep 5 19 | done 20 | restartPolicy: Always 21 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/namespace-isolation-pod-isolated.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: isolated-pod 5 | namespace: test-normal-mode-isolated 6 | labels: 7 | app: isolated-app 8 | spec: 9 | containers: 10 | - name: log-generator 11 | image: busybox:1.36 12 | command: 13 | - sh 14 | - -c 15 | - | 16 | while true; do 17 | echo '{"marker":"ISOLATED_NAMESPACE","message":"This is from isolated namespace"}' 18 | sleep 5 19 | done 20 | restartPolicy: Always 21 | -------------------------------------------------------------------------------- /config/rbac/vector_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view vectors. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: vector-operator 7 | app.kubernetes.io/managed-by: kustomize 8 | name: vector-viewer-role 9 | rules: 10 | - apiGroups: 11 | - observability.kaasops.io 12 | resources: 13 | - vectors 14 | verbs: 15 | - get 16 | - list 17 | - watch 18 | - apiGroups: 19 | - observability.kaasops.io 20 | resources: 21 | - vectors/status 22 | verbs: 23 | - get 24 | -------------------------------------------------------------------------------- /scripts/kind-config-ci.yaml: -------------------------------------------------------------------------------- 1 | kind: Cluster 2 | apiVersion: kind.x-k8s.io/v1alpha4 3 | nodes: 4 | - role: control-plane 5 | kubeadmConfigPatches: 6 | - | 7 | kind: InitConfiguration 8 | nodeRegistration: 9 | kubeletExtraArgs: 10 | node-labels: "ingress-ready=true" 11 | # Allow scheduling workloads on control-plane for faster CI 12 | extraPortMappings: 13 | - containerPort: 80 14 | hostPort: 80 15 | protocol: TCP 16 | - containerPort: 443 17 | hostPort: 443 18 | protocol: TCP 19 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/cluster-pipeline-pod-ns2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: cluster-monitored-pod-2 5 | namespace: test-normal-mode-isolated 6 | labels: 7 | app: cluster-test 8 | cluster-monitor: enabled 9 | spec: 10 | containers: 11 | - name: log-generator 12 | image: busybox:1.36 13 | command: 14 | - sh 15 | - -c 16 | - | 17 | while true; do 18 | echo '{"marker":"CLUSTER_MONITORED_NS2","message":"Cluster pipeline test from ns2"}' 19 | sleep 5 20 | done 21 | restartPolicy: Always 22 | -------------------------------------------------------------------------------- /helm/charts/vector-operator/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | {{- $outer := . -}} 2 | {{- range $secret := .Values.secrets }} 3 | {{- with $outer -}} 4 | --- 5 | apiVersion: v1 6 | kind: Secret 7 | metadata: 8 | name: {{ $secret.name }} 9 | namespace: {{ .Release.Namespace }} 10 | labels: 11 | {{- include "chart.labels" . | nindent 4 }} 12 | type: Opaque 13 | data: 14 | {{- range $k, $v := $secret.data }} 15 | {{- if kindIs "string" $v }} 16 | {{ $k }}: {{ $v | b64enc }} 17 | {{- else }} 18 | {{ $k }}: {{ $v | toJson | trim | b64enc }} 19 | {{- end }} 20 | {{- end }} 21 | {{- end }} 22 | {{- end }} 23 | -------------------------------------------------------------------------------- /proto/vector/vector.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package vector; 3 | 4 | import "event.proto"; 5 | 6 | message PushEventsRequest { 7 | repeated event.EventWrapper events = 1; 8 | } 9 | 10 | message PushEventsResponse {} 11 | 12 | enum ServingStatus { 13 | SERVING = 0; 14 | NOT_SERVING = 1; 15 | } 16 | 17 | message HealthCheckRequest {} 18 | 19 | message HealthCheckResponse { 20 | ServingStatus status = 1; 21 | } 22 | 23 | service Vector { 24 | rpc PushEvents(PushEventsRequest) returns (PushEventsResponse) {} 25 | 26 | rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse); 27 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | bin 9 | testbin/* 10 | 11 | # Test binary, build with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Kubernetes Generated files - skip generated files, except for vendored files 18 | 19 | !vendor/**/zz_generated.* 20 | 21 | # editor and IDE paraphernalia 22 | .idea 23 | .kilocode 24 | *.swp 25 | *.swo 26 | *~ 27 | .vscode 28 | __debug_bin 29 | 30 | vendor 31 | 32 | # E2E test results and artifacts 33 | test/e2e/results/ 34 | -------------------------------------------------------------------------------- /config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | version: v1 8 | group: apiextensions.k8s.io 9 | path: spec/conversion/webhook/clientConfig/service/name 10 | 11 | namespace: 12 | - kind: CustomResourceDefinition 13 | version: v1 14 | group: apiextensions.k8s.io 15 | path: spec/conversion/webhook/clientConfig/service/namespace 16 | create: false 17 | 18 | varReference: 19 | - path: metadata/annotations 20 | -------------------------------------------------------------------------------- /config/rbac/vectorpipeline_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view vectorpipelines. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: vector-operator 7 | app.kubernetes.io/managed-by: kustomize 8 | name: vectorpipeline-viewer-role 9 | rules: 10 | - apiGroups: 11 | - observability.kaasops.io 12 | resources: 13 | - vectorpipelines 14 | verbs: 15 | - get 16 | - list 17 | - watch 18 | - apiGroups: 19 | - observability.kaasops.io 20 | resources: 21 | - vectorpipelines/status 22 | verbs: 23 | - get 24 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/test-app-pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test-app 5 | labels: 6 | app: test-app 7 | spec: 8 | containers: 9 | - name: log-generator 10 | image: busybox:1.36 11 | command: 12 | - sh 13 | - -c 14 | - | 15 | while true; do 16 | echo '{"level":"info","message":"Test log from test-app","timestamp":"'$(date -Iseconds)'"}' 17 | echo '{"level":"debug","message":"Debug log should be filtered","timestamp":"'$(date -Iseconds)'"}' 18 | sleep 5 19 | done 20 | restartPolicy: Always 21 | -------------------------------------------------------------------------------- /config/rbac/vectoraggregator_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view vectoraggregators. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: vector-operator 7 | app.kubernetes.io/managed-by: kustomize 8 | name: vectoraggregator-viewer-role 9 | rules: 10 | - apiGroups: 11 | - observability.kaasops.io 12 | resources: 13 | - vectoraggregators 14 | verbs: 15 | - get 16 | - list 17 | - watch 18 | - apiGroups: 19 | - observability.kaasops.io 20 | resources: 21 | - vectoraggregators/status 22 | verbs: 23 | - get 24 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ -------------------------------------------------------------------------------- /config/rbac/vector_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit vectors. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: vector-operator 7 | app.kubernetes.io/managed-by: kustomize 8 | name: vector-editor-role 9 | rules: 10 | - apiGroups: 11 | - observability.kaasops.io 12 | resources: 13 | - vectors 14 | verbs: 15 | - create 16 | - delete 17 | - get 18 | - list 19 | - patch 20 | - update 21 | - watch 22 | - apiGroups: 23 | - observability.kaasops.io 24 | resources: 25 | - vectors/status 26 | verbs: 27 | - get 28 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/pipeline-kubernetes-logs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorPipeline 3 | metadata: 4 | name: k8s-logs-pipeline 5 | spec: 6 | sources: 7 | kubernetes_logs: 8 | type: kubernetes_logs 9 | extra_label_selector: "app=test-app" 10 | transforms: 11 | filter: 12 | type: filter 13 | inputs: 14 | - kubernetes_logs 15 | condition: 16 | type: vrl 17 | source: '.level != "debug"' 18 | sinks: 19 | console: 20 | type: console 21 | inputs: 22 | - filter 23 | encoding: 24 | codec: json 25 | -------------------------------------------------------------------------------- /config/rbac/clustervectorpipeline_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view clustervectorpipelines. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: vector-operator 7 | app.kubernetes.io/managed-by: kustomize 8 | name: clustervectorpipeline-viewer-role 9 | rules: 10 | - apiGroups: 11 | - observability.kaasops.io 12 | resources: 13 | - clustervectorpipelines 14 | verbs: 15 | - get 16 | - list 17 | - watch 18 | - apiGroups: 19 | - observability.kaasops.io 20 | resources: 21 | - clustervectorpipelines/status 22 | verbs: 23 | - get 24 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | ## Documentation 2 | - Quick start [doc](https://github.com/kaasops/vector-operator/blob/main/docs/quick-start.md) 3 | - Design [doc](https://github.com/kaasops/vector-operator/blob/main/docs/design.md) 4 | - Specification [doc](https://github.com/kaasops/vector-operator/blob/main/docs/specification.md) 5 | - Secure credentials [doc](https://github.com/kaasops/vector-operator/blob/main/docs/secure-credential.md) 6 | - Collect logs from file [doc](https://github.com/kaasops/vector-operator/blob/main/docs/logs-from-file.md) 7 | - Collect journald services logs [doc](https://github.com/kaasops/vector-operator/blob/main/docs/journald-logs.md) 8 | -------------------------------------------------------------------------------- /test/e2e/testdata_helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | -------------------------------------------------------------------------------- /config/rbac/clustervectoraggregator_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view clustervectoraggregators. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: vector-operator 7 | app.kubernetes.io/managed-by: kustomize 8 | name: clustervectoraggregator-viewer-role 9 | rules: 10 | - apiGroups: 11 | - observability.kaasops.io 12 | resources: 13 | - clustervectoraggregators 14 | verbs: 15 | - get 16 | - list 17 | - watch 18 | - apiGroups: 19 | - observability.kaasops.io 20 | resources: 21 | - clustervectoraggregators/status 22 | verbs: 23 | - get 24 | -------------------------------------------------------------------------------- /config/rbac/vectorpipeline_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit vectorpipelines. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: vector-operator 7 | app.kubernetes.io/managed-by: kustomize 8 | name: vectorpipeline-editor-role 9 | rules: 10 | - apiGroups: 11 | - observability.kaasops.io 12 | resources: 13 | - vectorpipelines 14 | verbs: 15 | - create 16 | - delete 17 | - get 18 | - list 19 | - patch 20 | - update 21 | - watch 22 | - apiGroups: 23 | - observability.kaasops.io 24 | resources: 25 | - vectorpipelines/status 26 | verbs: 27 | - get 28 | -------------------------------------------------------------------------------- /config/rbac/vectoraggregator_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit vectoraggregators. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: vector-operator 7 | app.kubernetes.io/managed-by: kustomize 8 | name: vectoraggregator-editor-role 9 | rules: 10 | - apiGroups: 11 | - observability.kaasops.io 12 | resources: 13 | - vectoraggregators 14 | verbs: 15 | - create 16 | - delete 17 | - get 18 | - list 19 | - patch 20 | - update 21 | - watch 22 | - apiGroups: 23 | - observability.kaasops.io 24 | resources: 25 | - vectoraggregators/status 26 | verbs: 27 | - get 28 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/cluster-pipeline.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: ClusterVectorPipeline 3 | metadata: 4 | name: cluster-wide-pipeline 5 | spec: 6 | sources: 7 | cluster_logs: 8 | type: kubernetes_logs 9 | extra_label_selector: "cluster-monitor=enabled" 10 | transforms: 11 | add_cluster_info: 12 | type: remap 13 | inputs: 14 | - cluster_logs 15 | source: | 16 | .cluster_pipeline = "cluster-wide-pipeline" 17 | .collected_at = now() 18 | sinks: 19 | console: 20 | type: console 21 | inputs: 22 | - add_cluster_info 23 | encoding: 24 | codec: json 25 | -------------------------------------------------------------------------------- /test/e2e/testdata/selector-matching/cvp-no-labels.yaml: -------------------------------------------------------------------------------- 1 | # ClusterVectorPipeline without labels - should match aggregator with no selector 2 | apiVersion: observability.kaasops.io/v1alpha1 3 | kind: ClusterVectorPipeline 4 | metadata: 5 | name: unlabeled-pipeline 6 | spec: 7 | sources: 8 | logs: 9 | type: kubernetes_logs 10 | extra_label_selector: "app.kubernetes.io/name=vector" 11 | transforms: 12 | parse: 13 | type: remap 14 | inputs: 15 | - logs 16 | source: | 17 | .unlabeled_pipeline = true 18 | sinks: 19 | console: 20 | type: console 21 | inputs: 22 | - parse 23 | encoding: 24 | codec: json 25 | -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/pipeline-aggregator-role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorPipeline 3 | metadata: 4 | name: aggregator-pipeline 5 | labels: 6 | app: test 7 | role: aggregator 8 | annotations: 9 | vector.kaasops.io/role: "Aggregator" 10 | spec: 11 | sources: 12 | vector_source: 13 | type: vector 14 | address: "0.0.0.0:9000" 15 | transforms: 16 | process: 17 | type: remap 18 | inputs: 19 | - vector_source 20 | source: | 21 | .processed = true 22 | sinks: 23 | console: 24 | type: console 25 | inputs: 26 | - process 27 | encoding: 28 | codec: json 29 | -------------------------------------------------------------------------------- /config/rbac/clustervectorpipeline_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit clustervectorpipelines. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: vector-operator 7 | app.kubernetes.io/managed-by: kustomize 8 | name: clustervectorpipeline-editor-role 9 | rules: 10 | - apiGroups: 11 | - observability.kaasops.io 12 | resources: 13 | - clustervectorpipelines 14 | verbs: 15 | - create 16 | - delete 17 | - get 18 | - list 19 | - patch 20 | - update 21 | - watch 22 | - apiGroups: 23 | - observability.kaasops.io 24 | resources: 25 | - clustervectorpipelines/status 26 | verbs: 27 | - get 28 | -------------------------------------------------------------------------------- /config/rbac/clustervectoraggregator_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit clustervectoraggregators. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: vector-operator 7 | app.kubernetes.io/managed-by: kustomize 8 | name: clustervectoraggregator-editor-role 9 | rules: 10 | - apiGroups: 11 | - observability.kaasops.io 12 | resources: 13 | - clustervectoraggregators 14 | verbs: 15 | - create 16 | - delete 17 | - get 18 | - list 19 | - patch 20 | - update 21 | - watch 22 | - apiGroups: 23 | - observability.kaasops.io 24 | resources: 25 | - clustervectoraggregators/status 26 | verbs: 27 | - get 28 | -------------------------------------------------------------------------------- /test/e2e/testdata/selector-matching/cvp-with-labels.yaml: -------------------------------------------------------------------------------- 1 | # ClusterVectorPipeline with labels that match "matching-aggregator" selector 2 | apiVersion: observability.kaasops.io/v1alpha1 3 | kind: ClusterVectorPipeline 4 | metadata: 5 | name: labeled-pipeline 6 | labels: 7 | team: platform 8 | spec: 9 | sources: 10 | logs: 11 | type: kubernetes_logs 12 | extra_label_selector: "app.kubernetes.io/name=vector" 13 | transforms: 14 | parse: 15 | type: remap 16 | inputs: 17 | - logs 18 | source: | 19 | .labeled_pipeline = true 20 | sinks: 21 | console: 22 | type: console 23 | inputs: 24 | - parse 25 | encoding: 26 | codec: json 27 | -------------------------------------------------------------------------------- /test/e2e/testdata/podmonitor/pipeline-with-custom-exporter.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorPipeline 3 | metadata: 4 | name: custom-exporter-pipeline 5 | labels: 6 | app: test 7 | spec: 8 | sources: 9 | k8s_logs: 10 | type: kubernetes_logs 11 | extra_label_selector: "app=test" 12 | transforms: 13 | log_to_metric: 14 | type: log_to_metric 15 | inputs: 16 | - k8s_logs 17 | metrics: 18 | - type: counter 19 | field: message 20 | name: log_lines_total 21 | namespace: custom 22 | sinks: 23 | custom_prom_exporter: 24 | type: prometheus_exporter 25 | inputs: 26 | - log_to_metric 27 | address: "0.0.0.0:9599" 28 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: vector-operator 7 | app.kubernetes.io/managed-by: kustomize 8 | name: leader-election-role 9 | rules: 10 | - apiGroups: 11 | - "" 12 | resources: 13 | - configmaps 14 | verbs: 15 | - get 16 | - list 17 | - watch 18 | - create 19 | - update 20 | - patch 21 | - delete 22 | - apiGroups: 23 | - coordination.k8s.io 24 | resources: 25 | - leases 26 | verbs: 27 | - get 28 | - list 29 | - watch 30 | - create 31 | - update 32 | - patch 33 | - delete 34 | - apiGroups: 35 | - "" 36 | resources: 37 | - events 38 | verbs: 39 | - create 40 | - patch 41 | -------------------------------------------------------------------------------- /internal/utils/hash/hash.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package hash 18 | 19 | import "hash/crc32" 20 | 21 | func Get(input []byte) uint32 { 22 | crc32q := crc32.MakeTable(crc32.IEEE) 23 | 24 | return crc32.Checksum(input, crc32q) 25 | } 26 | -------------------------------------------------------------------------------- /helm/charts/vector-operator/templates/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | 2 | {{- if .Values.rbac.create -}} 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRoleBinding 5 | metadata: 6 | name: {{ template "chart.fullname" . }}-clusterrolebinding 7 | namespace: {{ .Release.Namespace }} 8 | labels: 9 | {{- include "chart.labels" . | nindent 4 }} 10 | {{- with .Values.rbac.extraLabels }} 11 | {{ toYaml . | indent 4}} 12 | {{- end }} 13 | {{- with .Values.rbac.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | roleRef: 18 | apiGroup: rbac.authorization.k8s.io 19 | kind: ClusterRole 20 | name: {{ template "chart.fullname" . }}-clusterrole 21 | subjects: 22 | - kind: ServiceAccount 23 | name: {{ template "chart.serviceAccountName" . }} 24 | namespace: {{ .Release.Namespace }} 25 | {{- end -}} 26 | -------------------------------------------------------------------------------- /helm/charts/vector-operator/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | 2 | {{- if .Values.serviceAccount.create -}} 3 | {{- if semverCompare ">= 1.24" .Capabilities.KubeVersion.Version }} 4 | apiVersion: v1 5 | kind: Secret 6 | metadata: 7 | name: {{ include "chart.serviceAccountName" . }} 8 | namespace: {{ .Release.Namespace }} 9 | annotations: 10 | kubernetes.io/service-account.name: {{ include "chart.serviceAccountName" . }} 11 | type: kubernetes.io/service-account-token 12 | {{- end }} 13 | --- 14 | apiVersion: v1 15 | kind: ServiceAccount 16 | metadata: 17 | name: {{ include "chart.serviceAccountName" . }} 18 | namespace: {{ .Release.Namespace }} 19 | labels: 20 | {{- include "chart.labels" . | nindent 4 }} 21 | {{- with .Values.serviceAccount.annotations }} 22 | annotations: 23 | {{- toYaml . | nindent 4 }} 24 | {{- end }} 25 | {{- end -}} -------------------------------------------------------------------------------- /test/e2e/testdata/normal-mode/pipeline-complex.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: observability.kaasops.io/v1alpha1 2 | kind: VectorPipeline 3 | metadata: 4 | name: complex-pipeline 5 | spec: 6 | sources: 7 | kubernetes_logs: 8 | type: kubernetes_logs 9 | extra_label_selector: "app=test-app" 10 | transforms: 11 | parse: 12 | type: remap 13 | inputs: 14 | - kubernetes_logs 15 | source: | 16 | .parsed = parse_json!(.message) 17 | filter: 18 | type: filter 19 | inputs: 20 | - parse 21 | condition: '.level == "info"' 22 | sinks: 23 | console_all: 24 | type: console 25 | inputs: 26 | - parse 27 | encoding: 28 | codec: json 29 | console_filtered: 30 | type: console 31 | inputs: 32 | - filter 33 | encoding: 34 | codec: text 35 | -------------------------------------------------------------------------------- /internal/config/configcheck/configcheck_error.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package configcheck 18 | 19 | import ( 20 | "errors" 21 | ) 22 | 23 | var ( 24 | ValidationError = errors.New("config validation error") 25 | ConfigcheckTimeoutError = errors.New("timeout waiting configcheck pod result") 26 | ) 27 | -------------------------------------------------------------------------------- /.github/workflows/unit-tests.yaml: -------------------------------------------------------------------------------- 1 | name: Unit Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - master 8 | pull_request: 9 | branches: 10 | - main 11 | - master 12 | workflow_dispatch: 13 | 14 | jobs: 15 | unit-tests: 16 | name: Run Unit Tests 17 | runs-on: ubuntu-latest 18 | timeout-minutes: 10 19 | 20 | steps: 21 | - name: Checkout code 22 | uses: actions/checkout@v4 23 | 24 | - name: Set up Go 25 | uses: actions/setup-go@v5 26 | with: 27 | go-version-file: 'go.mod' 28 | cache: true 29 | 30 | - name: Run unit tests 31 | run: make test 32 | 33 | - name: Upload coverage report 34 | if: always() 35 | uses: actions/upload-artifact@v4 36 | with: 37 | name: coverage-report-${{ github.run_number }} 38 | path: cover.out 39 | retention-days: 7 40 | -------------------------------------------------------------------------------- /internal/evcollector/metrics.go: -------------------------------------------------------------------------------- 1 | package evcollector 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | var ( 9 | eventsHandled = promauto.NewCounterVec(prometheus.CounterOpts{ 10 | Namespace: "event_collector", 11 | Name: "handled_events_total", 12 | Help: "The total number of handled events", 13 | }, []string{"service", "namespace"}) 14 | eventsSkipped = promauto.NewCounterVec(prometheus.CounterOpts{ 15 | Namespace: "event_collector", 16 | Name: "skipped_events_total", 17 | Help: "The total number of skipped events", 18 | }, []string{"service", "namespace"}) 19 | eventsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{ 20 | Namespace: "event_collector", 21 | Name: "processed_events_total", 22 | Help: "The total number of processed events", 23 | }, []string{"service", "namespace"}) 24 | ) 25 | -------------------------------------------------------------------------------- /config/network-policy/allow-metrics-traffic.yaml: -------------------------------------------------------------------------------- 1 | # This NetworkPolicy allows ingress traffic 2 | # with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those 3 | # namespaces are able to gathering data from the metrics endpoint. 4 | apiVersion: networking.k8s.io/v1 5 | kind: NetworkPolicy 6 | metadata: 7 | labels: 8 | app.kubernetes.io/name: vector-operator 9 | app.kubernetes.io/managed-by: kustomize 10 | name: allow-metrics-traffic 11 | namespace: system 12 | spec: 13 | podSelector: 14 | matchLabels: 15 | control-plane: controller-manager 16 | policyTypes: 17 | - Ingress 18 | ingress: 19 | # This allows ingress traffic from any namespace with the label metrics: enabled 20 | - from: 21 | - namespaceSelector: 22 | matchLabels: 23 | metrics: enabled # Only from namespaces with this label 24 | ports: 25 | - port: 8443 26 | protocol: TCP 27 | -------------------------------------------------------------------------------- /internal/config/configcheck/configcheck_rbac.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package configcheck 18 | 19 | import ( 20 | corev1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | ) 23 | 24 | func (cc *ConfigCheck) createVectorConfigCheckServiceAccount() *corev1.ServiceAccount { 25 | labels := cc.labelsForVectorConfigCheck() 26 | 27 | serviceAccount := &corev1.ServiceAccount{ 28 | ObjectMeta: metav1.ObjectMeta{ 29 | Name: "vector-configcheck", 30 | Namespace: cc.Namespace, 31 | Labels: labels, 32 | }, 33 | } 34 | 35 | return serviceAccount 36 | } 37 | -------------------------------------------------------------------------------- /docs/logs-from-file.md: -------------------------------------------------------------------------------- 1 | # Logs from file 2 | 3 | If you want collect logs from file (like k8s-audit logs) you can use example. 4 | 5 | > Type `file` in source block work only in ClusterVectorPipeline. In VectorPipeline can use only `kubernetes_logs` type 6 | 7 | 8 | ```yaml 9 | apiVersion: observability.kaasops.io/v1alpha1 10 | kind: ClusterVectorPipeline 11 | metadata: 12 | labels: 13 | app.kubernetes.io/instance: vector 14 | name: k9s-audit 15 | namespace: vector 16 | spec: 17 | sources: 18 | k8s-audit: 19 | include: 20 | - /var/log/kubernetes/audit/kube-apiserver-audit.log 21 | type: file 22 | transforms: 23 | k8s-audit-transform: 24 | inputs: 25 | - k8s-audit 26 | source: | 27 | . = parse_json!(.message) 28 | 29 | .@timestamp = .stageTimestamp 30 | 31 | type: remap 32 | sinks: 33 | k8s-audit-sink: 34 | auth: 35 | password: ${ELASTIC_PASSWORD} 36 | strategy: basic 37 | user: ${ELASTIC_USER} 38 | bulk: 39 | index: k8s-audit-%Y-%m-%d 40 | endpoint: ${ELASTIC_HOST} 41 | inputs: 42 | - k8s-audit-transform 43 | tls: 44 | verify_certificate: false 45 | type: elasticsearch 46 | ``` 47 | -------------------------------------------------------------------------------- /test/e2e/scripts/README.md: -------------------------------------------------------------------------------- 1 | # E2E Test Scripts 2 | 3 | Utilities for working with e2e test results and test environment. 4 | 5 | ## Available Scripts 6 | 7 | ### generate_report.py 8 | 9 | Generates an interactive HTML pivot grid report from e2e test results. 10 | 11 | **Usage:** 12 | ```bash 13 | # From project root 14 | make test-report 15 | 16 | # Or directly 17 | cd test/e2e/results 18 | python3 ../scripts/generate_report.py 19 | ``` 20 | 21 | **What it does:** 22 | - Scans all `run-*` directories in `test/e2e/results/` 23 | - Parses test metadata and results from each run 24 | - Generates `test_results_report.html` with interactive pivot grid 25 | - Shows test stability across multiple runs (flaky tests, always-failing tests, etc.) 26 | 27 | **Requirements:** 28 | - Python 3.6+ 29 | - Test results in `test/e2e/results/run-YYYY-MM-DD-HHMMSS/` format 30 | 31 | **Output:** 32 | - `test/e2e/results/test_results_report.html` - Interactive HTML report 33 | 34 | ## Adding New Scripts 35 | 36 | When adding new test utilities: 37 | 1. Place the script in this directory 38 | 2. Update this README with usage instructions 39 | 3. Add a Makefile target if appropriate (see `make help`) 40 | 4. Ensure the script has proper error handling and help text 41 | -------------------------------------------------------------------------------- /internal/vector/vectoragent/vectoragent_podmonitor.go: -------------------------------------------------------------------------------- 1 | package vectoragent 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | 6 | monitorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" 7 | ) 8 | 9 | func (ctrl *Controller) createVectorAgentPodMonitor() *monitorv1.PodMonitor { 10 | labels := ctrl.labelsForVectorAgent() 11 | matchLabels := ctrl.matchLabelsForVectorAgent() 12 | annotations := ctrl.annotationsForVectorAgent() 13 | 14 | endpoint := monitorv1.PodMetricsEndpoint{ 15 | Path: "/metrics", 16 | Port: "prom-exporter", 17 | } 18 | 19 | if ctrl.Vector.Spec.Agent.ScrapeInterval != "" { 20 | endpoint.Interval = monitorv1.Duration(ctrl.Vector.Spec.Agent.ScrapeInterval) 21 | } 22 | if ctrl.Vector.Spec.Agent.ScrapeTimeout != "" { 23 | endpoint.ScrapeTimeout = monitorv1.Duration(ctrl.Vector.Spec.Agent.ScrapeTimeout) 24 | } 25 | 26 | podmonitor := &monitorv1.PodMonitor{ 27 | ObjectMeta: ctrl.objectMetaVectorAgent(labels, annotations, ctrl.Vector.Namespace), 28 | Spec: monitorv1.PodMonitorSpec{ 29 | PodMetricsEndpoints: []monitorv1.PodMetricsEndpoint{endpoint}, 30 | Selector: metav1.LabelSelector{ 31 | MatchLabels: matchLabels, 32 | }, 33 | }, 34 | } 35 | 36 | return podmonitor 37 | } 38 | -------------------------------------------------------------------------------- /helm/charts/vector-operator/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: vector-operator 3 | description: A Helm chart to install Vector Operator 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: "0.8.0" 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "v0.4.0" 25 | 26 | home: https://github.com/kaasops/vector-operator 27 | sources: 28 | - https://github.com/kaasops/vector-operator 29 | 30 | -------------------------------------------------------------------------------- /api/v1alpha1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1alpha1 contains API Schema definitions for the observability v1alpha1 API group 18 | // +kubebuilder:object:generate=true 19 | // +groupName=observability.kaasops.io 20 | package v1alpha1 21 | 22 | import ( 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "sigs.k8s.io/controller-runtime/pkg/scheme" 25 | ) 26 | 27 | var ( 28 | // GroupVersion is group version used to register these objects 29 | GroupVersion = schema.GroupVersion{Group: "observability.kaasops.io", Version: "v1alpha1"} 30 | 31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 33 | 34 | // AddToScheme adds the types in this group-version to the given scheme. 35 | AddToScheme = SchemeBuilder.AddToScheme 36 | ) 37 | -------------------------------------------------------------------------------- /test/e2e/framework/resources.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package framework 18 | 19 | import ( 20 | "github.com/kaasops/vector-operator/test/e2e/framework/assertions" 21 | ) 22 | 23 | // Pipeline returns a pipeline resource wrapper for custom matchers 24 | func (f *Framework) Pipeline(name string) *assertions.PipelineResource { 25 | return assertions.NewPipelineResource(f.namespace, name) 26 | } 27 | 28 | // ClusterPipeline returns a cluster-scoped pipeline resource wrapper for custom matchers 29 | func (f *Framework) ClusterPipeline(name string) *assertions.PipelineResource { 30 | return assertions.NewPipelineResource("", name) 31 | } 32 | 33 | // Service returns a service resource wrapper for custom matchers 34 | func (f *Framework) Service(name string) *assertions.ServiceResource { 35 | return assertions.NewServiceResource(f.namespace, name) 36 | } 37 | -------------------------------------------------------------------------------- /internal/vector/aggregator/config.go: -------------------------------------------------------------------------------- 1 | package aggregator 2 | 3 | import ( 4 | "context" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | "sigs.k8s.io/controller-runtime/pkg/log" 8 | 9 | "github.com/kaasops/vector-operator/internal/utils/compression" 10 | "github.com/kaasops/vector-operator/internal/utils/k8s" 11 | ) 12 | 13 | func (ctrl *Controller) ensureVectorAggregatorConfig(ctx context.Context) error { 14 | log := log.FromContext(ctx).WithValues(ctrl.prefix()+"vector-aggregator-secret", ctrl.Name) 15 | log.Info("start Reconcile Vector Aggregator Secret") 16 | 17 | vectorAggregatorSecret, err := ctrl.createVectorAggregatorConfig(ctx) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | return k8s.CreateOrUpdateResource(ctx, vectorAggregatorSecret, ctrl.Client) 23 | } 24 | 25 | func (ctrl *Controller) createVectorAggregatorConfig(ctx context.Context) (*corev1.Secret, error) { 26 | log := log.FromContext(ctx).WithValues(ctrl.prefix()+"vector-aggregator-config", ctrl.Name) 27 | labels := ctrl.labelsForVectorAggregator() 28 | annotations := ctrl.annotationsForVectorAggregator() 29 | data := ctrl.ConfigBytes 30 | 31 | if ctrl.Spec.CompressConfigFile { 32 | data = compression.Compress(ctrl.ConfigBytes, log) 33 | } 34 | config := map[string][]byte{ 35 | "config.json": data, 36 | } 37 | secret := &corev1.Secret{ 38 | ObjectMeta: ctrl.objectMetaVectorAggregator(labels, annotations, ctrl.Namespace), 39 | Data: config, 40 | } 41 | return secret, nil 42 | } 43 | -------------------------------------------------------------------------------- /test/e2e/framework/config/constants.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | // Test labels for selective test execution 20 | const ( 21 | // Execution speed labels 22 | LabelSmoke = "smoke" 23 | LabelFast = "fast" 24 | LabelSlow = "slow" 25 | LabelRegression = "regression" 26 | LabelStress = "stress" 27 | LabelParallel = "parallel" 28 | 29 | // Priority labels (P0 = critical, must always pass) 30 | LabelP0 = "p0" 31 | LabelP1 = "p1" 32 | LabelP2 = "p2" 33 | 34 | // Category labels 35 | LabelSecurity = "security" 36 | LabelConstraint = "constraint" 37 | ) 38 | 39 | // Resource naming suffixes 40 | const ( 41 | AggregatorSuffix = "-aggregator" 42 | AgentSuffix = "-agent" 43 | ) 44 | 45 | // Kubernetes labels 46 | const ( 47 | ComponentLabel = "app.kubernetes.io/component" 48 | AggregatorComponent = "Aggregator" 49 | AgentComponent = "Agent" 50 | ) 51 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | # Prometheus Monitor Service (Metrics) 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: ServiceMonitor 4 | metadata: 5 | labels: 6 | control-plane: controller-manager 7 | app.kubernetes.io/name: vector-operator 8 | app.kubernetes.io/managed-by: kustomize 9 | name: controller-manager-metrics-monitor 10 | namespace: system 11 | spec: 12 | endpoints: 13 | - path: /metrics 14 | port: https # Ensure this is the name of the port that exposes HTTPS metrics 15 | scheme: https 16 | bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token 17 | tlsConfig: 18 | # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables 19 | # certificate verification. This poses a significant security risk by making the system vulnerable to 20 | # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between 21 | # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, 22 | # compromising the integrity and confidentiality of the information. 23 | # Please use the following options for secure configurations: 24 | # caFile: /etc/metrics-certs/ca.crt 25 | # certFile: /etc/metrics-certs/tls.crt 26 | # keyFile: /etc/metrics-certs/tls.key 27 | insecureSkipVerify: true 28 | selector: 29 | matchLabels: 30 | control-plane: controller-manager 31 | -------------------------------------------------------------------------------- /internal/vector/vectoragent/vectoragent_service.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package vectoragent 18 | 19 | import ( 20 | corev1 "k8s.io/api/core/v1" 21 | 22 | "github.com/kaasops/vector-operator/internal/config" 23 | 24 | "k8s.io/apimachinery/pkg/util/intstr" 25 | ) 26 | 27 | func (ctrl *Controller) createVectorAgentService() *corev1.Service { 28 | labels := ctrl.labelsForVectorAgent() 29 | matchLabels := ctrl.matchLabelsForVectorAgent() 30 | annotations := ctrl.annotationsForVectorAgent() 31 | 32 | return &corev1.Service{ 33 | ObjectMeta: ctrl.objectMetaVectorAgent(labels, annotations, ctrl.Vector.Namespace), 34 | Spec: corev1.ServiceSpec{ 35 | Ports: []corev1.ServicePort{ 36 | { 37 | Name: "api", 38 | Protocol: "TCP", 39 | Port: config.AgentApiPort, 40 | TargetPort: intstr.FromInt(config.AgentApiPort), 41 | }, 42 | }, 43 | Selector: matchLabels, 44 | }, 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.22 AS builder 3 | ARG TARGETOS 4 | ARG TARGETARCH 5 | ARG VERSION="dev" 6 | 7 | WORKDIR /workspace 8 | # Copy the Go Modules manifests 9 | COPY go.mod go.mod 10 | COPY go.sum go.sum 11 | # cache deps before building and copying source so that we don't need to re-download as much 12 | # and so that source changes don't invalidate our downloaded layer 13 | RUN go mod download 14 | 15 | # Copy the go source 16 | COPY cmd/manager/main.go cmd/main.go 17 | COPY api/ api/ 18 | COPY internal/ internal/ 19 | 20 | # Build 21 | # the GOARCH has not a default value to allow the binary be built according to the host where the command 22 | # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO 23 | # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, 24 | # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. 25 | RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ 26 | -ldflags="-X github.com/kaasops/vector-operator/internal/buildinfo.Version=${VERSION}" \ 27 | -a -o manager cmd/main.go 28 | 29 | # Use distroless as minimal base image to package the manager binary 30 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 31 | FROM gcr.io/distroless/static:nonroot 32 | WORKDIR / 33 | COPY --from=builder /workspace/manager . 34 | USER 65532:65532 35 | 36 | ENTRYPOINT ["/manager"] 37 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | - bases/observability.kaasops.io_vectors.yaml 6 | - bases/observability.kaasops.io_vectorpipelines.yaml 7 | - bases/observability.kaasops.io_clustervectorpipelines.yaml 8 | - bases/observability.kaasops.io_vectoraggregators.yaml 9 | - bases/observability.kaasops.io_clustervectoraggregators.yaml 10 | # +kubebuilder:scaffold:crdkustomizeresource 11 | 12 | patches: 13 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 14 | # patches here are for enabling the conversion webhook for each CRD 15 | # +kubebuilder:scaffold:crdkustomizewebhookpatch 16 | 17 | # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. 18 | # patches here are for enabling the CA injection for each CRD 19 | #- path: patches/cainjection_in_vectors.yaml 20 | #- path: patches/cainjection_in_vectorpipelines.yaml 21 | #- path: patches/cainjection_in_clustervectorpipelines.yaml 22 | #- path: patches/cainjection_in_vectoraggregators.yaml 23 | #- path: patches/cainjection_in_clustervectoraggregators.yaml 24 | # +kubebuilder:scaffold:crdkustomizecainjectionpatch 25 | 26 | # [WEBHOOK] To enable webhook, uncomment the following section 27 | # the following config is for teaching kustomize how to do kustomization for CRDs. 28 | 29 | configurations: 30 | - kustomizeconfig.yaml 31 | -------------------------------------------------------------------------------- /internal/vector/vectoragent/vectoragent_config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package vectoragent 18 | 19 | import ( 20 | "context" 21 | 22 | corev1 "k8s.io/api/core/v1" 23 | "sigs.k8s.io/controller-runtime/pkg/log" 24 | 25 | "github.com/kaasops/vector-operator/internal/utils/compression" 26 | ) 27 | 28 | func (ctrl *Controller) createVectorAgentConfig(ctx context.Context) (*corev1.Secret, error) { 29 | log := log.FromContext(ctx).WithValues("vector-agent-rbac", ctrl.Vector.Name) 30 | labels := ctrl.labelsForVectorAgent() 31 | annotations := ctrl.annotationsForVectorAgent() 32 | var data = ctrl.ByteConfig 33 | 34 | if ctrl.Vector.Spec.Agent.CompressConfigFile { 35 | data = compression.Compress(data, log) 36 | } 37 | config := map[string][]byte{ 38 | "agent.json": data, 39 | } 40 | secret := &corev1.Secret{ 41 | ObjectMeta: ctrl.objectMetaVectorAgent(labels, annotations, ctrl.Vector.Namespace), 42 | Data: config, 43 | } 44 | 45 | return secret, nil 46 | } 47 | -------------------------------------------------------------------------------- /event_collector.Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.22 AS builder 3 | ARG TARGETOS 4 | ARG TARGETARCH 5 | ARG VERSION="dev" 6 | 7 | WORKDIR /workspace 8 | # Copy the Go Modules manifests 9 | COPY go.mod go.mod 10 | COPY go.sum go.sum 11 | # cache deps before building and copying source so that we don't need to re-download as much 12 | # and so that source changes don't invalidate our downloaded layer 13 | RUN go mod download 14 | 15 | # Copy the go source 16 | COPY cmd/event_collector/main.go cmd/main.go 17 | COPY api/ api/ 18 | COPY internal/ internal/ 19 | 20 | # Build 21 | # the GOARCH has not a default value to allow the binary be built according to the host where the command 22 | # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO 23 | # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, 24 | # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. 25 | RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ 26 | -ldflags="-X github.com/kaasops/vector-operator/internal/buildinfo.Version=${VERSION}" \ 27 | -a -o collector cmd/main.go 28 | 29 | # Use distroless as minimal base image to package the manager binary 30 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 31 | FROM gcr.io/distroless/static:nonroot 32 | WORKDIR / 33 | COPY --from=builder /workspace/collector . 34 | USER 65532:65532 35 | EXPOSE 8080 36 | 37 | ENTRYPOINT ["/collector"] 38 | -------------------------------------------------------------------------------- /internal/utils/hash/hash_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package hash_test 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/require" 23 | 24 | "github.com/kaasops/vector-operator/internal/utils/hash" 25 | ) 26 | 27 | func TestGet(t *testing.T) { 28 | hashCase := func(bytes []byte, want uint32) func(t *testing.T) { 29 | return func(t *testing.T) { 30 | t.Helper() 31 | t.Parallel() 32 | req := require.New(t) 33 | 34 | result := hash.Get(bytes) 35 | req.Equal(result, want) 36 | } 37 | } 38 | 39 | type testCase struct { 40 | name string 41 | bytes []byte 42 | want uint32 43 | } 44 | 45 | testCases := []testCase{ 46 | { 47 | name: "Simple case", 48 | bytes: []byte("test"), 49 | want: uint32(3632233996), 50 | }, 51 | { 52 | name: "Zero case", 53 | bytes: []byte(""), 54 | want: uint32(0), 55 | }, 56 | } 57 | 58 | // t.Parallel() 59 | for _, tc := range testCases { 60 | t.Run(tc.name, hashCase(tc.bytes, tc.want)) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /internal/config/configcheck/configcheck_config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package configcheck 18 | 19 | import ( 20 | "context" 21 | 22 | corev1 "k8s.io/api/core/v1" 23 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | "sigs.k8s.io/controller-runtime/pkg/log" 25 | 26 | "github.com/kaasops/vector-operator/internal/utils/compression" 27 | ) 28 | 29 | func (cc *ConfigCheck) createVectorConfigCheckConfig(ctx context.Context) (*corev1.Secret, error) { 30 | log := log.FromContext(ctx).WithValues("Vector ConfigCheck", cc.Initiator) 31 | labels := cc.labelsForVectorConfigCheck() 32 | var data = cc.Config 33 | 34 | if cc.CompressedConfig { 35 | data = compression.Compress(cc.Config, log) 36 | } 37 | 38 | config := map[string][]byte{ 39 | "config.json": data, 40 | } 41 | 42 | secret := &corev1.Secret{ 43 | ObjectMeta: metav1.ObjectMeta{ 44 | Name: cc.getNameVectorConfigCheck(), 45 | Namespace: cc.Namespace, 46 | Labels: labels, 47 | }, 48 | Data: config, 49 | } 50 | 51 | return secret, nil 52 | } 53 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | golangci-lint: 14 | name: golangci-lint 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout code 18 | uses: actions/checkout@v4 19 | 20 | - name: Set up Go 21 | uses: actions/setup-go@v5 22 | with: 23 | go-version-file: go.mod 24 | cache: true 25 | 26 | - name: Run golangci-lint 27 | uses: golangci/golangci-lint-action@v6 28 | with: 29 | version: v1.64 30 | args: --timeout=5m 31 | 32 | go-fmt: 33 | name: go fmt 34 | runs-on: ubuntu-latest 35 | steps: 36 | - name: Checkout code 37 | uses: actions/checkout@v4 38 | 39 | - name: Set up Go 40 | uses: actions/setup-go@v5 41 | with: 42 | go-version-file: go.mod 43 | cache: true 44 | 45 | - name: Check formatting 46 | run: | 47 | if [ -n "$(gofmt -l .)" ]; then 48 | echo "The following files are not formatted:" 49 | gofmt -l . 50 | exit 1 51 | fi 52 | 53 | go-vet: 54 | name: go vet 55 | runs-on: ubuntu-latest 56 | steps: 57 | - name: Checkout code 58 | uses: actions/checkout@v4 59 | 60 | - name: Set up Go 61 | uses: actions/setup-go@v5 62 | with: 63 | go-version-file: go.mod 64 | cache: true 65 | 66 | - name: Run go vet 67 | run: go vet ./... 68 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | # Code generated by tool. DO NOT EDIT. 2 | # This file is used to track the info used to scaffold your project 3 | # and allow the plugins properly work. 4 | # More info: https://book.kubebuilder.io/reference/project-config.html 5 | domain: kaasops.io 6 | layout: 7 | - go.kubebuilder.io/v4 8 | projectName: vector-operator 9 | repo: github.com/kaasops/vector-operator 10 | resources: 11 | - api: 12 | crdVersion: v1 13 | namespaced: true 14 | controller: true 15 | domain: kaasops.io 16 | group: observability 17 | kind: Vector 18 | path: github.com/kaasops/vector-operator/api/v1alpha1 19 | version: v1alpha1 20 | - api: 21 | crdVersion: v1 22 | namespaced: true 23 | controller: true 24 | domain: kaasops.io 25 | group: observability 26 | kind: VectorPipeline 27 | path: github.com/kaasops/vector-operator/api/v1alpha1 28 | version: v1alpha1 29 | - api: 30 | crdVersion: v1 31 | namespaced: true 32 | domain: kaasops.io 33 | group: observability 34 | kind: ClusterVectorPipeline 35 | path: github.com/kaasops/vector-operator/api/v1alpha1 36 | version: v1alpha1 37 | - api: 38 | crdVersion: v1 39 | namespaced: true 40 | controller: true 41 | domain: kaasops.io 42 | group: observability 43 | kind: VectorAggregator 44 | path: github.com/kaasops/vector-operator/api/v1alpha1 45 | version: v1alpha1 46 | - api: 47 | crdVersion: v1 48 | namespaced: true 49 | controller: true 50 | domain: kaasops.io 51 | group: observability 52 | kind: ClusterVectorAggregator 53 | path: github.com/kaasops/vector-operator/api/v1alpha1 54 | version: v1alpha1 55 | version: "3" 56 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | # All RBAC will be applied under this service account in 3 | # the deployment namespace. You may comment out this resource 4 | # if your manager will use a service account that exists at 5 | # runtime. Be sure to update RoleBinding and ClusterRoleBinding 6 | # subjects if changing service account names. 7 | - service_account.yaml 8 | - role.yaml 9 | - role_binding.yaml 10 | - leader_election_role.yaml 11 | - leader_election_role_binding.yaml 12 | # The following RBAC configurations are used to protect 13 | # the metrics endpoint with authn/authz. These configurations 14 | # ensure that only authorized users and service accounts 15 | # can access the metrics endpoint. Comment the following 16 | # permissions if you want to disable this protection. 17 | # More info: https://book.kubebuilder.io/reference/metrics.html 18 | - metrics_auth_role.yaml 19 | - metrics_auth_role_binding.yaml 20 | - metrics_reader_role.yaml 21 | # For each CRD, "Editor" and "Viewer" roles are scaffolded by 22 | # default, aiding admins in cluster management. Those roles are 23 | # not used by the Project itself. You can comment the following lines 24 | # if you do not want those helpers be installed with your Project. 25 | - clustervectoraggregator_editor_role.yaml 26 | - clustervectoraggregator_viewer_role.yaml 27 | - vectoraggregator_editor_role.yaml 28 | - vectoraggregator_viewer_role.yaml 29 | - clustervectorpipeline_editor_role.yaml 30 | - clustervectorpipeline_viewer_role.yaml 31 | - vectorpipeline_editor_role.yaml 32 | - vectorpipeline_viewer_role.yaml 33 | - vector_editor_role.yaml 34 | - vector_viewer_role.yaml 35 | 36 | -------------------------------------------------------------------------------- /docs/journald-logs.md: -------------------------------------------------------------------------------- 1 | # Secure credential 2 | 3 | If you want collect service journald logs from node you can use example. 4 | 5 | > Type `journald` in source block work only in ClusterVectorPipeline. In VectorPipeline can use only `kubernetes_logs` type 6 | 7 | > If you want collect journald logs, needs to use vector-agent container with journalctl. `timberio/vector:0.48.0-debian` - for example 8 | 9 | 10 | ```yaml 11 | apiVersion: observability.kaasops.io/v1alpha1 12 | kind: ClusterVectorPipeline 13 | metadata: 14 | name: journald 15 | namespace: vector 16 | spec: 17 | sources: 18 | containerd: 19 | include_units: 20 | - containerd 21 | type: journald 22 | kubelet: 23 | include_units: 24 | - kubelet 25 | type: journald 26 | transforms: 27 | containerd-transform: 28 | inputs: 29 | - containerd 30 | source: | 31 | . = parse_regex!(.message, r'^time="(?P