├── .dockerignore ├── .editorconfig ├── .github ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE │ ├── bug-report.yaml │ ├── config.yaml │ └── enhancement.yaml ├── PULL_REQUEST_TEMPLATE.md ├── SECURITY.md ├── dependabot.yml └── workflows │ ├── build.yml │ ├── ci.yml │ └── docker-build.yml ├── .gitignore ├── .golangci.yaml ├── Dockerfile ├── GOVERNANCE.md ├── LICENSE ├── MAINTAINERS.md ├── Makefile ├── README.md ├── charts └── kelemetry │ ├── .helmignore │ ├── Chart.yaml │ ├── templates │ ├── _helpers.yaml │ ├── collector.deployment.yaml │ ├── consumer.deployment.yaml │ ├── consumer.rbac.yaml │ ├── etcd.yaml │ ├── frontend.deployment.yaml │ ├── informers.deployment.yaml │ ├── informers.rbac.yaml │ ├── ingress.yaml │ ├── kubeconfig-literal-secrets.yaml │ ├── scan.deployment.yaml │ ├── scan.rbac.yaml │ └── storage.deployment.yaml │ └── values.yaml ├── cmd └── kelemetry │ ├── logging.go │ ├── pprof.go │ └── run.go ├── dev.docker-compose.yaml ├── docs ├── DEPLOY.md ├── DEV.md ├── QUICK_START.md └── USER_GUIDE.md ├── e2e ├── ancestors │ ├── config.sh │ └── validate.jq ├── deployment │ ├── config.sh │ └── validate.jq ├── lib │ ├── assert.jq │ └── graph.jq └── run-all.sh ├── go.mod ├── go.sum ├── hack ├── audit-kubeconfig.yaml ├── audit-policy.yaml ├── kelemetrix.toml ├── kwok-cluster.yaml ├── kwok-node.yaml ├── local.Dockerfile ├── tfconfig.yaml ├── tracing-config.yaml └── typos.toml ├── images └── trace-view.png ├── main.go ├── pkg ├── aggregator │ ├── aggregator.go │ ├── aggregatorevent │ │ └── event.go │ ├── eventdecorator │ │ ├── decorator.go │ │ └── eventtagger │ │ │ └── eventtagger.go │ ├── linker │ │ ├── job │ │ │ ├── interface.go │ │ │ ├── local │ │ │ │ └── local.go │ │ │ └── worker │ │ │ │ └── worker.go │ │ └── linker.go │ ├── objectspandecorator │ │ ├── decorator.go │ │ └── resourcetagger │ │ │ └── objecttagger.go │ ├── resourcetagger │ │ └── resource_tagger.go │ ├── spancache │ │ ├── etcd │ │ │ ├── etcd.go │ │ │ └── etcd_test.go │ │ ├── interface.go │ │ └── local │ │ │ ├── local.go │ │ │ └── local_test.go │ └── tracer │ │ ├── interface.go │ │ └── otel │ │ └── otel.go ├── annotationlinker │ ├── linker.go │ └── schema.go ├── audit │ ├── consumer │ │ └── consumer.go │ ├── decorator.go │ ├── dump │ │ └── dump.go │ ├── forward │ │ └── forward.go │ ├── message.go │ ├── mq │ │ ├── interface.go │ │ └── local │ │ │ └── local.go │ ├── producer │ │ └── producer.go │ └── webhook │ │ ├── clustername │ │ ├── address │ │ │ └── address.go │ │ └── interface.go │ │ └── webhook.go ├── diff │ ├── api │ │ └── api.go │ ├── cache │ │ ├── etcd │ │ │ └── etcd.go │ │ ├── interface.go │ │ ├── local │ │ │ └── local.go │ │ ├── memory_wrapper.go │ │ └── snapshot_names.go │ ├── cmp │ │ ├── cmp.go │ │ └── cmp_test.go │ ├── controller │ │ └── controller.go │ └── decorator │ │ └── decorator.go ├── event │ └── controller.go ├── filter │ └── filter.go ├── frontend │ ├── backend │ │ ├── interface.go │ │ └── jaeger-storage │ │ │ └── backend.go │ ├── clusterlist │ │ ├── interface.go │ │ └── options │ │ │ └── options.go │ ├── extension │ │ ├── httptrace │ │ │ └── httptrace.go │ │ ├── jaeger-storage │ │ │ └── storage.go │ │ └── provider.go │ ├── http │ │ ├── redirect │ │ │ └── server.go │ │ └── trace │ │ │ └── server.go │ ├── plugin.go │ ├── reader │ │ ├── merge │ │ │ ├── merge.go │ │ │ └── merge_test.go │ │ └── reader.go │ ├── tf │ │ ├── config │ │ │ ├── config.go │ │ │ ├── file │ │ │ │ └── file.go │ │ │ ├── link_selector.go │ │ │ ├── modifier.go │ │ │ └── step.go │ │ ├── defaults │ │ │ ├── modifier │ │ │ │ ├── extension.go │ │ │ │ └── link_selector.go │ │ │ └── step │ │ │ │ ├── collapse_nesting.go │ │ │ │ ├── compact_duration.go │ │ │ │ ├── extract_nesting.go │ │ │ │ ├── group_by_trace_source.go │ │ │ │ ├── object_tags.go │ │ │ │ ├── operation_service.go │ │ │ │ ├── prune_childless.go │ │ │ │ └── prune_tags.go │ │ ├── extension.go │ │ ├── transform.go │ │ └── tree │ │ │ └── tree.go │ └── tracecache │ │ ├── etcd │ │ └── etcd.go │ │ ├── interface.go │ │ └── local │ │ └── local.go ├── http │ └── server.go ├── imports.go ├── k8s │ ├── config │ │ ├── interface.go │ │ └── mapoption │ │ │ └── mapoption.go │ ├── discovery │ │ └── discovery.go │ ├── k8s.go │ ├── k8smock.go │ ├── multileader │ │ ├── elector.go │ │ └── elector_test.go │ └── objectcache │ │ ├── objectcache.go │ │ └── objectcache_test.go ├── kelemetrix │ ├── base_quantity_comp.go │ ├── config │ │ └── config.go │ ├── consumer │ │ ├── consumer.go │ │ └── consumer_test.go │ ├── defaults │ │ ├── quantities │ │ │ ├── mq_latency.go │ │ │ ├── request_count.go │ │ │ └── request_latency.go │ │ └── tags │ │ │ └── tags.go │ └── registry.go ├── manager │ ├── interface.go │ └── mux.go ├── metrics │ ├── interface.go │ ├── mock.go │ ├── noop │ │ └── noop.go │ └── prometheus │ │ └── prometheus.go ├── ownerlinker │ └── linker.go └── util │ ├── cache │ └── ttl.go │ ├── channel │ ├── channel.go │ └── channel_test.go │ ├── errors │ ├── errors.go │ └── errors_test.go │ ├── filter │ └── object_filter.go │ ├── informer │ ├── decaying_informer.go │ ├── prepush_undelta_store.go │ └── swap_map.go │ ├── jaeger │ └── constants.go │ ├── marshal │ └── marshal.go │ ├── object │ ├── key.go │ └── rich.go │ ├── reflect │ └── reflect.go │ ├── semaphore │ ├── semaphore.go │ └── semaphore_test.go │ ├── shutdown │ └── shutdown.go │ └── zconstants │ ├── link.go │ └── zconstants.go ├── quickstart.docker-compose.yaml ├── scan ├── Dockerfile └── main.sh └── tools └── imports.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .idea 3 | output 4 | /local 5 | log 6 | *.log 7 | coverage.out 8 | *.local.yaml 9 | dump*.json 10 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | charset = utf-8 7 | 8 | [*.go] 9 | indent_style = tab 10 | indent_size = 4 11 | 12 | [*.js] 13 | indent_style = space 14 | indent_size = 4 15 | 16 | [*.{yaml,tpl}] 17 | indent_style = space 18 | indent_size = 2 19 | 20 | [*.toml] 21 | indent_style = space 22 | indent_size = 4 23 | 24 | [*.sh] 25 | indent_style = tab 26 | indent_size = 2 27 | 28 | [*.dot] 29 | indent_style = tab 30 | indent_size = 2 31 | 32 | [*.md] 33 | indent_style = space 34 | indent_size = 2 35 | 36 | [Makefile] 37 | indent_style = tab 38 | indent_size = 2 39 | 40 | [*.jq] 41 | indent_style = space 42 | indent_size = 2 43 | -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # ByteDance Contributor Code of Conduct 2 | 3 | Our Pledge 4 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. 5 | 6 | Our Standards 7 | Examples of behavior that contributes to creating a positive environment include: 8 | - Using welcoming and inclusive language 9 | - Being respectful of differing viewpoints and experiences 10 | - Gracefully accepting constructive criticism 11 | - Focusing on what is best for the community 12 | - Showing empathy towards other community members 13 | 14 | Examples of unacceptable behavior by participants include: 15 | - The use of sexualized language or imagery and unwelcome sexual attention or advances 16 | - Trolling, insulting/derogatory comments, and personal or political attacks 17 | - Public or private harassment 18 | - Publishing others’ private information, such as a physical or electronic address, without explicit permission 19 | - Other conduct which could reasonably be considered inappropriate in a professional setting 20 | 21 | Our Responsibilities 22 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 23 | 24 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 25 | 26 | Scope 27 | This Code of Conduct applies within all project spaces, and it also applies when an individual is representing the project or its community in public spaces. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 28 | 29 | Enforcement 30 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at kubewharf.conduct@bytedance.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 31 | 32 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project’s leadership. 33 | 34 | Attribution 35 | This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 36 | 37 | For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq 38 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.yaml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: Report incorrect behavior from Kelemetry 3 | labels: bug 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | If you believe this is a security vulnerability, 9 | do **not** submit a bug report. 10 | Refer to our [security policy](https://github.com/kubewharf/kelemetry/blob/main/SECURITY.md) instead. 11 | - type: textarea 12 | id: repro 13 | attributes: 14 | label: Steps to reproduce 15 | description: | 16 | Please provide the minimal full steps to reproduce the bug. 17 | - type: textarea 18 | id: expect 19 | attributes: 20 | label: Expected behavior 21 | description: | 22 | What did you expect to happen? 23 | - type: textarea 24 | id: actual 25 | attributes: 26 | label: Actual behavior 27 | description: | 28 | What actually happened? 29 | - type: input 30 | id: kelemetry-version 31 | attributes: 32 | label: Kelemetry version 33 | description: | 34 | "latest" is *not* a valid version name. 35 | Please provide the git commit, git tag or docker image tag. 36 | - type: textarea 37 | id: env 38 | attributes: 39 | label: Environment 40 | description: | 41 | If relevant, please provide information about relevant environment and components: 42 | - Kubernetes version and cloud provider 43 | - Jaeger version 44 | - Choice of cache and storage backends, and their relevant version info 45 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yaml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/enhancement.yaml: -------------------------------------------------------------------------------- 1 | name: Featue request 2 | description: Propose new features 3 | labels: enhancement 4 | body: 5 | - type: textarea 6 | id: description 7 | attributes: 8 | label: Description 9 | description: | 10 | What would you like to be added? 11 | - type: textarea 12 | id: justification 13 | attributes: 14 | label: User story 15 | description: | 16 | Why is this needed? 17 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Description 2 | 3 | 4 | 5 | ### Related issues 6 | 7 | 14 | 15 | ### Special notes for your reviewer: 16 | 17 | 18 | -------------------------------------------------------------------------------- /.github/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security policy 2 | 3 | 6 | 7 | ## Reporting a Vulnerability 8 | 9 | **DO NOT report vulnerabilities on the GitHub issue tracker.** 10 | If you believe you discovered a security vulnerability, 11 | please contact us **by email** at . 12 | 13 | ## Fixing a Vulnerability 14 | 15 | **DO NOT submit pull requests that fix vulnerabilities via GitHub.** 16 | If you would like to contribute a fix, 17 | please use [`git format-patch`](https://git-scm.com/docs/git-format-patch) 18 | and submit the patch **by email** at . 19 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: / 5 | schedule: 6 | interval: weekly 7 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | push: 6 | workflow_dispatch: 7 | 8 | jobs: 9 | test: 10 | name: Run unit tests 11 | runs-on: [ubuntu-24.04] 12 | steps: 13 | - uses: actions/checkout@v3 14 | - uses: actions/setup-go@v3 15 | with: 16 | go-version: '1.22.3' 17 | - name: Ensure coverage is reported for every package 18 | run: | 19 | find pkg -type d -print0 | while IFS= read -r -d '' dir; do 20 | go_file=$(ls -1 $dir | grep '\.go$' | grep -v '_test\.go$' | head -n1) 21 | if [[ $go_file ]]; then 22 | package_line=$(grep '^package' ${dir}/${go_file} | head -n1) 23 | echo "${package_line}_test" >$dir/package_coverage_test.go 24 | fi 25 | done 26 | - name: Run unit tests 27 | run: make test 28 | - name: Upload coverage reports to Codecov 29 | uses: codecov/codecov-action@v3 30 | env: 31 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 32 | 33 | lint: 34 | name: golangci-lint 35 | runs-on: [ubuntu-24.04] 36 | steps: 37 | - uses: actions/checkout@v3 38 | - uses: actions/setup-go@v3 39 | with: 40 | go-version: '1.22.3' 41 | - run: rm -r tools 42 | - name: Run lint 43 | uses: golangci/golangci-lint-action@v3 44 | with: 45 | args: --timeout=10m 46 | only-new-issues: true 47 | 48 | typos: 49 | name: Check typos 50 | runs-on: [ubuntu-24.04] 51 | steps: 52 | - uses: actions/checkout@v3 53 | - uses: crate-ci/typos@v1 54 | with: 55 | config: hack/typos.toml 56 | -------------------------------------------------------------------------------- /.github/workflows/docker-build.yml: -------------------------------------------------------------------------------- 1 | name: Publish helm chart 2 | 3 | on: 4 | push: 5 | tags: "*" 6 | branches: ["main"] 7 | 8 | jobs: 9 | publish-scan: 10 | name: Build scan image 11 | runs-on: [ubuntu-24.04] 12 | if: ${{ github.ref_type == 'tag' }} 13 | steps: 14 | - uses: actions/checkout@v3 15 | - name: Docker login for ghcr 16 | uses: docker/login-action@v2 17 | with: 18 | registry: ghcr.io 19 | username: ${{github.actor}} 20 | password: ${{secrets.GITHUB_TOKEN}} 21 | 22 | - name: Set image tag for release 23 | id: tag-name 24 | run: echo "IMAGE_TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT 25 | 26 | - run: docker build -t ghcr.io/kubewharf/kelemetry-scan:${{steps.tag-name.outputs.IMAGE_TAG}} scan 27 | - run: docker push ghcr.io/kubewharf/kelemetry-scan:${{steps.tag-name.outputs.IMAGE_TAG}} 28 | publish-chart: 29 | name: Build helm chart 30 | runs-on: [ubuntu-24.04] 31 | if: ${{ github.ref_type == 'tag' }} 32 | steps: 33 | - uses: actions/checkout@v3 34 | - name: Docker login for ghcr 35 | uses: docker/login-action@v2 36 | with: 37 | registry: ghcr.io 38 | username: ${{github.actor}} 39 | password: ${{secrets.GITHUB_TOKEN}} 40 | 41 | - name: Set image tag for release 42 | id: tag-name 43 | run: echo "IMAGE_TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT 44 | 45 | - run: helm package charts/kelemetry --app-version="${{steps.tag-name.outputs.IMAGE_TAG}}" --version="${{steps.tag-name.outputs.IMAGE_TAG}}" -d output 46 | - run: helm push output/kelemetry-chart-${{steps.tag-name.outputs.IMAGE_TAG}}.tgz oci://ghcr.io/kubewharf 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | /output 3 | log 4 | *.log 5 | coverage.out 6 | *.local.yaml 7 | dump*.json 8 | /depgraph.* 9 | /local 10 | /USAGE.txt 11 | node_modules 12 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 5m 3 | 4 | linters: 5 | enable: 6 | - asasalint 7 | - asciicheck 8 | - bidichk 9 | - bodyclose 10 | - containedctx 11 | - contextcheck 12 | - copyloopvar 13 | - decorder 14 | - dogsled 15 | - dupl 16 | - dupword 17 | - durationcheck 18 | - errchkjson 19 | - errname 20 | - errorlint 21 | - exhaustive 22 | - forbidigo 23 | - gci 24 | - gocheckcompilerdirectives 25 | - goconst 26 | - gofumpt 27 | - goheader 28 | # - gomnd # TODO 29 | - goprintffuncname 30 | - gosec 31 | - grouper 32 | - importas 33 | - interfacebloat 34 | - lll 35 | - maintidx 36 | - makezero 37 | - misspell 38 | - nakedret 39 | - nilerr 40 | - nosprintfhostport 41 | - prealloc 42 | - predeclared 43 | - reassign 44 | - revive 45 | - tenv 46 | - testpackage 47 | - thelper 48 | - tparallel 49 | - unconvert 50 | - unparam 51 | - usestdlibvars 52 | - whitespace 53 | # - wrapcheck # TODO 54 | 55 | issues: 56 | exclude-dirs: 57 | - tools 58 | exclude-rules: 59 | - path: pkg/util/errors 60 | linters: [errorlint] 61 | - path: cmd/kelemetry/pprof.go 62 | linters: [gosec] 63 | linters-settings: 64 | gci: 65 | sections: 66 | - standard 67 | - default 68 | - prefix(github.com/kubewharf/kelemetry) 69 | goheader: 70 | values: 71 | regexp: 72 | YEAR: '20\d\d' 73 | template: |- 74 | Copyright {{ YEAR }} The Kelemetry Authors. 75 | 76 | Licensed under the Apache License, Version 2.0 (the "License"); 77 | you may not use this file except in compliance with the License. 78 | You may obtain a copy of the License at 79 | 80 | http://www.apache.org/licenses/LICENSE-2.0 81 | 82 | Unless required by applicable law or agreed to in writing, software 83 | distributed under the License is distributed on an "AS IS" BASIS, 84 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 85 | See the License for the specific language governing permissions and 86 | limitations under the License. 87 | gomnd: 88 | ignored-numbers: ["2", "200", "300", "400", "500"] 89 | ignored-files: 90 | - 'pkg/jaeger/tf/config/default/default\.go' 91 | ignored-functions: 92 | - 'ctx.AbortWithError' 93 | - '.*Var' 94 | - 'os.OpenFile' 95 | gosec: 96 | excludes: 97 | - G404 # not all code are security-sensitive 98 | importas: 99 | no-unaliased: true 100 | alias: 101 | - pkg: k8s.io/apimachinery/pkg/api/errors 102 | alias: k8serrors 103 | - pkg: k8s.io/apimachinery/pkg/apis/meta/v1 104 | alias: metav1 105 | - pkg: k8s.io/api/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? 106 | alias: ${group}${v1}${v2}${v3} 107 | - pkg: k8s.io/apiserver/pkg/apis/audit/v1 108 | alias: auditv1 109 | - pkg: k8s.io/client-go/kubernetes/typed/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? 110 | alias: ${group}${v1}${v2}${v3}client 111 | - pkg: k8s.io/client-go/informers/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? 112 | alias: ${group}${v1}${v2}${v3}informers 113 | - pkg: k8s.io/client-go/listers/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? 114 | alias: ${group}${v1}${v2}${v3}listers 115 | lll: 116 | line-length: 140 117 | tab-width: 4 118 | nakedret: 119 | max-func-lines: 1 120 | revive: 121 | rules: 122 | - name: var-naming 123 | disabled: true 124 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23.4-alpine AS build 2 | 3 | ARG GOPROXY 4 | ARG GOPRIVATE 5 | ARG GONOPROXY 6 | ARG GONOSUMDB 7 | 8 | RUN mkdir /src 9 | WORKDIR /src 10 | ADD go.mod go.mod 11 | ADD go.sum go.sum 12 | RUN go mod download 13 | 14 | ADD pkg pkg 15 | ADD cmd cmd 16 | ADD main.go main.go 17 | RUN go build -v . 18 | 19 | FROM alpine 20 | COPY --from=build /src/kelemetry /usr/local/bin/kelemetry 21 | 22 | RUN mkdir -p /app/hack 23 | WORKDIR /app 24 | ADD hack/tfconfig.yaml hack/tfconfig.yaml 25 | RUN sed -i 's/127\.0\.0\.1:17272/remote-badger:17271/g' hack/tfconfig.yaml 26 | 27 | ENTRYPOINT ["kelemetry"] 28 | -------------------------------------------------------------------------------- /GOVERNANCE.md: -------------------------------------------------------------------------------- 1 | # Governance 2 | 3 | The governance model adopted in Kelemetry is influenced by many CNCF projects. 4 | 5 | ## Principles 6 | 7 | - **Open**: Kelemetry is open source community. See (TODO: add Contributor License Agreement). 8 | - **Welcoming and respectful**: See [Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). 9 | - **Transparent and accessible**: Work and collaboration should be done in public. 10 | - **Merit**: Ideas and contributions are accepted according to their technical merit 11 | and alignment with project objectives, scope and design principles. 12 | 13 | ## Code of Conduct 14 | 15 | The Kelemetry [Code of Conduct](CODE_OF_CONDUCT.md) is aligned with the CNCF Code of Conduct. 16 | 17 | ## Community MemberShip 18 | 19 | See community membership (TODO: add community membership). 20 | 21 | ## Decision making process 22 | 23 | Decisions are made based on consensus between maintainers. 24 | Proposals and ideas can either be submitted for agreement via a github issue or PR, 25 | or by sending an email to (TODO: register an email for the org). 26 | 27 | In general, we prefer that technical issues and maintainer membership are amicably worked out between the persons involved. 28 | If a dispute cannot be decided independently, get a third-party maintainer (e.g. a mutual contact with some background 29 | on the issue, but not involved in the conflict) to intercede and the final decision will be made. 30 | Decision making process should be transparent to adhere to the principles of Kelemetry project. 31 | 32 | ## Credits 33 | 34 | Some contents in this documents have been borrowed from 35 | [OpenYurt](https://github.com/openyurtio/openyurt/blob/master/GOVERNANCE.md), 36 | [BFE](https://github.com/bfenetworks/bfe/blob/develop/GOVERNANCE.md), 37 | [CoreDNS](https://github.com/coredns/coredns/blob/master/GOVERNANCE.md) 38 | and [Kubernetes governance](https://github.com/kubernetes/community/blob/master/governance.md) projects. 39 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # The Kelemetry Maintainers 2 | 3 | This file lists the maintainers of the Kelemetry project. 4 | The responsibilities of maintainers are listed in the [GOVERNANCE.md](GOVERNANCE.md) file. 5 | 6 | ## Project Maintainers 7 | 8 | | Name | GitHub username | Affiliation | 9 | | :---: | :---: | :---: | 10 | | Chen Xu | [xuchen-xiaoying](https://github.com/xuchen-xiaoying) | ByteDance | 11 | | Jonathan Chan | [SOF3](https://github.com/SOF3) | ByteDance | 12 | | Qingyun Xu | [xuqingyun](https://github.com/xuqingyun) | ByteDance | 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kelemetry: Global control plane tracing for Kubernetes 2 | 3 | ## Overview 4 | 5 | Kelemetry aggregates various data sources including 6 | Kubernetes events, audit log, informers 7 | into the form of traditional tracing, 8 | enabling visualization through Jaeger UI and automatic analysis. 9 | 10 | ![](images/trace-view.png) 11 | 12 | ## Motivation 13 | 14 | As a distributed asynchronous declarative API, 15 | Kubernetes suffers from lower explainability compared to traditional RPC-based services 16 | as there is no clear causal relationship between events; 17 | a change in one object indirectly effects changes in other objects, 18 | posing challenges to understanding and troubleshooting the system. 19 | Past attempts of tracing in Kubernetes were either limited to single components 20 | or excessively intrusive to individual components. 21 | 22 | Kelemetry addresses the problem by associating events of related objects into the same trace. 23 | By recognizing object relations such as OwnerReferences, 24 | related events can be visualized together without prior domain-specific knowledge. 25 | The behavior of various components are recorded on the same timeline 26 | to reconstruct the causal hierarchy of the actual events. 27 | 28 | ## Features 29 | 30 | - [x] Collect audit logs 31 | - [x] Collect controller events (i.e. the "Events" section in `kubectl describe`) 32 | - [x] Record object diff associated with audit logs 33 | - [x] Connect objects based on owner references 34 | - [x] Collect data from custom sources (Plugin API) 35 | - [x] Connect objects with custom rules with multi-cluster support (Plugin API) 36 | - [x] Navigate trace with Jaeger UI and API 37 | - [x] Scalable for multiple large clusters 38 | - [x] Construct tailormade metrics based on audit logs 39 | 40 | ```mermaid 41 | graph TB 42 | kelemetry[Kelemetry] 43 | audit-log[Audit log] --> kelemetry 44 | event[Event] --> kelemetry 45 | watch[Object watch] --> kelemetry 46 | kelemetry ---> |OpenTelemetry protocol| storage[Jaeger storage] 47 | plugin[Kelemetry storage plugin] --> storage 48 | user[User] --> ui[Jaeger UI] --> plugin 49 | ``` 50 | 51 | ## Getting started 52 | 53 | - [Deployment](./docs/DEPLOY.md) for production 54 | - [Quickstart](./docs/QUICK_START.md) for trying out Kelemetry with a test cluster 55 | - [Development setup](./docs/DEV.md) for developing Kelemetry 56 | 57 | ## Contribution/Development 58 | 59 | - [Contributing guidelines](CONTRIBUTING.md) 60 | - [Developer manual](./docs/DEV.md) 61 | - [Security policy](SECURITY.md) 62 | - [User guide](./docs/USER_GUIDE.md) 63 | 64 | ## Code of Conduct 65 | 66 | See [Code of Conduct](CODE_OF_CONDUCT.md). 67 | 68 | ## Community 69 | 70 | - [Discussions](https://github.com/kubewharf/kelemetry/discussions) 71 | 72 | ## License 73 | Kelemetry is licensed under [Apache License 2.0](LICENSE). 74 | -------------------------------------------------------------------------------- /charts/kelemetry/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/kelemetry/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: kelemetry-chart 3 | description: Production setup for Kelemetry 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.2.7 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "v0.2.7" 25 | -------------------------------------------------------------------------------- /charts/kelemetry/templates/collector.deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{.Release.Name}}-collector 6 | labels: {{ include "kelemetry.collector-labels" . }} 7 | spec: 8 | replicas: {{.Values.collector.replicaCount}} 9 | selector: 10 | matchLabels: {{ include "kelemetry.collector-labels" . }} 11 | template: 12 | metadata: 13 | labels: {{ include "kelemetry.collector-labels" . }} 14 | spec: 15 | imagePullSecrets: [ 16 | {{- range .Values.jaegerImages.pullSecrets }} 17 | {{ . | toJson }}, 18 | {{- end }} 19 | ] 20 | {{ include "kelemetry.pod-boilerplate" .Values.collector | nindent 6 }} 21 | containers: 22 | - name: jaeger-collector 23 | {{ include "kelemetry.container-boilerplate" .Values.collector | nindent 10 }} 24 | args: [ 25 | {{- include "kelemetry.storage-options-raw" . | include "kelemetry.yaml-to-args" }} 26 | ] 27 | env: 28 | - name: COLLECTOR_OTLP_ENABLED 29 | value: "true" 30 | image: {{ printf "%s:%s" .Values.jaegerImages.collector.repository .Values.jaegerImages.collector.tag | toJson }} 31 | imagePullPolicy: {{ toJson .Values.jaegerImages.pullPolicy }} 32 | livenessProbe: 33 | httpGet: 34 | path: / 35 | port: admin 36 | readinessProbe: 37 | httpGet: 38 | path: / 39 | port: admin 40 | ports: 41 | - name: collector 42 | containerPort: 4317 43 | - name: admin 44 | containerPort: 14269 45 | --- 46 | apiVersion: v1 47 | kind: Service 48 | metadata: 49 | name: {{.Release.Name}}-collector 50 | labels: {{ include "kelemetry.collector-labels" . }} 51 | spec: 52 | type: ClusterIP 53 | clusterIP: None 54 | ports: 55 | - name: collector 56 | port: 4317 57 | targetPort: collector 58 | selector: {{ include "kelemetry.collector-labels" . }} 59 | -------------------------------------------------------------------------------- /charts/kelemetry/templates/consumer.deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{.Release.Name}}-consumer 6 | labels: {{ include "kelemetry.consumer-labels" . }} 7 | spec: 8 | replicas: {{ include "kelemetry.consumer-replicas" . }} 9 | selector: 10 | matchLabels: {{ include "kelemetry.consumer-labels" . }} 11 | template: 12 | metadata: 13 | labels: {{ include "kelemetry.consumer-labels" . }} 14 | spec: 15 | imagePullSecrets: [ 16 | {{- range .Values.kelemetryImage.pullSecrets }} 17 | {{ . | toJson }}, 18 | {{- end }} 19 | ] 20 | serviceAccountName: {{.Release.Name}}-consumer 21 | {{- include "kelemetry.pod-boilerplate" .Values.consumer | nindent 6 }} 22 | containers: 23 | - name: main 24 | {{- include "kelemetry.container-boilerplate" .Values.consumer | nindent 10 }} 25 | image: {{ printf "%s:%s" .Values.kelemetryImage.repository (.Values.kelemetryImage.tag | default .Chart.AppVersion) | toJson }} 26 | imagePullPolicy: {{ toJson .Values.kelemetryImage.pullPolicy }} 27 | # TODO: add readiness probe 28 | command: [ 29 | "/usr/local/bin/kelemetry", 30 | {{ include "kelemetry.aggregator-options" . }} 31 | {{ include "kelemetry.object-cache-options" . }} 32 | {{ include "kelemetry.in-cluster-config-options" . }} 33 | {{ include "kelemetry.logging-options" .Values.consumer }} 34 | {{ include "kelemetry.kube-options" .Values.consumer }} 35 | {{ include "kelemetry.audit-options" . }} 36 | {{- if .Values.informers.diff.enable }} 37 | {{ include "kelemetry.diff-cache-options" . }} 38 | {{ include "kelemetry.diff-decorator-options" . }} 39 | {{- end }} 40 | ] 41 | ports: [ 42 | {{- include "kelemetry.observe-ports" .Values.consumer }} 43 | {{- if .Values.consumer.source.type | eq "webhook"}} 44 | { 45 | name: webhook, 46 | containerPort: 8080, 47 | }, 48 | {{- end }} 49 | ] 50 | volumeMounts: [ 51 | {{ include "kelemetry.kubeconfig-volume-mounts" . }} 52 | {{ include "kelemetry.audit-volume-mounts" . }} 53 | ] 54 | volumes: [ 55 | {{ include "kelemetry.kubeconfig-volumes" . }} 56 | {{ include "kelemetry.audit-volumes" . }} 57 | ] 58 | {{- if .Values.consumer.source.type | eq "webhook" }} 59 | --- 60 | apiVersion: v1 61 | kind: Service 62 | metadata: 63 | name: {{.Release.Name}}-webhook 64 | labels: {{ include "kelemetry.consumer-labels" . }} 65 | {{- with .Values.consumer.source.webhook.service}} 66 | spec: 67 | type: {{toJson .type}} 68 | {{- if .otherConfig }} 69 | {{- toYaml .otherConfig | nindent 2 }} 70 | {{- end }} 71 | selector: {{ include "kelemetry.consumer-labels" $ }} 72 | ports: 73 | - name: webhook 74 | port: 8080 75 | targetPort: webhook 76 | {{- if .nodePort }} 77 | nodePort: {{toJson .nodePort}} 78 | {{- end }} 79 | {{- end }} 80 | {{- end }} 81 | -------------------------------------------------------------------------------- /charts/kelemetry/templates/consumer.rbac.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{.Release.Name}}-consumer 6 | labels: {{ include "kelemetry.consumer-labels" . }} 7 | --- 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | kind: ClusterRoleBinding 10 | metadata: 11 | name: {{.Release.Name}}-consumer 12 | labels: {{ include "kelemetry.consumer-labels" . }} 13 | roleRef: 14 | apiGroup: rbac.authorization.k8s.io 15 | kind: ClusterRole 16 | name: {{.Release.Name}}-consumer 17 | subjects: 18 | - kind: ServiceAccount 19 | name: {{.Release.Name}}-consumer 20 | namespace: {{.Release.Namespace}} 21 | --- 22 | apiVersion: rbac.authorization.k8s.io/v1 23 | kind: ClusterRole 24 | metadata: 25 | name: {{.Release.Name}}-consumer 26 | labels: {{ include "kelemetry.consumer-labels" . }} 27 | rules: 28 | - apiGroups: ["*"] 29 | resources: ["*"] 30 | verbs: ["get", "list", "watch"] 31 | -------------------------------------------------------------------------------- /charts/kelemetry/templates/etcd.yaml: -------------------------------------------------------------------------------- 1 | {{- if 2 | .Values.diffCache.type | eq "etcd" | and (not .Values.diffCache.etcd.externalEndpoint) 3 | | or (.Values.aggregator.spanCache.type | eq "etcd" | and (not .Values.aggregator.spanCache.etcd.externalEndpoint)) 4 | | or (.Values.frontend.traceCache.type | eq "etcd" | and (not .Values.frontend.traceCache.etcd.externalEndpoint)) 5 | }} 6 | --- 7 | apiVersion: apps/v1 8 | kind: StatefulSet 9 | metadata: 10 | name: {{.Release.Name}}-etcd 11 | labels: {{ include "kelemetry.etcd-labels" . }} 12 | spec: 13 | serviceName: {{.Release.Name}}-etcd 14 | replicas: {{.Values.sharedEtcd.replicaCount}} 15 | selector: 16 | matchLabels: {{ include "kelemetry.etcd-labels" . }} 17 | template: 18 | metadata: 19 | labels: {{ include "kelemetry.etcd-labels" . }} 20 | spec: 21 | imagePullSecrets: [ 22 | {{- range .Values.sharedEtcd.image.pullSecrets }} 23 | {{ . | toJson }}, 24 | {{- end }} 25 | ] 26 | {{- include "kelemetry.pod-boilerplate" .Values.sharedEtcd | nindent 6 }} 27 | containers: 28 | - name: main 29 | {{- include "kelemetry.container-boilerplate" .Values.sharedEtcd | nindent 10 }} 30 | image: {{ printf "%s:%s" .Values.sharedEtcd.image.repository .Values.sharedEtcd.image.tag | toJson }} 31 | imagePullPolicy: {{ toJson .Values.kelemetryImage.pullPolicy }} 32 | env: 33 | - name: POD_NAME 34 | valueFrom: 35 | fieldRef: {fieldPath: metadata.name} 36 | command: 37 | - etcd 38 | - --name=$(POD_NAME) 39 | - --advertise-client-urls=http://$(POD_NAME):2379 40 | - --listen-client-urls=http://0.0.0.0:2379 41 | - --initial-advertise-peer-urls=http://$(POD_NAME):2380 42 | - --listen-peer-urls=http://0.0.0.0:2380 43 | - --initial-cluster-token=etcd-cluster-1 44 | - --initial-cluster={{.Release.Name}}-etcd-0=http://{{.Release.Name}}-etcd-0.{{.Release.Name}}-etcd.{{.Release.Namespace}}.svc:2380 45 | {{- if gt (int .Values.sharedEtcd.replicaCount) 1 }} 46 | {{- range untilStep 1 (int .Values.sharedEtcd.replicaCount) 1 }} 47 | {{- printf ",%s-etcd-%d=http://%s-etcd-%d.%s-etcd.%s.svc:2380" $.Release.Name . $.Release.Name . $.Release.Name $.Release.Namespace }} 48 | {{- end }} 49 | {{- end }} 50 | - --initial-cluster-state=new 51 | - --data-dir=/var/run/etcd/default.etcd 52 | ports: 53 | - containerPort: 2379 54 | name: client 55 | - containerPort: 2380 56 | name: peer 57 | volumeMounts: 58 | - name: data 59 | mountPath: /var/run/etcd 60 | volumes: 61 | - name: data 62 | persistentVolumeClaim: 63 | claimName: data 64 | volumeClaimTemplates: 65 | - metadata: 66 | name: data 67 | labels: {{ include "kelemetry.etcd-labels" . }} 68 | spec: 69 | accessModes: {{toJson .Values.sharedEtcd.storageAccessModes}} 70 | storageClassName: {{toJson .Values.sharedEtcd.storageClassName}} 71 | resources: {{toJson .Values.sharedEtcd.storageResources}} 72 | --- 73 | apiVersion: v1 74 | kind: Service 75 | metadata: 76 | name: {{.Release.Name}}-etcd 77 | labels: {{ include "kelemetry.etcd-labels" . }} 78 | spec: 79 | type: ClusterIP 80 | clusterIP: None 81 | ports: 82 | - name: client 83 | port: 2379 84 | targetPort: client 85 | - name: peer 86 | port: 2380 87 | targetPort: peer 88 | selector: {{ include "kelemetry.etcd-labels" . }} 89 | {{- end }} 90 | -------------------------------------------------------------------------------- /charts/kelemetry/templates/frontend.deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{.Release.Name}}-frontend 6 | labels: {{ include "kelemetry.frontend-labels" . }} 7 | spec: 8 | replicas: {{.Values.frontend.replicaCount}} 9 | selector: 10 | matchLabels: {{ include "kelemetry.frontend-labels" . }} 11 | template: 12 | metadata: 13 | labels: {{ include "kelemetry.frontend-labels" . }} 14 | spec: 15 | imagePullSecrets: [ 16 | {{- range .Values.kelemetryImage.pullSecrets }} 17 | {{ . | toJson }}, 18 | {{- end }} 19 | {{- range .Values.jaegerImages.pullSecrets }} 20 | {{ . | toJson }}, 21 | {{- end }} 22 | ] 23 | {{ include "kelemetry.pod-boilerplate" .Values.frontend | nindent 6 }} 24 | containers: 25 | - name: jaeger-query 26 | {{ include "kelemetry.container-boilerplate" .Values.frontend.jaegerQuery | nindent 10 }} 27 | image: {{ printf "%s:%s" .Values.jaegerImages.query.repository .Values.jaegerImages.query.tag | toJson }} 28 | imagePullPolicy: {{ toJson .Values.jaegerImages.pullPolicy }} 29 | livenessProbe: 30 | httpGet: 31 | path: / 32 | port: admin 33 | readinessProbe: 34 | httpGet: 35 | path: / 36 | port: admin 37 | ports: 38 | - name: query 39 | containerPort: 16686 40 | - name: grpc 41 | containerPort: 16685 42 | - name: admin 43 | containerPort: 16687 44 | env: 45 | - name: GRPC_STORAGE_SERVER 46 | value: localhost:17271 47 | - name: SPAN_STORAGE_TYPE 48 | value: grpc 49 | - name: storage-plugin 50 | {{ include "kelemetry.container-boilerplate" .Values.frontend.storagePlugin | nindent 10 }} 51 | image: {{ printf "%s:%s" .Values.kelemetryImage.repository (.Values.kelemetryImage.tag | default .Chart.AppVersion) | toJson }} 52 | imagePullPolicy: {{ toJson .Values.kelemetryImage.pullPolicy }} 53 | # TODO: add readiness probe 54 | command: [ 55 | "/usr/local/bin/kelemetry", 56 | {{ include "kelemetry.logging-options" .Values.frontend.storagePlugin }} 57 | {{ include "kelemetry.storage-plugin-options" . }} 58 | ] 59 | ports: [ 60 | { 61 | name: storage-grpc, 62 | containerPort: 17271, 63 | }, 64 | { 65 | name: extension, 66 | containerPort: 8080, 67 | }, 68 | {{- include "kelemetry.observe-ports" .Values.frontend.storagePlugin }} 69 | ] 70 | 71 | --- 72 | apiVersion: v1 73 | kind: Service 74 | metadata: 75 | name: {{.Release.Name}}-query 76 | labels: {{ include "kelemetry.frontend-labels" . }} 77 | {{- with .Values.frontend.service }} 78 | spec: 79 | type: {{toJson .type}} 80 | {{- if .otherConfig }} 81 | {{- toYaml .otherConfig | nindent 2 }} 82 | {{- end }} 83 | selector: {{ include "kelemetry.frontend-labels" $ }} 84 | ports: 85 | - name: query 86 | port: 16686 87 | targetPort: query 88 | {{- if .queryNodePort }} 89 | nodePort: {{toJson .queryNodePort}} 90 | {{- end }} 91 | - name: extension 92 | port: 8080 93 | targetPort: extension 94 | {{- if .redirectNodePort }} 95 | nodePort: {{toJson .redirectNodePort}} 96 | {{- end }} 97 | {{- end }} 98 | -------------------------------------------------------------------------------- /charts/kelemetry/templates/informers.deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{.Release.Name}}-informers 6 | labels: {{ include "kelemetry.informers-labels" . }} 7 | spec: 8 | replicas: {{.Values.informers.replicaCount}} 9 | selector: 10 | matchLabels: {{ include "kelemetry.informers-labels" . }} 11 | template: 12 | metadata: 13 | labels: {{ include "kelemetry.informers-labels" . }} 14 | spec: 15 | imagePullSecrets: [ 16 | {{- range .Values.kelemetryImage.pullSecrets }} 17 | {{ . | toJson }}, 18 | {{- end }} 19 | ] 20 | serviceAccountName: {{.Release.Name}}-informers 21 | {{ include "kelemetry.pod-boilerplate" .Values.informers | nindent 6 }} 22 | containers: 23 | - name: main 24 | {{ include "kelemetry.container-boilerplate" .Values.informers | nindent 10 }} 25 | image: {{ printf "%s:%s" .Values.kelemetryImage.repository (.Values.kelemetryImage.tag | default .Chart.AppVersion) | toJson }} 26 | imagePullPolicy: {{ toJson .Values.kelemetryImage.pullPolicy }} 27 | # TODO: add readiness probe 28 | command: [ 29 | "/usr/local/bin/kelemetry", 30 | {{ include "kelemetry.aggregator-options" . }} 31 | {{ include "kelemetry.object-cache-options" . }} 32 | {{ include "kelemetry.in-cluster-config-options" . }} 33 | {{ include "kelemetry.logging-options" .Values.informers }} 34 | {{ include "kelemetry.kube-options" .Values.informers }} 35 | {{- if .Values.informers.diff.enable }} 36 | {{ include "kelemetry.diff-cache-options" . }} 37 | {{ include "kelemetry.diff-controller-options" . }} 38 | {{- end }} 39 | {{- if .Values.informers.event.enable }} 40 | {{ include "kelemetry.event-informer-options" . }} 41 | {{- end }} 42 | ] 43 | ports: [ 44 | {{- include "kelemetry.observe-ports" .Values.informers }} 45 | ] 46 | volumeMounts: [ 47 | {{ include "kelemetry.kubeconfig-volume-mounts" . }} 48 | ] 49 | volumes: [ 50 | {{ include "kelemetry.kubeconfig-volumes" . }} 51 | ] 52 | -------------------------------------------------------------------------------- /charts/kelemetry/templates/informers.rbac.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{.Release.Name}}-informers 6 | labels: {{ include "kelemetry.informers-labels" . }} 7 | --- 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | kind: ClusterRoleBinding 10 | metadata: 11 | name: {{.Release.Name}}-informers 12 | labels: {{ include "kelemetry.informers-labels" . }} 13 | roleRef: 14 | apiGroup: rbac.authorization.k8s.io 15 | kind: ClusterRole 16 | name: {{.Release.Name}}-informers 17 | subjects: 18 | - kind: ServiceAccount 19 | name: {{.Release.Name}}-informers 20 | namespace: {{.Release.Namespace}} 21 | --- 22 | apiVersion: rbac.authorization.k8s.io/v1 23 | kind: ClusterRole 24 | metadata: 25 | name: {{.Release.Name}}-informers 26 | labels: {{ include "kelemetry.informers-labels" . }} 27 | rules: 28 | - apiGroups: ["*"] 29 | resources: ["*"] 30 | verbs: ["get", "list", "watch"] 31 | - apiGroups: [""] 32 | resources: ["configmaps"] 33 | verbs: ["*"] 34 | - apiGroups: ["coordination.k8s.io"] 35 | resources: ["leases"] 36 | verbs: ["*"] 37 | - apiGroups: [""] 38 | resources: ["events"] 39 | verbs: ["*"] 40 | -------------------------------------------------------------------------------- /charts/kelemetry/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled }} 2 | --- 3 | apiVersion: networking.k8s.io/v1 4 | kind: Ingress 5 | metadata: 6 | name: {{.Release.Name}} 7 | labels: {{ include "kelemetry.default-labels-raw" . | fromYaml | toJson }} 8 | spec: 9 | ingressClassName: {{toJson .Values.ingress.className}} 10 | tls: {{toJson .Values.ingress.tls}} 11 | rules: 12 | - http: 13 | paths: 14 | {{- if .Values.consumer.source.type | eq "webhook" }} 15 | - path: /audit 16 | pathType: Prefix 17 | backend: 18 | service: 19 | name: {{.Release.Name}}-webhook 20 | port: 21 | name: webhook 22 | {{- end }} 23 | - path: /redirect 24 | pathType: Exact 25 | backend: 26 | service: 27 | name: {{.Release.Name}}-query 28 | port: 29 | name: extension 30 | - path: /extensions 31 | pathType: Prefix 32 | backend: 33 | service: 34 | name: {{.Release.Name}}-query 35 | port: 36 | name: extension 37 | - path: / 38 | pathType: Prefix 39 | backend: 40 | service: 41 | name: {{.Release.Name}}-query 42 | port: 43 | name: query 44 | {{- if .Values.ingress.host }} 45 | host: {{toJson .Values.ingress.host}} 46 | {{- end }} 47 | {{- end }} 48 | -------------------------------------------------------------------------------- /charts/kelemetry/templates/kubeconfig-literal-secrets.yaml: -------------------------------------------------------------------------------- 1 | {{- range .Values.multiCluster.clusters }} 2 | {{- if .kubeconfig.type | eq "literal" }} 3 | --- 4 | apiVersion: v1 5 | kind: Secret 6 | metadata: 7 | name: {{$.Release.Name}}-cluster-kubeconfig-{{.name}} 8 | data: 9 | kubeconfig: {{ b64enc .kubeconfig.literal | toJson }} 10 | {{- end }} 11 | {{- end }} 12 | -------------------------------------------------------------------------------- /charts/kelemetry/templates/scan.deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.scan}} 2 | --- 3 | apiVersion: apps/v1 4 | kind: StatefulSet 5 | metadata: 6 | name: {{.Release.Name}}-scan 7 | labels: {{ include "kelemetry.scan-labels" . }} 8 | 9 | spec: 10 | serviceName: {{.Release.Name}}-scan 11 | replicas: 1 12 | selector: 13 | matchLabels: {{ include "kelemetry.scan-labels" . }} 14 | template: 15 | metadata: 16 | labels: {{ include "kelemetry.scan-labels" . }} 17 | spec: 18 | imagePullSecrets: [ 19 | {{- range .Values.scanImage.pullSecrets }} 20 | {{ . | toJson }}, 21 | {{- end }} 22 | ] 23 | serviceAccountName: {{.Release.Name}}-scan 24 | {{ include "kelemetry.pod-boilerplate" .Values.informers | nindent 6 }} 25 | containers: 26 | - name: main 27 | {{ include "kelemetry.container-boilerplate" .Values.informers | nindent 10 }} 28 | image: {{ printf "%s:%s" .Values.scanImage.repository (.Values.scanImage.tag | default .Chart.AppVersion) | toJson }} 29 | imagePullPolicy: {{ toJson .Values.scanImage.pullPolicy }} 30 | # TODO: add readiness probe 31 | command: [sleep, infinity] 32 | env: 33 | - name: RELEASE_NAMESPACE 34 | value: {{toJson .Release.Namespace}} 35 | - name: RELEASE_NAME 36 | value: {{toJson .Release.Name}} 37 | {{- end }} 38 | -------------------------------------------------------------------------------- /charts/kelemetry/templates/scan.rbac.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.scan}} 2 | --- 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: {{.Release.Name}}-scan 7 | labels: {{ include "kelemetry.scan-labels" . }} 8 | --- 9 | apiVersion: rbac.authorization.k8s.io/v1 10 | kind: ClusterRoleBinding 11 | metadata: 12 | name: {{.Release.Name}}-scan 13 | labels: {{ include "kelemetry.scan-labels" . }} 14 | roleRef: 15 | apiGroup: rbac.authorization.k8s.io 16 | kind: ClusterRole 17 | name: {{.Release.Name}}-scan 18 | subjects: 19 | - kind: ServiceAccount 20 | name: {{.Release.Name}}-scan 21 | namespace: {{.Release.Namespace}} 22 | --- 23 | apiVersion: rbac.authorization.k8s.io/v1 24 | kind: ClusterRole 25 | metadata: 26 | name: {{.Release.Name}}-scan 27 | labels: {{ include "kelemetry.scan-labels" . }} 28 | rules: 29 | - apiGroups: ["*"] 30 | resources: ["*"] 31 | verbs: ["get", "list", "watch"] 32 | {{- end }} 33 | -------------------------------------------------------------------------------- /cmd/kelemetry/logging.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package kelemetry 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "path/filepath" 21 | "strings" 22 | 23 | "github.com/sirupsen/logrus" 24 | "github.com/spf13/pflag" 25 | 26 | "github.com/kubewharf/kelemetry/pkg/manager" 27 | ) 28 | 29 | type loggingOptions struct { 30 | level string 31 | formatter string 32 | file string 33 | dot string 34 | usage string 35 | } 36 | 37 | func (options *loggingOptions) setup(fs *pflag.FlagSet) { 38 | fs.StringVar(&options.level, "log-level", "debug", "logrus log level") 39 | fs.StringVar(&options.formatter, "log-format", "text", "logrus log format") 40 | fs.StringVar(&options.file, "log-file", "", "logrus log output file (leave empty for stdout)") 41 | 42 | fs.StringVar(&options.dot, "dot", "", "write dependencies as graphviz output") 43 | fs.StringVar(&options.usage, "usage", "", "write command usage to file") 44 | } 45 | 46 | func (options *loggingOptions) execute(logger *logrus.Logger, fs *pflag.FlagSet) error { 47 | logLevel, err := logrus.ParseLevel(options.level) 48 | if err != nil { 49 | return fmt.Errorf("invalid log level %q: %w", options.level, err) 50 | } 51 | logger.SetLevel(logLevel) 52 | 53 | switch options.formatter { 54 | case "text": 55 | logger.SetFormatter(&logrus.TextFormatter{}) 56 | case "json": 57 | logger.SetFormatter(&logrus.JSONFormatter{}) 58 | default: 59 | return fmt.Errorf("invalid log formatter %q", options.formatter) 60 | } 61 | 62 | if options.file != "" { 63 | writer, err := os.OpenFile(options.file, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) 64 | if err != nil { 65 | return fmt.Errorf("cannot open log file: %w", err) 66 | } 67 | 68 | logger.SetOutput(writer) 69 | } 70 | 71 | if options.dot != "" { 72 | file, err := os.Create(options.dot) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | _, err = file.WriteString(manager.Global.Dot()) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | logger.Infof("graphviz written to %s", options.dot) 83 | 84 | os.Exit(0) 85 | } 86 | 87 | if options.usage != "" { 88 | for _, replace := range []struct { 89 | f func() (string, error) 90 | v string 91 | }{ 92 | {f: os.Executable, v: "$EXEC"}, 93 | {f: func() (string, error) { 94 | if exec, err := os.Executable(); err == nil { 95 | return filepath.Dir(exec), nil 96 | } else { 97 | return "", err 98 | } 99 | }, v: "$(dirname $EXEC)"}, 100 | {f: os.Getwd, v: "$PWD"}, 101 | {f: os.UserHomeDir, v: "$HOME"}, 102 | } { 103 | if path, err := replace.f(); err == nil { 104 | fs.VisitAll(func(f *pflag.Flag) { 105 | f.DefValue = strings.ReplaceAll(f.DefValue, path, replace.v) 106 | }) 107 | } 108 | } 109 | 110 | flagUsages := fs.FlagUsages() 111 | 112 | file, err := os.Create(options.usage) 113 | if err != nil { 114 | return err 115 | } 116 | _, err = file.WriteString(flagUsages) 117 | if err != nil { 118 | return err 119 | } 120 | 121 | logger.Infof("options written to %s", options.usage) 122 | 123 | os.Exit(0) 124 | } 125 | 126 | return nil 127 | } 128 | -------------------------------------------------------------------------------- /cmd/kelemetry/pprof.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package kelemetry 16 | 17 | import ( 18 | "context" 19 | "net/http" 20 | _ "net/http/pprof" 21 | 22 | "github.com/sirupsen/logrus" 23 | "github.com/spf13/pflag" 24 | 25 | "github.com/kubewharf/kelemetry/pkg/manager" 26 | "github.com/kubewharf/kelemetry/pkg/util/shutdown" 27 | ) 28 | 29 | func init() { 30 | manager.Global.Provide("pprof", manager.Ptr(&pprofServer{})) 31 | } 32 | 33 | type pprofOptions struct { 34 | enable bool 35 | addr string 36 | } 37 | 38 | func (options *pprofOptions) Setup(fs *pflag.FlagSet) { 39 | fs.BoolVar(&options.enable, "pprof-enable", false, "enable pprof") 40 | fs.StringVar(&options.addr, "pprof-addr", ":6060", "pprof server bind address") 41 | } 42 | 43 | func (options *pprofOptions) EnableFlag() *bool { return &options.enable } 44 | 45 | type pprofServer struct { 46 | options pprofOptions 47 | Logger logrus.FieldLogger 48 | } 49 | 50 | func (server *pprofServer) Options() manager.Options { return &server.options } 51 | 52 | func (server *pprofServer) Init() error { 53 | go func() { 54 | defer shutdown.RecoverPanic(server.Logger) 55 | server.Logger.Error(http.ListenAndServe(server.options.addr, nil)) 56 | }() 57 | 58 | return nil 59 | } 60 | 61 | func (server *pprofServer) Start(ctx context.Context) error { return nil } 62 | 63 | func (server *pprofServer) Close(ctx context.Context) error { return nil } 64 | -------------------------------------------------------------------------------- /cmd/kelemetry/run.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package kelemetry 16 | 17 | import ( 18 | "context" 19 | "errors" 20 | "fmt" 21 | "os" 22 | 23 | "github.com/sirupsen/logrus" 24 | "github.com/spf13/pflag" 25 | "k8s.io/utils/clock" 26 | 27 | "github.com/kubewharf/kelemetry/pkg/manager" 28 | "github.com/kubewharf/kelemetry/pkg/util/shutdown" 29 | ) 30 | 31 | func Main() { 32 | rootLogger := logrus.New() 33 | 34 | if err := Run(rootLogger); err != nil { 35 | rootLogger.Error(err.Error()) 36 | os.Exit(1) 37 | } 38 | } 39 | 40 | func Run(rootLogger *logrus.Logger) error { 41 | manager := manager.Global 42 | ctx, shutdownTrigger := shutdown.ContextWithTrigger(context.Background()) 43 | provideUtils(manager, rootLogger, shutdownTrigger) 44 | 45 | err := manager.Build() 46 | if err != nil { 47 | return fmt.Errorf("cannot initialize components: %w", err) 48 | } 49 | 50 | fs := pflag.NewFlagSet("kelemetry", pflag.ContinueOnError) 51 | 52 | var loggingOptions loggingOptions 53 | loggingOptions.setup(fs) 54 | manager.SetupFlags(fs) 55 | 56 | if err := fs.Parse(os.Args); err != nil { 57 | if errors.Is(err, pflag.ErrHelp) { 58 | return nil 59 | } 60 | 61 | return err 62 | } 63 | 64 | if err := loggingOptions.execute(rootLogger, fs); err != nil { 65 | return err 66 | } 67 | 68 | fs.VisitAll(func(flag *pflag.Flag) { 69 | rootLogger.WithField("name", flag.Name).WithField("value", flag.Value.String()).Info("Option") 70 | }) 71 | 72 | manager.TrimDisabled(rootLogger) 73 | 74 | if err := manager.Init(ctx, rootLogger); err != nil { 75 | return err 76 | } 77 | 78 | shutdownTrigger.SetupSignalHandler() 79 | if err := manager.Start(rootLogger, ctx); err != nil { 80 | return err 81 | } 82 | 83 | rootLogger.Info("Startup complete") 84 | 85 | <-ctx.Done() 86 | rootLogger.Info("Received shutdown signal") 87 | 88 | return manager.Close(ctx, rootLogger) 89 | } 90 | 91 | func provideUtils(m *manager.Manager, logger logrus.FieldLogger, shutdownTrigger *shutdown.ShutdownTrigger) { 92 | m.ProvideUtil(func(ctx *manager.UtilContext) (logrus.FieldLogger, error) { 93 | return logger.WithField("mod", ctx.ComponentName), nil 94 | }) 95 | m.ProvideUtil(func() (*shutdown.ShutdownTrigger, error) { 96 | return shutdownTrigger, nil 97 | }) 98 | m.ProvideUtil(func() (clock.Clock, error) { 99 | return clock.RealClock{}, nil 100 | }) 101 | } 102 | -------------------------------------------------------------------------------- /dev.docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # docker-compose setup for development setup. 2 | # Use quickstart.docker-compose.yaml if you just want to try out Kelemetry. 3 | # Use the helm chart if you want to deploy in production. 4 | services: 5 | # ETCD cache storage, only required if etcd cache is used 6 | etcd: 7 | image: quay.io/coreos/etcd:v3.2 8 | entrypoint: [etcd] 9 | command: 10 | - -name=main 11 | - -advertise-client-urls=http://etcd:2379 12 | - -listen-client-urls=http://0.0.0.0:2379 13 | - -initial-advertise-peer-urls=http://etcd:2380 14 | - -listen-peer-urls=http://0.0.0.0:2380 15 | - -initial-cluster-state=new 16 | - -initial-cluster=main=http://etcd:2380 17 | - -initial-cluster-token=etcd-cluster-1 18 | - -data-dir=/var/run/etcd/default.etcd 19 | volumes: 20 | - etcd:/var/run/etcd/ 21 | ports: 22 | - 2379:2379 23 | restart: always 24 | deploy: 25 | resources: 26 | limits: 27 | memory: 1G 28 | # Web frontend for trace view. 29 | jaeger-query: 30 | image: jaegertracing/jaeger-query:1.65.0 31 | environment: 32 | SPAN_STORAGE_TYPE: grpc 33 | GRPC_STORAGE_SERVER: host.docker.internal:17271 # run on host directly 34 | ports: 35 | - 0.0.0.0:16686:16686 36 | restart: always 37 | # OTLP collector that writes to Badger 38 | jaeger-collector: 39 | image: jaegertracing/jaeger-collector:1.65.0 40 | environment: 41 | COLLECTOR_OTLP_ENABLED: "true" 42 | SPAN_STORAGE_TYPE: grpc 43 | GRPC_STORAGE_SERVER: remote-badger:17271 44 | ports: 45 | - 0.0.0.0:4317:4317 46 | restart: always 47 | # Backend badger storage 48 | # Feel free to override environment.SPAN_STORAGE_TYPE to other storages given the proper configuration. 49 | remote-badger: 50 | image: jaegertracing/jaeger-remote-storage:1.65.0 51 | environment: 52 | SPAN_STORAGE_TYPE: badger 53 | BADGER_EPHEMERAL: "false" 54 | BADGER_DIRECTORY_KEY: /mnt/badger/key 55 | BADGER_DIRECTORY_VALUE: /mnt/badger/data 56 | ports: 57 | - 127.0.0.1:17272:17271 58 | volumes: 59 | - badger:/mnt/badger 60 | restart: always 61 | deploy: 62 | resources: 63 | limits: 64 | memory: 1G 65 | # Web frontend for raw trace database view. 66 | jaeger-query-raw: 67 | image: jaegertracing/jaeger-query:1.65.0 68 | environment: 69 | SPAN_STORAGE_TYPE: grpc 70 | GRPC_STORAGE_SERVER: remote-badger:17271 71 | ports: 72 | - 0.0.0.0:26686:16686 73 | restart: always 74 | 75 | # Hack service to chown badger volume for 10001 (user of jaeger-remote-storage) 76 | chown-badger-volume: 77 | profiles: 78 | - chown-badger-volume 79 | image: jaegertracing/jaeger-remote-storage:1.65.0 80 | user: root 81 | entrypoint: /bin/ash 82 | command: ["-c", "chown 10001:10001 /mnt/badger && sleep infinity"] 83 | volumes: 84 | - badger:/mnt/badger 85 | 86 | volumes: 87 | etcd: {} 88 | badger: {} 89 | -------------------------------------------------------------------------------- /docs/DEPLOY.md: -------------------------------------------------------------------------------- 1 | # Deploying Kelemetry for Production Clusters 2 | 3 | > Note: Due to the variety of cloud providers and cluster management solutions, 4 | > deploying Kelemetry for production might be tricky. 5 | > If you just want to try out the features of Kelemetry, 6 | > follow the [quick start guide](QUICK_START.md) instead, 7 | > which sets up a basic stack locally using Docker. 8 | 9 | To minimize data loss and latency and ensure high availability, 10 | we recommend deploying Kelemetry in 3 separate components: 11 | consumers, informers and storage plugin. 12 | 13 | ```mermaid 14 | graph LR 15 | subgraph sg:self["From Helm chart"] 16 | subgraph sg:k["Kelemetry"] 17 | subgraph sg:informers["Informers (3 instances)"] 18 | event["Event controller (1 leader)"] 19 | diff/controller["Diff controller (2 leaders)"] 20 | end 21 | subgraph sg:consumer["Audit consumer"] 22 | audit/consumer[Audit consumer] 23 | end 24 | subgraph sg:plugin[Storage plugin] 25 | jaeger/transform[Kelemetry storage plugin] 26 | end 27 | end 28 | 29 | subgraph sg:kv["KV store (etcd/redis)"] 30 | diff/cache[Diff cache] 31 | spancache[Span cache] 32 | end 33 | 34 | subgraph sg:jaeger["Jaeger"] 35 | jaeger/query["Jaeger Query UI"] 36 | jaeger/storage["Jaeger storage"] 37 | end 38 | end 39 | 40 | subgraph sg:cloud[From cloud provider] 41 | cloud/mq[Audit message queue] 42 | %% k8s[Kubernetes cluster] 43 | end 44 | 45 | event --> spancache 46 | audit/consumer --> spancache 47 | %% k8s --> cloud/mq 48 | diff/controller --> diff/cache --> audit/consumer 49 | %% audit/consumer --> k8s 50 | %% event --> k8s 51 | %% diff/controller --> k8s 52 | audit/consumer --> jaeger/storage 53 | jaeger/transform --> jaeger/storage 54 | event --> jaeger/storage 55 | jaeger/query --> jaeger/transform 56 | cloud/mq --> audit/consumer 57 | 58 | classDef red fill:#ff8888 59 | class sg:cloud red 60 | classDef green fill:#88ff88 61 | class sg:k green 62 | classDef blue fill:#98daff 63 | class sg:jaeger,sg:kv blue 64 | ``` 65 | 66 | This setup is bundled into a Helm chart. 67 | 68 | ## Steps 69 | 70 | 1. Download [`values.yaml`](/charts/kelemetry/values.yaml) and configure the settings. 71 | 2. Install the chart: `helm install kelemetry oci://ghcr.io/kubewharf/kelemetry-chart --values values.yaml` 72 | 3. If you use an audit webhook directly, remember to 73 | [configure the apiserver](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/#webhook-backend) 74 | to send audit logs to the webhook: 75 | 76 | The default configuration is designed for single-cluster deployment. 77 | For multi-cluster deployment, configure the `sharedEtcd` and `storageBackend` to use a common database. 78 | 79 | ### Troubleshooting 80 | Run `kubectl exec kelemetry-scan-0 -- scan`. 81 | It should report a few key metrics and provide suggestions if the metrics look wrong. 82 | 83 | All containers running the Kelemetry image export Prometheus metrics on the `metrics` (9090) port. 84 | Jaeger containers export Prometheus metrics on the [`admin` port](https://www.jaegertracing.io/docs/latest/deployment/). 85 | You may set up your own monitoring based on the available metrics. 86 | -------------------------------------------------------------------------------- /docs/QUICK_START.md: -------------------------------------------------------------------------------- 1 | # Quick start 2 | 3 | Kelemetry requires setting up the audit webhook for kube-apiserver. 4 | To try out Kelemetry, the easiest way is to create a new test cluster 5 | using the pre-made kwok config we prepared for Kelemetry setup. 6 | 7 | 1. Ensure the prerequisites are available: 8 | - [kwok](https://kwok.sigs.k8s.io) 9 | - [docker-compose](https://docs.docker.com/compose/install/) 10 | 11 | 2. Run the quickstart scripts: 12 | 13 | ```console 14 | $ make kwok quickstart 15 | ``` 16 | 17 | 3. Open in your browser to view the trace output with Jaeger UI. 18 | 19 | 4. Check out what happens when you deploy! 20 | 21 | ```console 22 | $ kubectl --context kwok-tracetest create deployment hello --image=alpine:latest -- sleep infinity 23 | deployment.apps/hello created 24 | 25 | $ kubectl --context kwok-tracetest scale deployment hello --replicas=5 26 | deployment.apps/hello scaled 27 | 28 | $ kubectl --context kwok-tracetest set image deployments hello alpine=alpine:edge 29 | deployment.apps/hello image updated 30 | ``` 31 | 32 | Search `resource=deployments name=hello` in Jaeger UI: 33 | 34 | ![](../images/trace-view.png) 35 | -------------------------------------------------------------------------------- /docs/USER_GUIDE.md: -------------------------------------------------------------------------------- 1 | # User guide 2 | 3 | Kelemetry uses Jaeger UI for trace view. 4 | 5 | ## Trace search 6 | 7 | The first page is for locating a search. 8 | 9 | We override the semantics of some terms in the Jaeger search form: 10 | 11 | ### Display mode 12 | 13 | The "Service" field selects one of the display modes: 14 | 15 | - `tracing` (**recommended**): One object takes one span. 16 | The span hierarchy indicates the ownership relation between objects. 17 | Events are displayed as logs under the object. 18 | - `grouped`: Similar to `tracing`, but each data source displays as a child span. 19 | - `tree`: Similar to `tracing`, except events have their own spans under the object. 20 | Additional information is available in event tags. 21 | - `timeline`: All events are displayed as children of the root object. 22 | 23 | By default, only the trace for a single object is displayed. 24 | More traces are available by configuration: 25 | 26 | - `full tree`: view the full tree from the deepest ancestor 27 | - `ancestors`: include transitive owners 28 | - `children`: include child objects 29 | 30 | ### Cluster 31 | 32 | Kelemetry supports multi-cluster tracing. 33 | The "Operation" field limits the search results to a single cluster. 34 | 35 | ### Tags 36 | 37 | Tags are used to search traces based on the actual object. 38 | Currently, the following tags are supported: 39 | 40 | - `group`: Kubernetes API group, e.g. `apps`. 41 | - `resource`: Kubernetes API resource plural name, e.g. `deployments. 42 | - `namespace`: Namespace of the API object. Empty for cluster-scoped objects. 43 | - `name`: Name of the API object. 44 | 45 | ### Time 46 | 47 | Kelemetry merges and truncates traces based on the time range given in the user input. 48 | Only spans and events within this range are displayed. 49 | Some display modes further truncate the time range to the duration from the earliest to the latest event, 50 | so refer to the "Trace start" timestamp indicated in the trace view page. 51 | 52 | ## Trace view 53 | 54 | Click on a search result in trace view to access it. 55 | If the current half-hour is still in progress, reload the page to load the new data. 56 | 57 | For the recommended `tracing` display mode, 58 | 59 | - The timestamps are relative to the trace start. 60 | - Click on the arrow button on the left to collapse/expand a span. 61 | - Click on the empty space on a span row to reveal details of the span. 62 | - Hover cursor over a black vertical line on the span to reveal the events. 63 | 64 | > Protip: The 3rd to 10th characters in the trace ID (`20000000` in `ff20000000e1e3e7`) is the display mode ID. 65 | > Just edit it in the URL directly to switch to another display mode. 66 | -------------------------------------------------------------------------------- /e2e/ancestors/config.sh: -------------------------------------------------------------------------------- 1 | # script 2 | kubectl create deployment ancestors --image=alpine:3.16 --replicas=2 -- sleep infinity 3 | local first_rs_name=$(kubectl get replicasets --selector app=ancestors -o json | jq -r '.items[0].metadata.name') 4 | sleep 5 5 | kubectl set image deployments ancestors alpine=alpine:3.17 6 | sleep 30 7 | kubectl delete deployments ancestors 8 | 9 | # config 10 | local TEST_DISPLAY_MODE="22000000" # tracing, ancestors only 11 | local TRACE_DISPLAY_NAME="Deployment" 12 | 13 | TRACE_SEARCH_TAGS[cluster]=tracetest 14 | TRACE_SEARCH_TAGS[group]=apps 15 | TRACE_SEARCH_TAGS[resource]=replicasets 16 | TRACE_SEARCH_TAGS[namespace]=default 17 | TRACE_SEARCH_TAGS[name]=$first_rs_name 18 | -------------------------------------------------------------------------------- /e2e/ancestors/validate.jq: -------------------------------------------------------------------------------- 1 | include "assert"; 2 | include "graph"; 3 | 4 | .data[0] 5 | | ._ = ( 6 | # check root 7 | .spans 8 | | root 9 | | .tags | from_entries 10 | | assertEq("root span is deployment"; .resource; "deployments") 11 | | assertEq("root span has the correct name"; .name; "ancestors") 12 | ) | ._ = ( 13 | # root only has one child 14 | .spans 15 | | children(root.spanID) 16 | | assertEq("there is only one child replicaset"; length; 1) 17 | ) 18 | | null 19 | -------------------------------------------------------------------------------- /e2e/deployment/config.sh: -------------------------------------------------------------------------------- 1 | # script 2 | kubectl create deployment demo --image=alpine:3.16 --replicas=2 -- sleep infinity 3 | sleep 5 4 | kubectl scale deployment demo --replicas=4 5 | sleep 5 6 | kubectl set image deployments demo alpine=alpine:3.17 7 | sleep 5 8 | kubectl scale deployment demo --replicas=2 9 | sleep 5 10 | kubectl delete deployment demo 11 | sleep 30 12 | 13 | # config 14 | local TEST_DISPLAY_MODE="21000000" # tracing, full tree 15 | local TRACE_DISPLAY_NAME="Deployment" 16 | 17 | TRACE_SEARCH_TAGS[cluster]=tracetest 18 | TRACE_SEARCH_TAGS[group]=apps 19 | TRACE_SEARCH_TAGS[resource]=deployments 20 | TRACE_SEARCH_TAGS[namespace]=default 21 | TRACE_SEARCH_TAGS[name]=demo 22 | -------------------------------------------------------------------------------- /e2e/deployment/validate.jq: -------------------------------------------------------------------------------- 1 | include "assert"; 2 | include "graph"; 3 | 4 | .data[0] 5 | | ._ = ( 6 | # check tags 7 | .spans 8 | | root 9 | | .tags | from_entries 10 | | assertEq("root span is deployment"; .resource; "deployments") 11 | | assertEq("root span has the correct name"; .name; "demo") 12 | ) | ._ = ( 13 | .spans 14 | | root.logs 15 | | map(.fields |= from_entries) 16 | | assert( 17 | "delete operation was logged (" + (map(.fields.audit) | @json) + ")"; 18 | any(.fields.audit != null and (.fields.audit | endswith(" delete"))) 19 | ) 20 | | assertEq( 21 | "delete operation contains snapshot"; "demo"; 22 | .[] 23 | | select(.fields.audit != null and (.fields.audit | endswith(" delete"))) 24 | | .fields.snapshot 25 | | fromjson 26 | | .metadata.name 27 | ) 28 | | assert( 29 | "status update was logged"; 30 | any(.fields.audit == "kwok-admin update status") 31 | ) 32 | | assert( 33 | "status update contains diff (" + (map(.fields.audit) | @json) + ")"; 34 | map(select(.fields.diff)) 35 | | any(.fields.diff | contains("status.observedGeneration 1 -> 2")) 36 | ) 37 | | assert( 38 | "diff never contains managedFields"; 39 | map(select(.fields.diff)) 40 | | any(.fields.diff | contains("metadata.managedFields")) | not 41 | ) 42 | ) | ._ = ( 43 | .spans 44 | | children(root.spanID) 45 | | assertEq("there are two child replicasets"; length; 2) 46 | ) 47 | | null 48 | -------------------------------------------------------------------------------- /e2e/lib/assert.jq: -------------------------------------------------------------------------------- 1 | def assert(msg; condition): 2 | if condition then . else 3 | error("Failed assertion: " + msg) 4 | end 5 | ; 6 | 7 | def assertEq(msg; left; right): 8 | assert( 9 | msg + " (" + (left | @json) + " != " + (right | @json) + ")"; 10 | left == right 11 | ) 12 | ; 13 | -------------------------------------------------------------------------------- /e2e/lib/graph.jq: -------------------------------------------------------------------------------- 1 | include "assert"; 2 | 3 | def root: 4 | map(select(.references | length == 0)) 5 | | assertEq("only one root span"; length; 1) 6 | | .[0] 7 | ; 8 | 9 | def children($spanId): 10 | map(select(.references | any(.spanID == $spanId))) 11 | ; 12 | -------------------------------------------------------------------------------- /e2e/run-all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | cd $(dirname $0) 6 | 7 | run_test() { 8 | local test_name=$1 9 | 10 | local tmpdir=$(mktemp -d) 11 | 12 | declare -A TRACE_SEARCH_TAGS 13 | declare -a EXTRA_DISPLAY_MODES 14 | set -x 15 | source ${test_name}/config.sh 16 | set +x 17 | 18 | local curl_param_string="" 19 | for curl_param_key in "${!TRACE_SEARCH_TAGS[@]}"; do 20 | local curl_param_value="${TRACE_SEARCH_TAGS[$curl_param_key]}" 21 | curl_param_string="${curl_param_string} --data-urlencode ${curl_param_key}=${curl_param_value}" 22 | done 23 | 24 | curl -i -G --data-urlencode ts="$(date --iso-8601=seconds)" \ 25 | ${curl_param_string} \ 26 | -o ${tmpdir}/curl-output.http \ 27 | localhost:8080/redirect 28 | 29 | if ! (head -n1 ${tmpdir}/curl-output.http | grep '302 Found'); then 30 | echo "Trace not found for the parameters" 31 | cat curl-output.http 32 | exit 1 33 | fi 34 | 35 | local full_trace_id=$(grep -P "^Location: /trace/" ${tmpdir}/curl-output.http | cut -d/ -f3 | tr -d '\r') 36 | if [[ -z $full_trace_id ]]; then 37 | echo "Trace not found for the parameters" 38 | cat curl-output.http 39 | exit 1 40 | fi 41 | local fixed_id=${full_trace_id:10} 42 | 43 | if [[ -v OUTPUT_TRACE ]]; then 44 | test -d ${OUTPUT_TRACE}/api/traces || mkdir -p ${OUTPUT_TRACE}/api/traces || true 45 | test -d ${OUTPUT_TRACE}/trace-${test_name} || mkdir ${OUTPUT_TRACE}/trace-${test_name} 46 | echo "${TRACE_DISPLAY_NAME}" >${OUTPUT_TRACE}/trace-${test_name}/trace_display_name 47 | echo ${fixed_id} >${OUTPUT_TRACE}/trace-${test_name}/trace_id 48 | 49 | DISPLAY_MODES=($TEST_DISPLAY_MODE ${EXTRA_DISPLAY_MODES[@]}) 50 | 51 | for mode in ${DISPLAY_MODES}; do 52 | local mode_trace_id=ff${mode}${fixed_id} 53 | curl -o ${OUTPUT_TRACE}/api/traces/${mode_trace_id} localhost:16686/api/traces/${mode_trace_id} 54 | done 55 | fi 56 | 57 | local test_mode_trace_id=ff${TEST_DISPLAY_MODE}${fixed_id} 58 | curl -o ${tmpdir}/${test_mode_trace_id}.json localhost:16686/api/traces/${test_mode_trace_id} 59 | go run github.com/itchyny/gojq/cmd/gojq -L lib -f ${test_name}/validate.jq ${tmpdir}/${test_mode_trace_id}.json 60 | } 61 | 62 | declare -A pids 63 | 64 | for sh in */validate.jq; do 65 | test_name=$(basename $(dirname $sh)) 66 | if [[ ! -v FILTER_TESTS ]] || [[ $FILTER_TESTS == *$test_name* ]]; then 67 | run_test ${test_name} & 68 | pids[$test_name]=$! 69 | fi 70 | done 71 | 72 | failed_tests=() 73 | for test_name in ${!pids[@]}; do 74 | wait ${pids[$test_name]} || failed_tests+=($test_name) 75 | done 76 | if [[ ${#failed_tests[@]} -gt 0 ]]; then 77 | echo "Tests failed: ${failed_tests[@]}" 78 | exit 1 79 | fi 80 | -------------------------------------------------------------------------------- /hack/audit-kubeconfig.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | clusters: 3 | - cluster: 4 | server: http://host.docker.internal:8080/audit/tracetest 5 | name: audit 6 | contexts: 7 | - context: 8 | cluster: audit 9 | name: main 10 | current-context: main 11 | kind: Config 12 | preferences: {} 13 | -------------------------------------------------------------------------------- /hack/audit-policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: audit.k8s.io/v1 2 | kind: Policy 3 | rules: 4 | - level: RequestResponse 5 | -------------------------------------------------------------------------------- /hack/kelemetrix.toml: -------------------------------------------------------------------------------- 1 | [[metrics]] 2 | name = "baseline" 3 | quantifier = "mq_latency" 4 | tags = ["cluster"] 5 | 6 | 7 | [[metrics]] 8 | name = "errors_by_user" 9 | quantifier = "request_count" 10 | tags = ["username", "cluster", "group", "resource", "verb", "code", "errorReason"] 11 | [[metrics.tagFilters]] 12 | tag = "code" 13 | oneOf = ["[45].."] 14 | isRegex = true 15 | 16 | [[metrics]] 17 | name = "cache_penetrations" 18 | quantifier = "request_count" 19 | tags = ["username", "cluster", "group", "resource", "verb", "hasSelector"] 20 | [[metrics.tagFilters]] 21 | tag = "code" 22 | oneOf = ["200"] 23 | [[metrics.tagFilters]] 24 | tag = "resourceVersion" 25 | oneOf = ["Empty"] 26 | [[metrics.tagFilters]] 27 | tag = "verb" 28 | oneOf = ["list", "get"] 29 | 30 | [[metrics]] 31 | name = "list_timeout" 32 | quantifier = "request_latency_ratio" 33 | tags = ["username", "cluster", "group", "resource", "hasSelector"] 34 | [[metrics.tagFilters]] 35 | tag = "code" 36 | oneOf = ["200"] 37 | [[metrics.tagFilters]] 38 | tag = "verb" 39 | oneOf = ["list"] 40 | [[metrics.quantityFilters]] 41 | quantity = "request_latency_ratio" 42 | operator = "<" 43 | threshold = 1.0 44 | # E.g. if timeout is 60s, a list timeout would take >60s, so ratio is <1 45 | 46 | [[metrics]] 47 | name = "watch_error" 48 | quantifier = "request_latency_ratio" 49 | tags = ["username", "cluster", "group", "resource"] 50 | [[metrics.tagFilters]] 51 | tag = "code" 52 | oneOf = ["200"] 53 | [[metrics.tagFilters]] 54 | tag = "verb" 55 | oneOf = ["watch"] 56 | [[metrics.quantityFilters]] 57 | quantity = "request_latency_ratio" 58 | operator = ">" 59 | threshold = 100.0 60 | # client-go default watch timeout is [300s, 600s]: https://github.com/kubernetes/client-go/blob/v0.27.1/tools/cache/reflector.go#L155 61 | # if a watch request is rejected, it should terminate in less than one second. 62 | 63 | -------------------------------------------------------------------------------- /hack/kwok-cluster.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: config.kwok.x-k8s.io/v1alpha1 3 | kind: KwokConfiguration 4 | options: 5 | --- 6 | apiVersion: config.kwok.x-k8s.io/v1alpha1 7 | kind: KwokctlConfiguration 8 | componentsPatches: 9 | - name: kube-apiserver 10 | extraArgs: 11 | - key: audit-webhook-config-file 12 | value: /etc/kelemetry/audit-kubeconfig.local.yaml 13 | - key: audit-webhook-batch-max-wait 14 | value: 2s # speed up event consumption during test 15 | - key: tracing-config-file 16 | value: /etc/kelemetry/tracing-config.local.yaml 17 | - key: audit-policy-file 18 | value: /etc/kelemetry/audit-policy.yaml 19 | extraVolumes: 20 | - name: kelemetry-assets 21 | readOnly: true 22 | hostPath: . 23 | mountPath: /etc/kelemetry 24 | pathType: Directory 25 | -------------------------------------------------------------------------------- /hack/kwok-node.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Node 3 | metadata: 4 | annotations: 5 | node.alpha.kubernetes.io/ttl: "0" 6 | kwok.x-k8s.io/node: fake 7 | labels: 8 | beta.kubernetes.io/arch: amd64 9 | beta.kubernetes.io/os: linux 10 | kubernetes.io/arch: amd64 11 | kubernetes.io/hostname: kwok-node-0 12 | kubernetes.io/os: linux 13 | kubernetes.io/role: agent 14 | node-role.kubernetes.io/agent: "" 15 | type: kwok 16 | name: kwok-node-0 17 | spec: 18 | taints: # Avoid scheduling actual running pods to fake Node 19 | - effect: NoSchedule 20 | key: kwok.x-k8s.io/node 21 | value: fake 22 | status: 23 | allocatable: 24 | cpu: 32 25 | memory: 256Gi 26 | pods: 110 27 | capacity: 28 | cpu: 32 29 | memory: 256Gi 30 | pods: 110 31 | nodeInfo: 32 | architecture: amd64 33 | bootID: "" 34 | containerRuntimeVersion: "" 35 | kernelVersion: "" 36 | kubeProxyVersion: fake 37 | kubeletVersion: fake 38 | machineID: "" 39 | operatingSystem: linux 40 | osImage: "" 41 | systemUUID: "" 42 | phase: Running 43 | -------------------------------------------------------------------------------- /hack/local.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian 2 | 3 | ARG BIN_FILE 4 | ARG TFCONFIG 5 | 6 | ADD $BIN_FILE /usr/local/bin/kelemetry 7 | 8 | RUN mkdir -p /app/hack 9 | WORKDIR /app 10 | ADD $TFCONFIG hack/tfconfig.yaml 11 | 12 | ENTRYPOINT ["kelemetry"] 13 | -------------------------------------------------------------------------------- /hack/tracing-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiserver.config.k8s.io/v1beta1 2 | kind: TracingConfiguration 3 | endpoint: host.docker.internal:4317 4 | samplingRatePerMillion: 1000000 5 | -------------------------------------------------------------------------------- /hack/typos.toml: -------------------------------------------------------------------------------- 1 | [files] 2 | extend-exclude = ["go.mod", "go.sum"] 3 | 4 | [default.extend-identifiers] 5 | O_WRONLY = "O_WRONLY" 6 | -------------------------------------------------------------------------------- /images/trace-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubewharf/kelemetry/60aa685587a9ea928eb8066f095566e976cdc5a6/images/trace-view.png -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/kubewharf/kelemetry/cmd/kelemetry" 19 | _ "github.com/kubewharf/kelemetry/pkg" // include default packages 20 | ) 21 | 22 | func main() { 23 | kelemetry.Main() 24 | } 25 | -------------------------------------------------------------------------------- /pkg/aggregator/aggregatorevent/event.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package aggregatorevent 16 | 17 | import ( 18 | "time" 19 | 20 | "github.com/kubewharf/kelemetry/pkg/aggregator/tracer" 21 | "github.com/kubewharf/kelemetry/pkg/util/zconstants" 22 | ) 23 | 24 | type Event struct { 25 | Title string 26 | Time time.Time 27 | EndTime *time.Time 28 | TraceSource string 29 | Tags map[string]any 30 | Logs []tracer.Log 31 | } 32 | 33 | func NewEvent(title string, when time.Time, traceSource string) *Event { 34 | return &Event{ 35 | Title: title, 36 | Time: when, 37 | TraceSource: traceSource, 38 | Tags: map[string]any{}, 39 | Logs: []tracer.Log{}, 40 | } 41 | } 42 | 43 | func (event *Event) GetEndTime() time.Time { 44 | if event.EndTime != nil { 45 | return *event.EndTime 46 | } 47 | 48 | return event.Time.Add(zconstants.DummyDuration) 49 | } 50 | 51 | func (event *Event) SetEndTime(when time.Time) *Event { 52 | event.EndTime = &when 53 | return event 54 | } 55 | 56 | func (event *Event) SetDuration(duration time.Duration) *Event { 57 | return event.SetEndTime(event.Time.Add(duration)) 58 | } 59 | 60 | func (event *Event) SetTag(key string, value any) *Event { 61 | event.Tags[key] = value 62 | return event 63 | } 64 | 65 | func (event *Event) Log(ty zconstants.LogType, value string, attrs ...string) *Event { 66 | if len(attrs)%2 != 0 { 67 | panic("attrs must be key-value pairs") 68 | } 69 | 70 | pairs := [][2]string{} 71 | for i := 0; i < len(attrs); i += 2 { 72 | pairs = append(pairs, [2]string{attrs[i], attrs[i+1]}) 73 | } 74 | 75 | event.Logs = append(event.Logs, tracer.Log{ 76 | Type: ty, 77 | Message: value, 78 | Attrs: pairs, 79 | }) 80 | return event 81 | } 82 | -------------------------------------------------------------------------------- /pkg/aggregator/eventdecorator/decorator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package eventdecorator 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/kubewharf/kelemetry/pkg/aggregator/aggregatorevent" 21 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 22 | ) 23 | 24 | type Decorator interface { 25 | Decorate(ctx context.Context, object utilobject.Rich, event *aggregatorevent.Event) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/aggregator/eventdecorator/eventtagger/eventtagger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package eventtagger 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/spf13/pflag" 22 | 23 | "github.com/kubewharf/kelemetry/pkg/aggregator/aggregatorevent" 24 | "github.com/kubewharf/kelemetry/pkg/aggregator/eventdecorator" 25 | "github.com/kubewharf/kelemetry/pkg/aggregator/resourcetagger" 26 | "github.com/kubewharf/kelemetry/pkg/manager" 27 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 28 | ) 29 | 30 | func init() { 31 | manager.Global.ProvideListImpl("resource-event-tag", manager.Ptr(&eventTagDecorator{}), &manager.List[eventdecorator.Decorator]{}) 32 | } 33 | 34 | type options struct { 35 | enable bool 36 | filterVerbs []string 37 | } 38 | 39 | func (options *options) Setup(fs *pflag.FlagSet) { 40 | fs.BoolVar(&options.enable, "resource-event-tag-enable", false, "enable custom event tag for resource") 41 | fs.StringSliceVar( 42 | &options.filterVerbs, 43 | "resource-event-tag-filter-verbs", 44 | []string{"create"}, 45 | "add resource tag for audit verbs. e.g 'create,update,patch'") 46 | } 47 | 48 | func (options *options) EnableFlag() *bool { 49 | return &options.enable 50 | } 51 | 52 | type eventTagDecorator struct { 53 | ResourceTagger *resourcetagger.ResourceTagger 54 | options options 55 | filterVerbs map[string]struct{} 56 | } 57 | 58 | var _ manager.Component = &eventTagDecorator{} 59 | 60 | func (d *eventTagDecorator) Options() manager.Options { return &d.options } 61 | 62 | func (d *eventTagDecorator) Init() error { 63 | d.filterVerbs = map[string]struct{}{} 64 | for _, item := range d.options.filterVerbs { 65 | d.filterVerbs[item] = struct{}{} 66 | } 67 | 68 | return nil 69 | } 70 | 71 | func (d *eventTagDecorator) Start(ctx context.Context) error { return nil } 72 | func (d *eventTagDecorator) Close(ctx context.Context) error { return nil } 73 | 74 | func (d *eventTagDecorator) Decorate(ctx context.Context, object utilobject.Rich, event *aggregatorevent.Event) { 75 | if event == nil { 76 | return 77 | } 78 | 79 | if _, exist := d.filterVerbs[fmt.Sprint(event.Tags["tag"])]; !exist { 80 | return 81 | } 82 | d.ResourceTagger.DecorateTag(ctx, object, event.TraceSource, event.Tags) 83 | } 84 | -------------------------------------------------------------------------------- /pkg/aggregator/linker/job/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package linkjob 16 | 17 | import ( 18 | "context" 19 | "time" 20 | 21 | "github.com/kubewharf/kelemetry/pkg/aggregator/tracer" 22 | "github.com/kubewharf/kelemetry/pkg/manager" 23 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 24 | ) 25 | 26 | type Publisher interface { 27 | Publish(job *LinkJob) 28 | } 29 | 30 | type Subscriber interface { 31 | Subscribe(ctx context.Context, name string) <-chan *LinkJob 32 | } 33 | 34 | func init() { 35 | manager.Global.Provide("linker-job-publisher", manager.Ptr[Publisher](&publisherMux{ 36 | Mux: manager.NewMux("linker-job-publisher", false), 37 | })) 38 | manager.Global.Provide("linker-job-subscriber", manager.Ptr[Subscriber](&subscriberMux{ 39 | Mux: manager.NewMux("linker-job-subscriber", false), 40 | })) 41 | } 42 | 43 | type publisherMux struct { 44 | *manager.Mux 45 | } 46 | 47 | func (mux *publisherMux) Publish(job *LinkJob) { 48 | mux.Impl().(Publisher).Publish(job) 49 | } 50 | 51 | type subscriberMux struct { 52 | *manager.Mux 53 | } 54 | 55 | func (mux *subscriberMux) Subscribe(ctx context.Context, name string) <-chan *LinkJob { 56 | return mux.Impl().(Subscriber).Subscribe(ctx, name) 57 | } 58 | 59 | type LinkJob struct { 60 | Object utilobject.Rich 61 | EventTime time.Time 62 | Span tracer.SpanContext 63 | } 64 | -------------------------------------------------------------------------------- /pkg/aggregator/linker/linker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package linker 16 | 17 | import ( 18 | "context" 19 | 20 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 21 | "github.com/kubewharf/kelemetry/pkg/util/zconstants" 22 | ) 23 | 24 | type Linker interface { 25 | LinkerName() string 26 | Lookup(ctx context.Context, object utilobject.Rich) ([]LinkerResult, error) 27 | } 28 | 29 | type LinkerResult struct { 30 | Object utilobject.Rich 31 | Role zconstants.LinkRoleValue 32 | Class string 33 | DedupId string 34 | } 35 | -------------------------------------------------------------------------------- /pkg/aggregator/objectspandecorator/decorator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package objectspandecorator 16 | 17 | import ( 18 | "context" 19 | 20 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 21 | ) 22 | 23 | type Decorator interface { 24 | Decorate(ctx context.Context, object utilobject.Rich, traceSource string, tags map[string]string) 25 | } 26 | -------------------------------------------------------------------------------- /pkg/aggregator/objectspandecorator/resourcetagger/objecttagger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package objectspanresourcetagger 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/spf13/pflag" 22 | 23 | "github.com/kubewharf/kelemetry/pkg/aggregator/objectspandecorator" 24 | "github.com/kubewharf/kelemetry/pkg/aggregator/resourcetagger" 25 | "github.com/kubewharf/kelemetry/pkg/manager" 26 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 27 | ) 28 | 29 | func init() { 30 | manager.Global.ProvideListImpl("resource-object-tag", manager.Ptr(&ObjectSpanTag{}), &manager.List[objectspandecorator.Decorator]{}) 31 | } 32 | 33 | type options struct { 34 | enable bool 35 | } 36 | 37 | func (options *options) Setup(fs *pflag.FlagSet) { 38 | fs.BoolVar(&options.enable, "resource-object-tag-enable", false, "enable custom object span tag for resource") 39 | } 40 | 41 | func (options *options) EnableFlag() *bool { 42 | return &options.enable 43 | } 44 | 45 | type ObjectSpanTag struct { 46 | ResourceTagger *resourcetagger.ResourceTagger 47 | options options 48 | } 49 | 50 | var _ manager.Component = &ObjectSpanTag{} 51 | 52 | func (d *ObjectSpanTag) Options() manager.Options { return &d.options } 53 | func (d *ObjectSpanTag) Init() error { return nil } 54 | func (d *ObjectSpanTag) Start(ctx context.Context) error { return nil } 55 | func (d *ObjectSpanTag) Close(ctx context.Context) error { return nil } 56 | 57 | func (d *ObjectSpanTag) Decorate(ctx context.Context, object utilobject.Rich, traceSource string, tags map[string]string) { 58 | if tags == nil { 59 | return 60 | } 61 | 62 | newTags := map[string]any{} 63 | d.ResourceTagger.DecorateTag(ctx, object, traceSource, newTags) 64 | 65 | for tagKey, tagValue := range newTags { 66 | tags[tagKey] = fmt.Sprint(tagValue) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /pkg/aggregator/spancache/local/local_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package local_test 16 | 17 | import ( 18 | "context" 19 | "testing" 20 | "time" 21 | 22 | "github.com/stretchr/testify/assert" 23 | clocktesting "k8s.io/utils/clock/testing" 24 | 25 | "github.com/kubewharf/kelemetry/pkg/aggregator/spancache" 26 | "github.com/kubewharf/kelemetry/pkg/aggregator/spancache/local" 27 | ) 28 | 29 | func TestLocalReserve(t *testing.T) { 30 | assert := assert.New(t) 31 | 32 | clock := clocktesting.NewFakeClock(time.Time{}) 33 | cache := local.NewMockLocal(clock) 34 | 35 | entry1, err := cache.FetchOrReserve(context.Background(), "foo", 10) 36 | assert.Nil(err) 37 | assert.NotNil(entry1) 38 | assert.Nil(entry1.Value) 39 | 40 | clock.Step(5) 41 | 42 | _, err = cache.FetchOrReserve(context.Background(), "foo", 10) 43 | assert.NotNil(err) 44 | assert.ErrorIs(err, spancache.ErrAlreadyReserved) 45 | 46 | err = cache.SetReserved(context.Background(), "foo", []byte("bar"), spancache.Uid("no such uid"), 10) 47 | assert.NotNil(err) 48 | assert.ErrorIs(err, spancache.ErrUidMismatch) 49 | 50 | _, err = cache.FetchOrReserve(context.Background(), "foo", 10) 51 | assert.NotNil(err) 52 | assert.ErrorIs(err, spancache.ErrAlreadyReserved) 53 | 54 | err = cache.SetReserved(context.Background(), "foo", []byte("bar"), entry1.LastUid, 10) 55 | assert.Nil(err) 56 | } 57 | 58 | func TestLocalReserveExpired(t *testing.T) { 59 | assert := assert.New(t) 60 | 61 | clock := clocktesting.NewFakeClock(time.Time{}) 62 | cache := local.NewMockLocal(clock) 63 | 64 | entry1, err := cache.FetchOrReserve(context.Background(), "foo", 10) 65 | assert.Nil(err) 66 | assert.NotNil(entry1) 67 | assert.Nil(entry1.Value) 68 | 69 | clock.Step(15) 70 | 71 | err = cache.SetReserved(context.Background(), "foo", []byte("bar"), entry1.LastUid, 10) 72 | assert.NotNil(err) 73 | assert.ErrorIs(err, spancache.ErrInvalidKey) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/aggregator/tracer/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tracer 16 | 17 | import ( 18 | "time" 19 | 20 | "github.com/kubewharf/kelemetry/pkg/manager" 21 | "github.com/kubewharf/kelemetry/pkg/util/zconstants" 22 | ) 23 | 24 | func init() { 25 | manager.Global.Provide("tracer", manager.Ptr(newMux())) 26 | } 27 | 28 | type Tracer interface { 29 | CreateSpan(span Span) (SpanContext, error) 30 | InjectCarrier(spanContext SpanContext) ([]byte, error) 31 | ExtractCarrier(textMap []byte) (SpanContext, error) 32 | } 33 | 34 | type Span struct { 35 | Type string 36 | Name string 37 | StartTime time.Time 38 | FinishTime time.Time 39 | Parent SpanContext 40 | Follows SpanContext 41 | Tags map[string]string 42 | Logs []Log 43 | } 44 | 45 | type SpanContext any 46 | 47 | type Log struct { 48 | Type zconstants.LogType 49 | Message string 50 | Attrs [][2]string 51 | } 52 | 53 | type mux struct { 54 | *manager.Mux 55 | } 56 | 57 | func newMux() Tracer { 58 | return &mux{ 59 | Mux: manager.NewMux("tracer", false), 60 | } 61 | } 62 | 63 | func (mux *mux) CreateSpan(span Span) (SpanContext, error) { 64 | return mux.Impl().(Tracer).CreateSpan(span) 65 | } 66 | 67 | func (mux *mux) InjectCarrier(spanContext SpanContext) ([]byte, error) { 68 | return mux.Impl().(Tracer).InjectCarrier(spanContext) 69 | } 70 | 71 | func (mux *mux) ExtractCarrier(textMap []byte) (SpanContext, error) { 72 | return mux.Impl().(Tracer).ExtractCarrier(textMap) 73 | } 74 | -------------------------------------------------------------------------------- /pkg/annotationlinker/linker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package annotationlinker 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "fmt" 21 | 22 | "github.com/sirupsen/logrus" 23 | "github.com/spf13/pflag" 24 | 25 | "github.com/kubewharf/kelemetry/pkg/aggregator/linker" 26 | "github.com/kubewharf/kelemetry/pkg/k8s" 27 | "github.com/kubewharf/kelemetry/pkg/k8s/discovery" 28 | "github.com/kubewharf/kelemetry/pkg/k8s/objectcache" 29 | "github.com/kubewharf/kelemetry/pkg/manager" 30 | "github.com/kubewharf/kelemetry/pkg/metrics" 31 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 32 | "github.com/kubewharf/kelemetry/pkg/util/zconstants" 33 | ) 34 | 35 | func init() { 36 | manager.Global.ProvideListImpl("annotation-linker", manager.Ptr(&controller{}), &manager.List[linker.Linker]{}) 37 | } 38 | 39 | type options struct { 40 | enable bool 41 | } 42 | 43 | func (options *options) Setup(fs *pflag.FlagSet) { 44 | fs.BoolVar(&options.enable, "annotation-linker-enable", false, "enable annotation linker") 45 | } 46 | 47 | func (options *options) EnableFlag() *bool { return &options.enable } 48 | 49 | type controller struct { 50 | options options 51 | Logger logrus.FieldLogger 52 | Clients k8s.Clients 53 | DiscoveryCache discovery.DiscoveryCache 54 | ObjectCache *objectcache.ObjectCache 55 | } 56 | 57 | var _ manager.Component = &controller{} 58 | 59 | func (ctrl *controller) Options() manager.Options { return &ctrl.options } 60 | func (ctrl *controller) Init() error { return nil } 61 | func (ctrl *controller) Start(ctx context.Context) error { return nil } 62 | func (ctrl *controller) Close(ctx context.Context) error { return nil } 63 | 64 | func (ctrl *controller) LinkerName() string { return "annotation-linker" } 65 | func (ctrl *controller) Lookup(ctx context.Context, object utilobject.Rich) ([]linker.LinkerResult, error) { 66 | raw := object.Raw 67 | 68 | logger := ctrl.Logger.WithFields(object.AsFields("object")) 69 | 70 | if raw == nil { 71 | logger.Debug("Fetching dynamic object") 72 | 73 | var err error 74 | raw, err = ctrl.ObjectCache.Get(ctx, object.VersionedKey) 75 | if err != nil { 76 | return nil, metrics.LabelError(fmt.Errorf("cannot fetch object value: %w", err), "FetchCache") 77 | } 78 | 79 | if raw == nil { 80 | logger.Debug("object no longer exists") 81 | return nil, nil 82 | } 83 | } 84 | 85 | if ann, ok := raw.GetAnnotations()[LinkAnnotation]; ok { 86 | ref := &ParentLink{} 87 | err := json.Unmarshal([]byte(ann), ref) 88 | if err != nil { 89 | return nil, metrics.LabelError(fmt.Errorf("cannot parse ParentLink annotation: %w", err), "ParseAnnotation") 90 | } 91 | 92 | if ref.Cluster == "" { 93 | ref.Cluster = object.Cluster 94 | } 95 | 96 | objectRef := ref.ToRich() 97 | logger.WithFields(objectRef.AsFields("parent")).Debug("Resolved parent") 98 | 99 | return []linker.LinkerResult{{ 100 | Object: objectRef, 101 | Role: zconstants.LinkRoleParent, 102 | DedupId: "annotation", 103 | }}, nil 104 | } 105 | 106 | return nil, nil 107 | } 108 | -------------------------------------------------------------------------------- /pkg/annotationlinker/schema.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package annotationlinker 16 | 17 | import ( 18 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 19 | "k8s.io/apimachinery/pkg/types" 20 | 21 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 22 | ) 23 | 24 | const LinkAnnotation = "kelemetry.kubewharf.io/parent-link" 25 | 26 | type ParentLink struct { 27 | Cluster string `json:"cluster,omitempty"` 28 | 29 | metav1.GroupVersionResource 30 | 31 | Namespace string `json:"namespace,omitempty"` 32 | Name string `json:"name"` 33 | 34 | Uid types.UID `json:"uid"` 35 | } 36 | 37 | func (ln ParentLink) ToRich() utilobject.Rich { 38 | return utilobject.Rich{ 39 | VersionedKey: utilobject.VersionedKey{ 40 | Key: utilobject.Key{ 41 | Cluster: ln.Cluster, 42 | Group: ln.Group, 43 | Resource: ln.Resource, 44 | Namespace: ln.Namespace, 45 | Name: ln.Name, 46 | }, 47 | Version: ln.Version, 48 | }, 49 | Uid: ln.Uid, 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /pkg/audit/decorator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package audit 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/kubewharf/kelemetry/pkg/aggregator/aggregatorevent" 21 | ) 22 | 23 | type Decorator interface { 24 | Decorate(ctx context.Context, message *Message, event *aggregatorevent.Event) 25 | } 26 | -------------------------------------------------------------------------------- /pkg/audit/dump/dump.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package auditdump 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "fmt" 21 | "io" 22 | "os" 23 | 24 | "github.com/sirupsen/logrus" 25 | "github.com/spf13/pflag" 26 | 27 | "github.com/kubewharf/kelemetry/pkg/audit" 28 | auditwebhook "github.com/kubewharf/kelemetry/pkg/audit/webhook" 29 | "github.com/kubewharf/kelemetry/pkg/manager" 30 | "github.com/kubewharf/kelemetry/pkg/util/shutdown" 31 | ) 32 | 33 | func init() { 34 | manager.Global.Provide("audit-dump", manager.Ptr(&dumper{})) 35 | } 36 | 37 | type options struct { 38 | target string 39 | } 40 | 41 | func (options *options) Setup(fs *pflag.FlagSet) { 42 | fs.StringVar(&options.target, "audit-dump-file", "", "append received audit events to the specified path (leave empty to disable)") 43 | } 44 | 45 | func (options *options) EnableFlag() *bool { 46 | isEnable := options.target != "" 47 | return &isEnable 48 | } 49 | 50 | type dumper struct { 51 | options options 52 | Logger logrus.FieldLogger 53 | Webhook auditwebhook.Webhook 54 | stream interface { 55 | io.WriteCloser 56 | Sync() error 57 | } 58 | recvCh <-chan *audit.Message 59 | } 60 | 61 | func (dumper *dumper) Options() manager.Options { 62 | return &dumper.options 63 | } 64 | 65 | func (dumper *dumper) Init() (err error) { 66 | dumper.stream, err = os.OpenFile(dumper.options.target, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o600) 67 | if err != nil { 68 | return fmt.Errorf("cannot open file to dump audit events: %w", err) 69 | } 70 | 71 | dumper.recvCh = dumper.Webhook.AddSubscriber("audit-dump-subscriber") 72 | 73 | return nil 74 | } 75 | 76 | func (dumper *dumper) Start(ctx context.Context) error { 77 | go func() { 78 | defer shutdown.RecoverPanic(dumper.Logger) 79 | 80 | flushCh := chan struct{}(nil) 81 | 82 | for { 83 | select { 84 | case <-ctx.Done(): 85 | return 86 | case <-flushCh: 87 | if err := dumper.stream.Sync(); err != nil { 88 | dumper.Logger.WithError(err).Error("Cannot flush file") 89 | } 90 | case message, chOpen := <-dumper.recvCh: 91 | if !chOpen { 92 | return 93 | } 94 | 95 | if err := dumper.handleEvent(message); err != nil { 96 | dumper.Logger.WithError(err).Error("Cannot append message") 97 | } 98 | } 99 | } 100 | }() 101 | 102 | return nil 103 | } 104 | 105 | func (dumper *dumper) handleEvent(message *audit.Message) error { 106 | buf, err := json.Marshal(message) 107 | if err != nil { 108 | return fmt.Errorf("cannot reserialize message: %w", err) 109 | } 110 | 111 | for _, bytes := range [][]byte{buf, []byte("\n")} { 112 | _, err = dumper.stream.Write(bytes) 113 | if err != nil { 114 | return fmt.Errorf("cannot write to file: %w", err) 115 | } 116 | } 117 | 118 | return nil 119 | } 120 | 121 | func (dumper *dumper) Close(ctx context.Context) error { 122 | return dumper.stream.Close() 123 | } 124 | -------------------------------------------------------------------------------- /pkg/audit/message.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package audit 16 | 17 | import auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" 18 | 19 | type Message struct { 20 | Cluster string `json:"cluster"` 21 | ApiserverAddr string `json:"sourceAddr"` 22 | auditv1.Event 23 | } 24 | 25 | type RawMessage struct { 26 | Cluster string `json:"cluster"` 27 | SourceAddr string `json:"sourceAddr"` 28 | *auditv1.EventList 29 | } 30 | 31 | // ClusterOnlyMessage is a trimmed version of Message/RawMessage that only deserializes the `cluster` field. 32 | type ClusterOnlyMessage struct { 33 | Cluster string `json:"cluster"` 34 | } 35 | 36 | const ( 37 | VerbCreate = "create" 38 | VerbUpdate = "update" 39 | VerbDelete = "delete" 40 | VerbPatch = "patch" 41 | ) 42 | -------------------------------------------------------------------------------- /pkg/audit/mq/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package mq 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/sirupsen/logrus" 21 | 22 | "github.com/kubewharf/kelemetry/pkg/manager" 23 | ) 24 | 25 | func init() { 26 | manager.Global.Provide("mq", manager.Ptr[Queue](&mux{ 27 | Mux: manager.NewMux("mq", false), 28 | })) 29 | } 30 | 31 | type ( 32 | ConsumerGroup string 33 | PartitionId int32 34 | ) 35 | 36 | // Queue is the abstraction of a partitioned message queue. 37 | type Queue interface { 38 | CreateProducer() (Producer, error) 39 | CreateConsumer(group ConsumerGroup, partition PartitionId, handler MessageHandler) (Consumer, error) 40 | } 41 | 42 | type Producer interface { 43 | Send(partitionKey []byte, value []byte) error 44 | } 45 | 46 | type Consumer any 47 | 48 | type mux struct { 49 | *manager.Mux 50 | } 51 | 52 | func (mux *mux) CreateProducer() (Producer, error) { 53 | return mux.Impl().(Queue).CreateProducer() 54 | } 55 | 56 | func (mux *mux) CreateConsumer(group ConsumerGroup, partition PartitionId, handler MessageHandler) (Consumer, error) { 57 | return mux.Impl().(Queue).CreateConsumer(group, partition, handler) 58 | } 59 | 60 | type MessageHandler func(ctx context.Context, fieldLogger logrus.FieldLogger, key []byte, value []byte) 61 | -------------------------------------------------------------------------------- /pkg/audit/webhook/clustername/address/address.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package address 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/kubewharf/kelemetry/pkg/audit/webhook/clustername" 21 | "github.com/kubewharf/kelemetry/pkg/manager" 22 | ) 23 | 24 | func init() { 25 | manager.Global.ProvideMuxImpl("cluster-name/address", manager.Ptr(&AddressResolver{}), clustername.Resolver.Resolve) 26 | } 27 | 28 | type AddressResolver struct { 29 | manager.MuxImplBase 30 | } 31 | 32 | func (_ *AddressResolver) MuxImplName() (name string, isDefault bool) { return "address", true } 33 | 34 | func (resolver *AddressResolver) Options() manager.Options { return &manager.NoOptions{} } 35 | 36 | func (resolver *AddressResolver) Init() error { return nil } 37 | 38 | func (resolver *AddressResolver) Start(ctx context.Context) error { return nil } 39 | 40 | func (resolver *AddressResolver) Close(ctx context.Context) error { return nil } 41 | 42 | func (resolver *AddressResolver) Resolve(ip string) string { return ip } 43 | -------------------------------------------------------------------------------- /pkg/audit/webhook/clustername/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package clustername 16 | 17 | import ( 18 | "github.com/kubewharf/kelemetry/pkg/manager" 19 | ) 20 | 21 | func init() { 22 | manager.Global.Provide("cluster-name", manager.Ptr[Resolver](&mux{ 23 | Mux: manager.NewMux("cluster-name-resolver", false), 24 | })) 25 | } 26 | 27 | type Resolver interface { 28 | Resolve(ip string) string 29 | } 30 | 31 | type mux struct { 32 | *manager.Mux 33 | } 34 | 35 | func (mux *mux) Resolve(ip string) string { 36 | return mux.Impl().(Resolver).Resolve(ip) 37 | } 38 | -------------------------------------------------------------------------------- /pkg/diff/cache/snapshot_names.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package diffcache 16 | 17 | const ( 18 | SnapshotNameCreation = "creation" 19 | SnapshotNameDeletion = "deletion" 20 | ) 21 | 22 | var VerbToSnapshotName = map[string]string{ 23 | "create": SnapshotNameCreation, 24 | "delete": SnapshotNameDeletion, 25 | } 26 | -------------------------------------------------------------------------------- /pkg/frontend/backend/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package jaegerbackend 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "time" 21 | 22 | "github.com/jaegertracing/jaeger/model" 23 | "github.com/jaegertracing/jaeger/storage/spanstore" 24 | "k8s.io/utils/clock" 25 | 26 | tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree" 27 | "github.com/kubewharf/kelemetry/pkg/manager" 28 | "github.com/kubewharf/kelemetry/pkg/metrics" 29 | ) 30 | 31 | func init() { 32 | manager.Global.Provide("jaeger-backend", manager.Ptr[Backend](&mux{ 33 | Mux: manager.NewMux("jaeger-backend", false), 34 | })) 35 | } 36 | 37 | type Backend interface { 38 | // Lists the thumbnail previews of all traces. 39 | List( 40 | ctx context.Context, 41 | query *spanstore.TraceQueryParameters, 42 | ) ([]*TraceThumbnail, error) 43 | 44 | // Gets the full tree of a trace based on the identifier returned from a previous call to List. 45 | // 46 | // traceId is the fake trace ID that should be presented to the user. 47 | // 48 | // startTime and endTime are only for optimization hint. 49 | // The implementation is allowed to return spans beyond the range. 50 | Get( 51 | ctx context.Context, 52 | identifier json.RawMessage, 53 | traceId model.TraceID, 54 | startTime, endTime time.Time, 55 | ) (*model.Trace, error) 56 | } 57 | 58 | type TraceThumbnail struct { 59 | // Identifier is a serializable object that identifies the trace in GetTrace calls. 60 | Identifier any 61 | 62 | Spans *tftree.SpanTree 63 | } 64 | 65 | func (tt *TraceThumbnail) GetSpans() *tftree.SpanTree { return tt.Spans } 66 | func (tt *TraceThumbnail) GetMetadata() any { return tt.Identifier } 67 | func (tt *TraceThumbnail) FromThumbnail(src *TraceThumbnail) { *tt = *src } 68 | 69 | type mux struct { 70 | *manager.Mux 71 | Clock clock.Clock 72 | 73 | ListMetric *metrics.Metric[*listMetric] 74 | GetMetric *metrics.Metric[*getMetric] 75 | } 76 | 77 | type ( 78 | listMetric struct{} 79 | getMetric struct{} 80 | ) 81 | 82 | func (*listMetric) MetricName() string { return "jaeger_backend_list" } 83 | func (*getMetric) MetricName() string { return "jaeger_backend_get" } 84 | 85 | func (mux *mux) List( 86 | ctx context.Context, 87 | query *spanstore.TraceQueryParameters, 88 | ) ([]*TraceThumbnail, error) { 89 | defer mux.ListMetric.DeferCount(mux.Clock.Now(), &listMetric{}) 90 | return mux.Impl().(Backend).List(ctx, query) 91 | } 92 | 93 | func (mux *mux) Get( 94 | ctx context.Context, 95 | identifier json.RawMessage, 96 | traceId model.TraceID, 97 | startTime, endTime time.Time, 98 | ) (*model.Trace, error) { 99 | defer mux.GetMetric.DeferCount(mux.Clock.Now(), &getMetric{}) 100 | return mux.Impl().(Backend).Get(ctx, identifier, traceId, startTime, endTime) 101 | } 102 | -------------------------------------------------------------------------------- /pkg/frontend/clusterlist/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package clusterlist 16 | 17 | import ( 18 | "github.com/kubewharf/kelemetry/pkg/manager" 19 | ) 20 | 21 | func init() { 22 | manager.Global.Provide("jaeger-cluster-list", manager.Ptr[Lister](&mux{ 23 | Mux: manager.NewMux("jaeger-cluster-list", false), 24 | })) 25 | } 26 | 27 | type Lister interface { 28 | List() []string 29 | } 30 | 31 | type mux struct { 32 | *manager.Mux 33 | } 34 | 35 | func (mux *mux) List() []string { 36 | return mux.Impl().(Lister).List() 37 | } 38 | -------------------------------------------------------------------------------- /pkg/frontend/clusterlist/options/options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package options 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/spf13/pflag" 21 | 22 | "github.com/kubewharf/kelemetry/pkg/frontend/clusterlist" 23 | "github.com/kubewharf/kelemetry/pkg/manager" 24 | ) 25 | 26 | func init() { 27 | manager.Global.ProvideMuxImpl("jaeger-cluster-list/options", manager.Ptr(&Lister{}), clusterlist.Lister.List) 28 | } 29 | 30 | type options struct { 31 | clusters []string 32 | } 33 | 34 | func (options *options) Setup(fs *pflag.FlagSet) { 35 | fs.StringSliceVar(&options.clusters, "jaeger-cluster-names", []string{}, "cluster names allowed") 36 | } 37 | 38 | func (options *options) EnableFlag() *bool { return nil } 39 | 40 | type Lister struct { 41 | manager.MuxImplBase 42 | options options 43 | } 44 | 45 | var _ clusterlist.Lister = &Lister{} 46 | 47 | func (_ *Lister) MuxImplName() (name string, isDefault bool) { return "options", true } 48 | 49 | func (lister *Lister) Options() manager.Options { return &lister.options } 50 | 51 | func (lister *Lister) Init() error { return nil } 52 | 53 | func (lister *Lister) Start(ctx context.Context) error { return nil } 54 | 55 | func (lister *Lister) Close(ctx context.Context) error { return nil } 56 | 57 | func (lister *Lister) List() []string { return lister.options.clusters } 58 | -------------------------------------------------------------------------------- /pkg/frontend/extension/provider.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package extension 16 | 17 | import ( 18 | "context" 19 | "time" 20 | 21 | "github.com/jaegertracing/jaeger/model" 22 | 23 | "github.com/kubewharf/kelemetry/pkg/manager" 24 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 25 | ) 26 | 27 | // Produces new extension providers based on the configuration. 28 | // 29 | // This interface is used for manager.List. 30 | type ProviderFactory interface { 31 | manager.IndexedListImpl 32 | 33 | Configure(jsonBuf []byte) (Provider, error) 34 | } 35 | 36 | // An instance of extension provider as specified in the config. 37 | // Loads spans from a specific source, typically from one specific component. 38 | type Provider interface { 39 | // Kind of this extension provider, same as the one from `ProviderFactory.Kind()` 40 | Kind() string 41 | 42 | // The raw JSON config buffer used to configure this provider. 43 | RawConfig() []byte 44 | 45 | // Maximum wall time to execute all fetch calls. 46 | // Only successuful queries before this timestamp will be included in the output. 47 | TotalTimeout() time.Duration 48 | // Maximum concurrent fetch calls to execute for the same trace. 49 | MaxConcurrency() int 50 | 51 | // Searches for spans associated with an object. 52 | // 53 | // This method is invoked during the first time a main trace is loaded. 54 | // Subsequent invocations will call `LoadCache` instead. 55 | // 56 | // Returned traces are attached as subtrees under the object pseudospan. 57 | // 58 | // `tags` contains the tags in the object pseudospan in the main trace. 59 | FetchForObject( 60 | ctx context.Context, 61 | object utilobject.VersionedKey, 62 | tags model.KeyValues, 63 | start, end time.Time, 64 | ) (*FetchResult, error) 65 | 66 | // Searches for spans associated with an object version. 67 | // 68 | // This method is invoked during the first time a main trace is loaded. 69 | // Subsequent invocations will call `LoadCache` instead. 70 | // 71 | // Returned traces are attached as subtrees under the audit span. 72 | // 73 | // `tags` contains the tags in the object pseudospan in the main trace. 74 | FetchForVersion( 75 | ctx context.Context, 76 | object utilobject.VersionedKey, 77 | resourceVersion string, 78 | tags model.KeyValues, 79 | start, end time.Time, 80 | ) (*FetchResult, error) 81 | 82 | // Restores the FetchResult from the identifier 83 | // persisted from the result of a previous call to `Fetch`. 84 | LoadCache(ctx context.Context, identifier []byte) ([]*model.Span, error) 85 | } 86 | 87 | type FetchResult struct { 88 | // A JSON-serializable object used to persist the parameters of this fetch, 89 | // such as the extension trace ID. 90 | Identifier any 91 | // The spans in the extension trace. 92 | // 93 | // Orphan spans (with no or invalid parent reference) are treated as root spans. 94 | Spans []*model.Span 95 | } 96 | -------------------------------------------------------------------------------- /pkg/frontend/tf/config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tfconfig 16 | 17 | import ( 18 | "fmt" 19 | "sort" 20 | "strconv" 21 | "strings" 22 | 23 | "k8s.io/apimachinery/pkg/util/sets" 24 | 25 | "github.com/kubewharf/kelemetry/pkg/frontend/extension" 26 | "github.com/kubewharf/kelemetry/pkg/manager" 27 | ) 28 | 29 | func init() { 30 | manager.Global.Provide("jaeger-transform-config", manager.Ptr[Provider](&mux{ 31 | Mux: manager.NewMux("jaeger-transform-config", false), 32 | })) 33 | } 34 | 35 | type Provider interface { 36 | Names() []string 37 | DefaultName() string 38 | DefaultId() Id 39 | GetByName(name string) *Config 40 | GetById(id Id) *Config 41 | } 42 | 43 | type Id uint32 44 | 45 | func (id *Id) UnmarshalText(text []byte) error { 46 | i, err := strconv.ParseUint(string(text), 16, 32) 47 | if err != nil { 48 | return err 49 | } 50 | 51 | // #nosec G115 -- ParseUint bitSize is 32 52 | *id = Id(uint32(i)) 53 | return nil 54 | } 55 | 56 | type Config struct { 57 | // The config ID, used to generate the cache ID. 58 | Id Id 59 | // The config name, used in search page display. 60 | Name string 61 | // Base config name without modifiers, used to help reconstruct the name. 62 | BaseName string 63 | // Names of modifiers, used to help reconstruct the name. 64 | ModifierNames sets.Set[string] 65 | // Only links with roles in this set are followed. 66 | LinkSelector LinkSelector 67 | // The extension traces for this config. 68 | Extensions []extension.Provider 69 | // The steps to transform the tree 70 | Steps []Step 71 | } 72 | 73 | func (config *Config) RecomputeName() { 74 | modifiers := config.ModifierNames.UnsortedList() 75 | sort.Strings(modifiers) 76 | if len(modifiers) > 0 { 77 | config.Name = fmt.Sprintf("%s [%s]", config.BaseName, strings.Join(modifiers, "+")) 78 | } else { 79 | config.Name = config.BaseName 80 | } 81 | } 82 | 83 | func (config *Config) Clone() *Config { 84 | steps := make([]Step, len(config.Steps)) 85 | copy(steps, config.Steps) // no need to deep clone each step 86 | 87 | extensions := make([]extension.Provider, len(config.Extensions)) 88 | copy(extensions, config.Extensions) 89 | 90 | return &Config{ 91 | Id: config.Id, 92 | Name: config.Name, 93 | BaseName: config.BaseName, 94 | ModifierNames: config.ModifierNames.Clone(), 95 | LinkSelector: config.LinkSelector, // modifier changes LinkSelector by wrapping the previous value 96 | Extensions: extensions, 97 | Steps: steps, 98 | } 99 | } 100 | 101 | type mux struct { 102 | *manager.Mux 103 | } 104 | 105 | func (mux *mux) Names() []string { return mux.Impl().(Provider).Names() } 106 | func (mux *mux) DefaultName() string { return mux.Impl().(Provider).DefaultName() } 107 | func (mux *mux) DefaultId() Id { return mux.Impl().(Provider).DefaultId() } 108 | func (mux *mux) GetByName(name string) *Config { return mux.Impl().(Provider).GetByName(name) } 109 | func (mux *mux) GetById(id Id) *Config { return mux.Impl().(Provider).GetById(id) } 110 | -------------------------------------------------------------------------------- /pkg/frontend/tf/config/link_selector.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tfconfig 16 | 17 | import utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 18 | 19 | type LinkSelector interface { 20 | // Whether to follow the given link. 21 | // 22 | // If link should be followed, return a non-nil LinkSelector. 23 | // The returned object will be used to recursively follow links in the linked object. 24 | Admit(parent utilobject.Key, child utilobject.Key, parentIsSource bool, linkClass string) LinkSelector 25 | } 26 | 27 | type ConstantLinkSelector bool 28 | 29 | func (selector ConstantLinkSelector) Admit( 30 | parent utilobject.Key, 31 | child utilobject.Key, 32 | parentIsSource bool, 33 | linkClass string, 34 | ) LinkSelector { 35 | if selector { 36 | return selector 37 | } 38 | 39 | return nil 40 | } 41 | 42 | type IntersectLinkSelector []LinkSelector 43 | 44 | func (selector IntersectLinkSelector) Admit( 45 | parentKey utilobject.Key, 46 | childKey utilobject.Key, 47 | parentIsSource bool, 48 | linkClass string, 49 | ) LinkSelector { 50 | newChildren := make([]LinkSelector, len(selector)) 51 | 52 | for i, child := range selector { 53 | newChildren[i] = child.Admit(parentKey, childKey, parentIsSource, linkClass) 54 | if newChildren[i] == nil { 55 | return nil 56 | } 57 | } 58 | 59 | return IntersectLinkSelector(newChildren) 60 | } 61 | 62 | type UnionLinkSelector []LinkSelector 63 | 64 | func (selector UnionLinkSelector) Admit( 65 | parentKey utilobject.Key, 66 | childKey utilobject.Key, 67 | parentIsSource bool, 68 | linkClass string, 69 | ) LinkSelector { 70 | newChildren := make([]LinkSelector, len(selector)) 71 | 72 | ok := false 73 | for i, child := range selector { 74 | if child != nil { 75 | newChildren[i] = child.Admit(parentKey, childKey, parentIsSource, linkClass) 76 | if newChildren[i] != nil { 77 | ok = true 78 | } 79 | } 80 | } 81 | 82 | if ok { 83 | return UnionLinkSelector(newChildren) 84 | } 85 | 86 | return nil 87 | } 88 | -------------------------------------------------------------------------------- /pkg/frontend/tf/config/modifier.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tfconfig 16 | 17 | import "github.com/kubewharf/kelemetry/pkg/manager" 18 | 19 | type ModifierFactory interface { 20 | manager.IndexedListImpl 21 | 22 | Build(jsonBuf []byte) (Modifier, error) 23 | } 24 | 25 | type Modifier interface { 26 | // Modifiers with the same class are incompatible with one another. 27 | // Used to reduce cardinality of available display modes in frontend. 28 | ModifierClass() string 29 | 30 | Modify(config *Config) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/frontend/tf/config/step.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tfconfig 16 | 17 | import ( 18 | "encoding/json" 19 | "fmt" 20 | 21 | tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree" 22 | "github.com/kubewharf/kelemetry/pkg/manager" 23 | ) 24 | 25 | type Step interface { 26 | Run(tree *tftree.SpanTree) 27 | } 28 | 29 | type RegisteredStep interface { 30 | Step 31 | manager.IndexedListImpl 32 | 33 | Kind() string 34 | 35 | UnmarshalNewJSON(buf []byte) (Step, error) 36 | } 37 | 38 | type RegisteredVisitor interface { 39 | tftree.TreeVisitor 40 | 41 | Kind() string 42 | } 43 | 44 | type VisitorStep[V RegisteredVisitor] struct { 45 | manager.BaseComponent 46 | 47 | Visitor V `managerSkipFill:""` 48 | } 49 | 50 | func (step *VisitorStep[V]) Options() manager.Options { return &manager.AlwaysEnableOptions{} } 51 | 52 | func (step *VisitorStep[V]) UnmarshalJSON(buf []byte) error { 53 | return json.Unmarshal(buf, &step.Visitor) 54 | } 55 | 56 | func (step *VisitorStep[V]) Run(tree *tftree.SpanTree) { 57 | tree.Visit(step.Visitor) 58 | } 59 | 60 | func (step *VisitorStep[V]) ListIndex() string { return step.Visitor.Kind() } 61 | func (step *VisitorStep[V]) Kind() string { return step.Visitor.Kind() } 62 | 63 | func (*VisitorStep[V]) UnmarshalNewJSON(buf []byte) (Step, error) { 64 | step := &VisitorStep[V]{} 65 | if err := step.UnmarshalJSON(buf); err != nil { 66 | return nil, err 67 | } 68 | return step, nil 69 | } 70 | 71 | type BatchStep struct { 72 | Steps []Step 73 | } 74 | 75 | func (step *BatchStep) Run(tree *tftree.SpanTree) { 76 | for _, child := range step.Steps { 77 | child.Run(tree) 78 | } 79 | } 80 | 81 | func ParseSteps(buf []byte, batches map[string][]Step, registeredSteps map[string]RegisteredStep) ([]Step, error) { 82 | raw := []json.RawMessage{} 83 | 84 | if err := json.Unmarshal(buf, &raw); err != nil { 85 | return nil, fmt.Errorf("json parse error: %w", err) 86 | } 87 | 88 | steps := make([]Step, len(raw)) 89 | for i, stepBuf := range raw { 90 | step, err := UnmarshalStep(batches, stepBuf, registeredSteps) 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | steps[i] = step 96 | } 97 | 98 | return steps, nil 99 | } 100 | 101 | func UnmarshalStep(batches map[string][]Step, buf []byte, registeredSteps map[string]RegisteredStep) (Step, error) { 102 | var hasKind struct { 103 | Kind string `json:"kind"` 104 | } 105 | if err := json.Unmarshal(buf, &hasKind); err != nil { 106 | return nil, fmt.Errorf("JSON object must have \"kind\" field: %w", err) 107 | } 108 | 109 | if hasKind.Kind == "Batch" { 110 | var hasBatchName struct { 111 | BatchName string `json:"batchName"` 112 | } 113 | if err := json.Unmarshal(buf, &hasBatchName); err != nil { 114 | return nil, fmt.Errorf("JSON object must have \"batchName\" field: %w", err) 115 | } 116 | 117 | batch, exists := batches[hasBatchName.BatchName] 118 | if !exists { 119 | return nil, fmt.Errorf("unknown batch %q", hasBatchName.BatchName) 120 | } 121 | 122 | return &BatchStep{Steps: batch}, nil 123 | } 124 | 125 | if step, hasStep := registeredSteps[hasKind.Kind]; hasStep { 126 | return step.UnmarshalNewJSON(buf) 127 | } 128 | 129 | return nil, fmt.Errorf("unknown kind %q", hasKind.Kind) 130 | } 131 | -------------------------------------------------------------------------------- /pkg/frontend/tf/defaults/modifier/extension.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tfmodifier 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "fmt" 21 | 22 | "github.com/spf13/pflag" 23 | 24 | "github.com/kubewharf/kelemetry/pkg/frontend/extension" 25 | tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config" 26 | "github.com/kubewharf/kelemetry/pkg/manager" 27 | ) 28 | 29 | func init() { 30 | manager.Global.ProvideListImpl( 31 | "tf-modifier/extension", 32 | manager.Ptr(&ExtensionModifierFactory{}), 33 | &manager.List[tfconfig.ModifierFactory]{}, 34 | ) 35 | } 36 | 37 | type ExtensionModifierOptions struct { 38 | enable bool 39 | } 40 | 41 | func (options *ExtensionModifierOptions) Setup(fs *pflag.FlagSet) { 42 | fs.BoolVar(&options.enable, "jaeger-tf-extension-modifier-enable", true, "enable extension modifier") 43 | } 44 | 45 | func (options *ExtensionModifierOptions) EnableFlag() *bool { return &options.enable } 46 | 47 | type ExtensionModifierFactory struct { 48 | options ExtensionModifierOptions 49 | ProviderList *manager.List[extension.ProviderFactory] 50 | } 51 | 52 | var _ manager.Component = &ExtensionModifierFactory{} 53 | 54 | func (m *ExtensionModifierFactory) Options() manager.Options { return &m.options } 55 | func (m *ExtensionModifierFactory) Init() error { return nil } 56 | func (m *ExtensionModifierFactory) Start(ctx context.Context) error { return nil } 57 | func (m *ExtensionModifierFactory) Close(ctx context.Context) error { return nil } 58 | 59 | func (*ExtensionModifierFactory) ListIndex() string { return "extension" } 60 | 61 | func (m *ExtensionModifierFactory) Build(jsonBuf []byte) (tfconfig.Modifier, error) { 62 | var hasKind struct { 63 | Kind string `json:"kind"` 64 | Class string `json:"modifierClass"` 65 | } 66 | 67 | if err := json.Unmarshal(jsonBuf, &hasKind); err != nil || hasKind.Kind == "" { 68 | return nil, fmt.Errorf("no extension kind specified: %w", err) 69 | } 70 | 71 | matchedFactory, hasMatchedFactory := m.ProviderList.Indexed[hasKind.Kind] 72 | if !hasMatchedFactory { 73 | return nil, fmt.Errorf("no extension provider with kind %q", hasKind.Kind) 74 | } 75 | 76 | provider, err := matchedFactory.Configure(jsonBuf) 77 | if err != nil { 78 | return nil, fmt.Errorf("parse extension provider config error: %w", err) 79 | } 80 | 81 | return &ExtensionModifier{ 82 | provider: provider, 83 | class: hasKind.Class, 84 | }, nil 85 | } 86 | 87 | type ExtensionModifier struct { 88 | class string 89 | provider extension.Provider 90 | } 91 | 92 | func (modifier *ExtensionModifier) ModifierClass() string { 93 | return fmt.Sprintf("kelemetry.kubewharf.io/extension/%s", modifier.class) 94 | } 95 | 96 | func (modifier *ExtensionModifier) Modify(config *tfconfig.Config) { 97 | config.Extensions = append(config.Extensions, modifier.provider) 98 | } 99 | -------------------------------------------------------------------------------- /pkg/frontend/tf/defaults/step/compact_duration.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tfstep 16 | 17 | import ( 18 | "time" 19 | 20 | "github.com/jaegertracing/jaeger/model" 21 | 22 | tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config" 23 | tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree" 24 | "github.com/kubewharf/kelemetry/pkg/manager" 25 | "github.com/kubewharf/kelemetry/pkg/util/zconstants" 26 | ) 27 | 28 | func init() { 29 | manager.Global.ProvideListImpl( 30 | "tf-step/compact-duration-visitor", 31 | manager.Ptr(&tfconfig.VisitorStep[CompactDurationVisitor]{}), 32 | &manager.List[tfconfig.RegisteredStep]{}, 33 | ) 34 | } 35 | 36 | // Reduce pseudospan duration into a "flame" shape. 37 | type CompactDurationVisitor struct{} 38 | 39 | func (CompactDurationVisitor) Kind() string { return "CompactDurationVisitor" } 40 | 41 | func (visitor CompactDurationVisitor) Enter(tree *tftree.SpanTree, span *model.Span) tftree.TreeVisitor { 42 | return visitor 43 | } 44 | 45 | func (visitor CompactDurationVisitor) Exit(tree *tftree.SpanTree, span *model.Span) { 46 | // use exit hook to use compact results of children 47 | 48 | if _, isPseudo := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType); !isPseudo { 49 | return 50 | } 51 | 52 | var r timeRange 53 | 54 | for childId := range tree.Children(span.SpanID) { 55 | child := tree.Span(childId) 56 | 57 | childStart := child.StartTime 58 | childEnd := childStart.Add(child.Duration) 59 | 60 | r.union(childStart, childEnd) 61 | } 62 | 63 | for _, log := range span.Logs { 64 | r.union(log.Timestamp, log.Timestamp.Add(zconstants.DummyDuration)) 65 | } 66 | 67 | if r.hasStart && r.hasEnd { 68 | span.StartTime = r.start 69 | span.Duration = r.end.Sub(r.start) 70 | 71 | if tree.Root == span { 72 | // add 5% padding for the root span for better visualization 73 | 74 | padding := span.Duration / 20 75 | span.StartTime = span.StartTime.Add(-padding) 76 | span.Duration += padding * 2 77 | } 78 | } 79 | } 80 | 81 | type timeRange struct { 82 | start, end time.Time 83 | hasStart, hasEnd bool 84 | } 85 | 86 | func (r *timeRange) union(start time.Time, end time.Time) { 87 | if !r.hasStart || r.start.After(start) { 88 | r.hasStart = true 89 | r.start = start 90 | } 91 | 92 | if !r.hasEnd || r.end.Before(end) { 93 | r.hasEnd = true 94 | r.end = end 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /pkg/frontend/tf/defaults/step/extract_nesting.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tfstep 16 | 17 | import ( 18 | "github.com/jaegertracing/jaeger/model" 19 | 20 | tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config" 21 | tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree" 22 | "github.com/kubewharf/kelemetry/pkg/manager" 23 | utilmarshal "github.com/kubewharf/kelemetry/pkg/util/marshal" 24 | "github.com/kubewharf/kelemetry/pkg/util/zconstants" 25 | ) 26 | 27 | func init() { 28 | manager.Global.ProvideListImpl( 29 | "tf-step/extract-nesting-visitor", 30 | manager.Ptr(&tfconfig.VisitorStep[ExtractNestingVisitor]{}), 31 | &manager.List[tfconfig.RegisteredStep]{}, 32 | ) 33 | } 34 | 35 | // Deletes spans matching MatchesPseudoType and brings their children one level up. 36 | type ExtractNestingVisitor struct { 37 | // Filters the trace sources to delete. 38 | MatchesPseudoType utilmarshal.StringFilter `json:"matchesPseudoType"` 39 | } 40 | 41 | func (ExtractNestingVisitor) Kind() string { return "ExtractNestingVisitor" } 42 | 43 | func (visitor ExtractNestingVisitor) Enter(tree *tftree.SpanTree, span *model.Span) tftree.TreeVisitor { 44 | if len(span.References) == 0 { 45 | // we cannot extract the root span 46 | return visitor 47 | } 48 | 49 | if pseudoType, ok := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType); ok { 50 | if visitor.MatchesPseudoType.Matches(pseudoType.AsString()) { 51 | childrenMap := tree.Children(span.SpanID) 52 | childrenCopy := make([]model.SpanID, 0, len(childrenMap)) 53 | for childId := range childrenMap { 54 | childrenCopy = append(childrenCopy, childId) 55 | } 56 | 57 | for _, child := range childrenCopy { 58 | tree.Move(child, span.References[0].SpanID) 59 | } 60 | 61 | tree.Delete(span.SpanID) 62 | } 63 | } 64 | 65 | return visitor 66 | } 67 | func (visitor ExtractNestingVisitor) Exit(tree *tftree.SpanTree, span *model.Span) {} 68 | -------------------------------------------------------------------------------- /pkg/frontend/tf/defaults/step/group_by_trace_source.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tfstep 16 | 17 | import ( 18 | "math/rand" 19 | 20 | "github.com/jaegertracing/jaeger/model" 21 | 22 | tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config" 23 | tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree" 24 | "github.com/kubewharf/kelemetry/pkg/manager" 25 | utilmarshal "github.com/kubewharf/kelemetry/pkg/util/marshal" 26 | "github.com/kubewharf/kelemetry/pkg/util/zconstants" 27 | ) 28 | 29 | func init() { 30 | manager.Global.ProvideListImpl( 31 | "tf-step/group-by-trace-source-visitor", 32 | manager.Ptr(&tfconfig.VisitorStep[GroupByTraceSourceVisitor]{}), 33 | &manager.List[tfconfig.RegisteredStep]{}, 34 | ) 35 | } 36 | 37 | const myPseudoType = "groupByTraceSource" 38 | 39 | // Splits span logs into pseudospans grouped by traceSource. 40 | type GroupByTraceSourceVisitor struct { 41 | ShouldBeGrouped utilmarshal.StringFilter `json:"shouldBeGrouped"` 42 | } 43 | 44 | func (GroupByTraceSourceVisitor) Kind() string { return "GroupByTraceSourceVisitor" } 45 | 46 | func (visitor GroupByTraceSourceVisitor) Enter(tree *tftree.SpanTree, span *model.Span) tftree.TreeVisitor { 47 | pseudoType, hasPseudoType := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType) 48 | if hasPseudoType && pseudoType.AsString() == myPseudoType { 49 | // already grouped, don't recurse 50 | return visitor 51 | } 52 | 53 | remainingLogs := []model.Log{} 54 | 55 | index := map[string][]model.Log{} 56 | for _, log := range span.Logs { 57 | traceSource, hasTraceSource := model.KeyValues(log.Fields).FindByKey(zconstants.TraceSource) 58 | if hasTraceSource && visitor.ShouldBeGrouped.Matches(traceSource.AsString()) { 59 | index[traceSource.AsString()] = append(index[traceSource.AsString()], log) 60 | } else { 61 | remainingLogs = append(remainingLogs, log) 62 | } 63 | } 64 | 65 | span.Logs = remainingLogs 66 | 67 | for traceSource, logs := range index { 68 | newSpanId := model.SpanID(rand.Uint64()) 69 | newSpan := &model.Span{ 70 | TraceID: span.TraceID, 71 | SpanID: newSpanId, 72 | OperationName: traceSource, 73 | Flags: 0, 74 | StartTime: span.StartTime, 75 | Duration: span.Duration, 76 | Tags: []model.KeyValue{ 77 | { 78 | Key: zconstants.PseudoType, 79 | VType: model.StringType, 80 | VStr: myPseudoType, 81 | }, 82 | }, 83 | Logs: logs, 84 | Process: &model.Process{ 85 | ServiceName: traceSource, 86 | }, 87 | ProcessID: "1", 88 | } 89 | tree.Add(newSpan, span.SpanID) 90 | } 91 | 92 | return visitor 93 | } 94 | 95 | func (visitor GroupByTraceSourceVisitor) Exit(tree *tftree.SpanTree, span *model.Span) {} 96 | -------------------------------------------------------------------------------- /pkg/frontend/tf/defaults/step/object_tags.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tfstep 16 | 17 | import ( 18 | "github.com/jaegertracing/jaeger/model" 19 | 20 | tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config" 21 | tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree" 22 | "github.com/kubewharf/kelemetry/pkg/manager" 23 | "github.com/kubewharf/kelemetry/pkg/util/zconstants" 24 | ) 25 | 26 | func init() { 27 | manager.Global.ProvideListImpl( 28 | "tf-step/object-tags-visitor", 29 | manager.Ptr(&tfconfig.VisitorStep[ObjectTagsVisitor]{}), 30 | &manager.List[tfconfig.RegisteredStep]{}, 31 | ) 32 | } 33 | 34 | // Copy tags from child spans to the object. 35 | type ObjectTagsVisitor struct { 36 | ResourceTags []string `json:"resourceTags"` 37 | } 38 | 39 | func (ObjectTagsVisitor) Kind() string { return "ObjectTagsVisitor" } 40 | 41 | func (visitor ObjectTagsVisitor) Enter(tree *tftree.SpanTree, span *model.Span) tftree.TreeVisitor { 42 | if tagKv, isPseudo := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType); !isPseudo || 43 | tagKv.VStr != string(zconstants.PseudoTypeObject) { 44 | return visitor 45 | } 46 | if _, hasTag := model.KeyValues(span.Tags).FindByKey("resource"); !hasTag { 47 | return visitor 48 | } 49 | 50 | for _, resourceKey := range visitor.ResourceTags { 51 | _ = visitor.findTagRecursively(tree, span, resourceKey) 52 | } 53 | 54 | return visitor 55 | } 56 | 57 | func (visitor ObjectTagsVisitor) Exit(tree *tftree.SpanTree, span *model.Span) {} 58 | 59 | func (visitor ObjectTagsVisitor) findTagRecursively(tree *tftree.SpanTree, span *model.Span, tagKey string) model.KeyValue { 60 | if kv, hasTag := model.KeyValues(span.Tags).FindByKey(tagKey); hasTag { 61 | return kv 62 | } 63 | 64 | for childId := range tree.Children(span.SpanID) { 65 | childSpan := tree.Span(childId) 66 | { 67 | tagKv, isPseudo := model.KeyValues(childSpan.Tags).FindByKey(zconstants.PseudoType) 68 | if isPseudo && tagKv.VStr == string(zconstants.PseudoTypeObject) { 69 | // do not copy from another object 70 | continue 71 | } 72 | } 73 | 74 | kv := visitor.findTagRecursively(tree, childSpan, tagKey) 75 | if len(kv.Key) > 0 { 76 | span.Tags = append(span.Tags, kv) 77 | return kv 78 | } 79 | } 80 | return model.KeyValue{} 81 | } 82 | -------------------------------------------------------------------------------- /pkg/frontend/tf/defaults/step/prune_childless.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tfstep 16 | 17 | import ( 18 | "github.com/jaegertracing/jaeger/model" 19 | 20 | tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config" 21 | tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree" 22 | "github.com/kubewharf/kelemetry/pkg/manager" 23 | "github.com/kubewharf/kelemetry/pkg/util/zconstants" 24 | ) 25 | 26 | func init() { 27 | manager.Global.ProvideListImpl( 28 | "tf-step/prune-childless-visitor", 29 | manager.Ptr(&tfconfig.VisitorStep[PruneChildlessVisitor]{}), 30 | &manager.List[tfconfig.RegisteredStep]{}, 31 | ) 32 | } 33 | 34 | type PruneChildlessVisitor struct{} 35 | 36 | func (PruneChildlessVisitor) Kind() string { return "PruneChildlessVisitor" } 37 | 38 | func (visitor PruneChildlessVisitor) Enter(tree *tftree.SpanTree, span *model.Span) tftree.TreeVisitor { 39 | return visitor 40 | } 41 | 42 | // Prune in postorder traversal to recursively remove higher pseudospans without leaves. 43 | func (visitor PruneChildlessVisitor) Exit(tree *tftree.SpanTree, span *model.Span) { 44 | if _, isPseudo := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType); isPseudo { 45 | if len(tree.Children(span.SpanID)) == 0 && span.SpanID != tree.Root.SpanID { 46 | tree.Delete(span.SpanID) 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pkg/frontend/tf/defaults/step/prune_tags.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tfstep 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | 21 | "github.com/jaegertracing/jaeger/model" 22 | 23 | tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config" 24 | tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree" 25 | "github.com/kubewharf/kelemetry/pkg/manager" 26 | "github.com/kubewharf/kelemetry/pkg/util/zconstants" 27 | ) 28 | 29 | func init() { 30 | manager.Global.ProvideListImpl( 31 | "tf-step/replace-name-visitor", 32 | manager.Ptr(&tfconfig.VisitorStep[ReplaceNameVisitor]{}), 33 | &manager.List[tfconfig.RegisteredStep]{}, 34 | ) 35 | manager.Global.ProvideListImpl( 36 | "tf-step/prune-tags-visitor", 37 | manager.Ptr(&tfconfig.VisitorStep[PruneTagsVisitor]{}), 38 | &manager.List[tfconfig.RegisteredStep]{}, 39 | ) 40 | } 41 | 42 | type ReplaceNameVisitor struct{} 43 | 44 | func (ReplaceNameVisitor) Kind() string { return "ReplaceNameVisitor" } 45 | 46 | func (visitor ReplaceNameVisitor) Enter(tree *tftree.SpanTree, span *model.Span) tftree.TreeVisitor { 47 | for _, tag := range span.Tags { 48 | if tag.Key == zconstants.SpanName { 49 | span.OperationName = tag.VStr 50 | break 51 | } 52 | } 53 | 54 | return visitor 55 | } 56 | 57 | func (visitor ReplaceNameVisitor) Exit(tree *tftree.SpanTree, span *model.Span) {} 58 | 59 | type PruneTagsVisitor struct{} 60 | 61 | func (PruneTagsVisitor) Kind() string { return "PruneTagsVisitor" } 62 | 63 | func (visitor PruneTagsVisitor) Enter(tree *tftree.SpanTree, span *model.Span) tftree.TreeVisitor { 64 | span.Tags = removeZconstantKeys(span.Tags) 65 | 66 | for i := range span.Logs { 67 | log := &span.Logs[i] 68 | log.Fields = removeZconstantKeys(log.Fields) 69 | } 70 | 71 | // add the timestamp to assist viewing time on list 72 | if span == tree.Root { 73 | span.OperationName = fmt.Sprintf( 74 | "%s / %s..%s", 75 | span.OperationName, 76 | span.StartTime.Format("15:04:05"), 77 | span.StartTime.Add(span.Duration).Format("15:04:05"), 78 | ) 79 | } 80 | 81 | return visitor 82 | } 83 | 84 | func (visitor PruneTagsVisitor) Exit(tree *tftree.SpanTree, span *model.Span) {} 85 | 86 | func removeZconstantKeys(tags model.KeyValues) model.KeyValues { 87 | newTags := model.KeyValues{} 88 | for _, tag := range tags { 89 | if !strings.HasPrefix(tag.Key, zconstants.Prefix) { 90 | newTags = append(newTags, tag) 91 | } 92 | } 93 | return newTags 94 | } 95 | -------------------------------------------------------------------------------- /pkg/frontend/tf/transform.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package transform 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "time" 21 | 22 | "github.com/jaegertracing/jaeger/model" 23 | "github.com/sirupsen/logrus" 24 | "github.com/spf13/pflag" 25 | 26 | "github.com/kubewharf/kelemetry/pkg/frontend/extension" 27 | tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config" 28 | tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree" 29 | "github.com/kubewharf/kelemetry/pkg/manager" 30 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 31 | ) 32 | 33 | func init() { 34 | manager.Global.Provide("jaeger-transform", manager.Ptr(&Transformer{})) 35 | } 36 | 37 | type TransformerOptions struct{} 38 | 39 | func (options *TransformerOptions) Setup(fs *pflag.FlagSet) {} 40 | 41 | func (options *TransformerOptions) EnableFlag() *bool { return nil } 42 | 43 | type Transformer struct { 44 | options TransformerOptions 45 | Logger logrus.FieldLogger 46 | Configs tfconfig.Provider 47 | ExtensionFactory *manager.List[extension.ProviderFactory] 48 | } 49 | 50 | func (transformer *Transformer) Options() manager.Options { return &transformer.options } 51 | func (transformer *Transformer) Init() error { return nil } 52 | func (transformer *Transformer) Start(ctx context.Context) error { return nil } 53 | func (transformer *Transformer) Close(ctx context.Context) error { return nil } 54 | 55 | func (transformer *Transformer) Transform( 56 | ctx context.Context, 57 | trace *model.Trace, 58 | rootObject *utilobject.Key, 59 | configId tfconfig.Id, 60 | extensionProcessor ExtensionProcessor, 61 | start, end time.Time, 62 | ) error { 63 | if len(trace.Spans) == 0 { 64 | return fmt.Errorf("cannot transform empty trace") 65 | } 66 | 67 | config := transformer.Configs.GetById(configId) 68 | if config == nil { 69 | config = transformer.Configs.GetById(transformer.Configs.DefaultId()) 70 | } 71 | 72 | tree := tftree.NewSpanTree(trace.Spans) 73 | 74 | newSpans, err := extensionProcessor.ProcessExtensions(ctx, transformer, config.Extensions, trace.Spans, start, end) 75 | if err != nil { 76 | return fmt.Errorf("cannot prepare extension trace: %w", err) 77 | } 78 | 79 | newSpans = append(newSpans, tree.GetSpans()...) 80 | tree = tftree.NewSpanTree(newSpans) 81 | 82 | for _, step := range config.Steps { 83 | step.Run(tree) 84 | } 85 | 86 | trace.Spans = tree.GetSpans() 87 | 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /pkg/frontend/tracecache/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tracecache 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "time" 21 | 22 | "github.com/jaegertracing/jaeger/model" 23 | 24 | "github.com/kubewharf/kelemetry/pkg/manager" 25 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 26 | ) 27 | 28 | func init() { 29 | manager.Global.Provide("jaeger-trace-cache", manager.Ptr[Cache](&mux{ 30 | Mux: manager.NewMux("jaeger-trace-cache", false), 31 | })) 32 | } 33 | 34 | type Cache interface { 35 | Persist(ctx context.Context, entries []Entry) error 36 | Fetch(ctx context.Context, lowId uint64) (*EntryValue, error) 37 | } 38 | 39 | type Entry struct { 40 | LowId uint64 41 | Value EntryValue 42 | } 43 | 44 | type EntryValue struct { 45 | Identifiers []json.RawMessage `json:"identifiers"` 46 | StartTime time.Time `json:"startTime"` 47 | EndTime time.Time `json:"endTime"` 48 | RootObject *utilobject.Key `json:"rootObject"` 49 | 50 | Extensions []ExtensionCache `json:"extensions"` 51 | Spans []*model.Span `json:"spans"` 52 | } 53 | 54 | type ExtensionCache struct { 55 | ParentTrace model.TraceID `json:"parentTrace"` 56 | ParentSpan model.SpanID `json:"parentSpan"` 57 | ProviderKind string `json:"providerKind"` 58 | ProviderConfig json.RawMessage `json:"providerConfig"` 59 | CachedIdentifier json.RawMessage `json:"cachedIdentifier"` 60 | } 61 | 62 | type mux struct { 63 | *manager.Mux 64 | } 65 | 66 | func (mux *mux) Persist(ctx context.Context, entries []Entry) error { 67 | return mux.Impl().(Cache).Persist(ctx, entries) 68 | } 69 | 70 | func (mux *mux) Fetch(ctx context.Context, lowId uint64) (*EntryValue, error) { 71 | return mux.Impl().(Cache).Fetch(ctx, lowId) 72 | } 73 | -------------------------------------------------------------------------------- /pkg/frontend/tracecache/local/local.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package local 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "sync" 21 | 22 | "github.com/sirupsen/logrus" 23 | 24 | tracecache "github.com/kubewharf/kelemetry/pkg/frontend/tracecache" 25 | "github.com/kubewharf/kelemetry/pkg/manager" 26 | ) 27 | 28 | func init() { 29 | manager.Global.ProvideMuxImpl("jaeger-trace-cache/local", manager.Ptr(&localCache{ 30 | data: map[uint64]tracecache.EntryValue{}, 31 | }), tracecache.Cache.Persist) 32 | } 33 | 34 | var _ tracecache.Cache = &localCache{} 35 | 36 | type localCache struct { 37 | manager.MuxImplBase 38 | 39 | Logger logrus.FieldLogger 40 | 41 | data map[uint64]tracecache.EntryValue 42 | dataLock sync.RWMutex 43 | } 44 | 45 | func (_ *localCache) MuxImplName() (name string, isDefault bool) { return "local", true } 46 | 47 | func (cache *localCache) Options() manager.Options { return &manager.NoOptions{} } 48 | 49 | func (cache *localCache) Init() error { return nil } 50 | 51 | func (cache *localCache) Start(ctx context.Context) error { return nil } 52 | 53 | func (cache *localCache) Close(ctx context.Context) error { return nil } 54 | 55 | func (cache *localCache) Persist(ctx context.Context, entries []tracecache.Entry) error { 56 | cache.dataLock.Lock() 57 | defer cache.dataLock.Unlock() 58 | 59 | for _, entry := range entries { 60 | cache.data[entry.LowId] = entry.Value 61 | } 62 | 63 | return nil 64 | } 65 | 66 | func (cache *localCache) Fetch(ctx context.Context, lowId uint64) (*tracecache.EntryValue, error) { 67 | cache.dataLock.RLock() 68 | defer cache.dataLock.RUnlock() 69 | 70 | if value, exists := cache.data[lowId]; exists { 71 | return &value, nil 72 | } 73 | 74 | return nil, fmt.Errorf("No trace cache for key %x", lowId) 75 | } 76 | -------------------------------------------------------------------------------- /pkg/imports.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Default packages in the default build of kelemetry 16 | package kelemetry_pkg 17 | 18 | import ( 19 | _ "github.com/kubewharf/kelemetry/pkg/aggregator/aggregatorevent" 20 | _ "github.com/kubewharf/kelemetry/pkg/aggregator/eventdecorator/eventtagger" 21 | _ "github.com/kubewharf/kelemetry/pkg/aggregator/linker/job/local" 22 | _ "github.com/kubewharf/kelemetry/pkg/aggregator/linker/job/worker" 23 | _ "github.com/kubewharf/kelemetry/pkg/aggregator/objectspandecorator/resourcetagger" 24 | _ "github.com/kubewharf/kelemetry/pkg/aggregator/spancache/etcd" 25 | _ "github.com/kubewharf/kelemetry/pkg/aggregator/spancache/local" 26 | _ "github.com/kubewharf/kelemetry/pkg/aggregator/tracer/otel" 27 | _ "github.com/kubewharf/kelemetry/pkg/annotationlinker" 28 | _ "github.com/kubewharf/kelemetry/pkg/audit" 29 | _ "github.com/kubewharf/kelemetry/pkg/audit/consumer" 30 | _ "github.com/kubewharf/kelemetry/pkg/audit/dump" 31 | _ "github.com/kubewharf/kelemetry/pkg/audit/forward" 32 | _ "github.com/kubewharf/kelemetry/pkg/audit/mq/local" 33 | _ "github.com/kubewharf/kelemetry/pkg/audit/producer" 34 | _ "github.com/kubewharf/kelemetry/pkg/audit/webhook" 35 | _ "github.com/kubewharf/kelemetry/pkg/audit/webhook/clustername" 36 | _ "github.com/kubewharf/kelemetry/pkg/audit/webhook/clustername/address" 37 | _ "github.com/kubewharf/kelemetry/pkg/diff/api" 38 | _ "github.com/kubewharf/kelemetry/pkg/diff/cache/etcd" 39 | _ "github.com/kubewharf/kelemetry/pkg/diff/cache/local" 40 | _ "github.com/kubewharf/kelemetry/pkg/diff/controller" 41 | _ "github.com/kubewharf/kelemetry/pkg/diff/decorator" 42 | _ "github.com/kubewharf/kelemetry/pkg/event" 43 | _ "github.com/kubewharf/kelemetry/pkg/frontend" 44 | _ "github.com/kubewharf/kelemetry/pkg/frontend/backend/jaeger-storage" 45 | _ "github.com/kubewharf/kelemetry/pkg/frontend/clusterlist/options" 46 | _ "github.com/kubewharf/kelemetry/pkg/frontend/extension/httptrace" 47 | _ "github.com/kubewharf/kelemetry/pkg/frontend/extension/jaeger-storage" 48 | _ "github.com/kubewharf/kelemetry/pkg/frontend/http/redirect" 49 | _ "github.com/kubewharf/kelemetry/pkg/frontend/http/trace" 50 | _ "github.com/kubewharf/kelemetry/pkg/frontend/tf/config/file" 51 | _ "github.com/kubewharf/kelemetry/pkg/frontend/tf/defaults/modifier" 52 | _ "github.com/kubewharf/kelemetry/pkg/frontend/tf/defaults/step" 53 | _ "github.com/kubewharf/kelemetry/pkg/frontend/tracecache/etcd" 54 | _ "github.com/kubewharf/kelemetry/pkg/frontend/tracecache/local" 55 | _ "github.com/kubewharf/kelemetry/pkg/k8s/config/mapoption" 56 | _ "github.com/kubewharf/kelemetry/pkg/kelemetrix/consumer" 57 | _ "github.com/kubewharf/kelemetry/pkg/kelemetrix/defaults/quantities" 58 | _ "github.com/kubewharf/kelemetry/pkg/kelemetrix/defaults/tags" 59 | _ "github.com/kubewharf/kelemetry/pkg/metrics/noop" 60 | _ "github.com/kubewharf/kelemetry/pkg/metrics/prometheus" 61 | _ "github.com/kubewharf/kelemetry/pkg/ownerlinker" 62 | ) 63 | -------------------------------------------------------------------------------- /pkg/k8s/config/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Provide connection details to a Kubernetes cluster. 16 | package k8sconfig 17 | 18 | import ( 19 | "time" 20 | 21 | "k8s.io/client-go/rest" 22 | 23 | "github.com/kubewharf/kelemetry/pkg/manager" 24 | "github.com/kubewharf/kelemetry/pkg/metrics" 25 | ) 26 | 27 | func init() { 28 | manager.Global.Provide("kube-config", manager.Ptr[Config](&mux{ 29 | Mux: manager.NewMux("kube-config", false), 30 | })) 31 | } 32 | 33 | type Config interface { 34 | // TargetName returns the name of thetarget cluster. 35 | // ProvideTarget() is equivalent to Provide(TargetName()). 36 | TargetName() string 37 | 38 | // Provide returns the rest config for a named cluster. 39 | // Returns nil if the cluster is not available. 40 | // Mainly used when the main cluster contains references to other clusters. 41 | Provide(clusterName string) *Cluster 42 | } 43 | 44 | type Cluster struct { 45 | Config *rest.Config 46 | DefaultRequestTimeout time.Duration 47 | UseOldResourceVersion bool 48 | } 49 | 50 | func (cluster *Cluster) ChooseResourceVersion(oldRv string, newRv *string) (string, error) { 51 | useOld := false 52 | if cluster != nil { 53 | useOld = cluster.UseOldResourceVersion 54 | } 55 | 56 | if useOld { 57 | return oldRv, nil 58 | } 59 | 60 | if newRv != nil { 61 | return *newRv, nil 62 | } 63 | 64 | return "", metrics.MakeLabeledError("NoNewRv") 65 | } 66 | 67 | type mux struct { 68 | *manager.Mux 69 | } 70 | 71 | func (mux *mux) TargetName() string { 72 | return mux.Impl().(Config).TargetName() 73 | } 74 | 75 | func (mux *mux) Provide(clusterName string) *Cluster { 76 | return mux.Impl().(Config).Provide(clusterName) 77 | } 78 | 79 | type MockConfig struct { 80 | TargetClusterName string 81 | Clusters map[string]*Cluster 82 | } 83 | 84 | func (c *MockConfig) TargetName() string { return c.TargetClusterName } 85 | 86 | func (c *MockConfig) Provide(clusterName string) *Cluster { return c.Clusters[clusterName] } 87 | -------------------------------------------------------------------------------- /pkg/k8s/objectcache/objectcache_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package objectcache_test 16 | 17 | import ( 18 | "context" 19 | "testing" 20 | 21 | "github.com/sirupsen/logrus" 22 | "github.com/stretchr/testify/assert" 23 | corev1 "k8s.io/api/core/v1" 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 26 | "k8s.io/apimachinery/pkg/runtime" 27 | "k8s.io/utils/clock" 28 | 29 | "github.com/kubewharf/kelemetry/pkg/k8s" 30 | "github.com/kubewharf/kelemetry/pkg/k8s/objectcache" 31 | "github.com/kubewharf/kelemetry/pkg/metrics" 32 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 33 | ) 34 | 35 | func TestGet(t *testing.T) { 36 | assert := assert.New(t) 37 | 38 | clock := clock.RealClock{} // the clock is unused 39 | metricsClient, metricsOutput := metrics.NewMock(clock) 40 | 41 | cache := &objectcache.ObjectCache{ 42 | Logger: logrus.New(), 43 | Clock: clock, 44 | Clients: &k8s.MockClients{ 45 | TargetClusterName: "test-cluster", 46 | Clients: map[string]*k8s.MockClient{ 47 | "test-cluster": { 48 | Name: "test-cluster", 49 | Objects: []runtime.Object{ 50 | &corev1.ConfigMap{ 51 | TypeMeta: metav1.TypeMeta{ 52 | APIVersion: "v1", 53 | Kind: "ConfigMap", 54 | }, 55 | ObjectMeta: metav1.ObjectMeta{ 56 | Name: "test-cm", 57 | Namespace: "default", 58 | }, 59 | Data: map[string]string{"foo": "bar"}, 60 | }, 61 | }, 62 | }, 63 | }, 64 | }, 65 | Metrics: metricsClient, 66 | DiffCache: nil, // TODO 67 | CacheRequestMetric: metrics.New[*objectcache.CacheRequestMetric](metricsClient), 68 | } 69 | 70 | assert.NoError(cache.Init()) 71 | 72 | for i := 0; i < 2; i++ { 73 | uns, err := cache.Get(context.Background(), utilobject.VersionedKey{ 74 | Key: utilobject.Key{ 75 | Cluster: "test-cluster", 76 | Namespace: "default", 77 | Name: "test-cm", 78 | Group: corev1.GroupName, 79 | Resource: "configmaps", 80 | }, 81 | Version: corev1.SchemeGroupVersion.Version, 82 | }) 83 | assert.NoError(err) 84 | 85 | fooValue, fooExists, err := unstructured.NestedString(uns.Object, "data", "foo") 86 | assert.NoError(err) 87 | 88 | if i == 0 { 89 | assert.True(fooExists) 90 | assert.Equal("bar", fooValue) 91 | 92 | penetrations := metricsOutput.Get("object_cache_request", map[string]string{ 93 | "cluster": "test-cluster", 94 | "error": "Apiserver", 95 | "hit": "false", 96 | }) 97 | assert.Equal(float64(1), penetrations.Int, metricsOutput.PrintAll()) 98 | } else { 99 | assert.True(fooExists) 100 | assert.Equal("bar", fooValue) 101 | 102 | hits := metricsOutput.Get("object_cache_request", map[string]string{ 103 | "cluster": "test-cluster", 104 | "error": "nil", 105 | "hit": "true", 106 | }) 107 | assert.Equal(float64(1), hits.Int, metricsOutput.PrintAll()) 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /pkg/kelemetrix/base_quantity_comp.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package kelemetrix 16 | 17 | import ( 18 | "github.com/sirupsen/logrus" 19 | 20 | "github.com/kubewharf/kelemetry/pkg/audit" 21 | "github.com/kubewharf/kelemetry/pkg/manager" 22 | ) 23 | 24 | type BaseQuantityDef interface { 25 | Name() string 26 | Type() MetricType 27 | DefaultEnable() bool 28 | 29 | Quantify(message *audit.Message) (float64, bool, error) 30 | } 31 | 32 | type BaseQuantifier[T BaseQuantityDef] struct { 33 | manager.BaseComponent 34 | Logger logrus.FieldLogger 35 | Def T `managerRecurse:""` 36 | } 37 | 38 | func NewBaseQuantifierForTest[T BaseQuantityDef](def T) *BaseQuantifier[T] { 39 | return &BaseQuantifier[T]{Logger: logrus.New(), Def: def} 40 | } 41 | 42 | func (q *BaseQuantifier[T]) Name() string { return q.Def.Name() } 43 | func (q *BaseQuantifier[T]) Type() MetricType { return q.Def.Type() } 44 | func (q *BaseQuantifier[T]) Quantify(message *audit.Message) (float64, bool) { 45 | if value, hasValue, err := q.Def.Quantify(message); err != nil { 46 | q.Logger.WithError(err).Error("error generating quantity") 47 | return 0, false 48 | } else { 49 | return value, hasValue 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /pkg/kelemetrix/config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package config 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | 22 | "github.com/pelletier/go-toml/v2" 23 | "github.com/spf13/pflag" 24 | 25 | "github.com/kubewharf/kelemetry/pkg/manager" 26 | ) 27 | 28 | func init() { 29 | manager.Global.Provide("kelemetrix-config", manager.Ptr[Provider](&comp{})) 30 | } 31 | 32 | type Provider interface { 33 | Get() *Config 34 | } 35 | 36 | type MockProvider struct { 37 | Config *Config 38 | } 39 | 40 | func (provider *MockProvider) Get() *Config { return provider.Config } 41 | 42 | type Config struct { 43 | Metrics []Metric 44 | } 45 | 46 | type Metric struct { 47 | Name string 48 | Quantifier string 49 | Tags []string 50 | TagFilters []TagFilter 51 | QuantityFilters []QuantityFilter 52 | } 53 | 54 | type TagFilter struct { 55 | Tag string 56 | OneOf []string 57 | IsRegex bool 58 | Negate bool 59 | } 60 | 61 | type QuantityFilter struct { 62 | Quantity string 63 | Operator QuantityOperator 64 | Threshold float64 65 | } 66 | 67 | type QuantityOperator string 68 | 69 | const ( 70 | QuantityOperatorLess QuantityOperator = "<" 71 | QuantityOperatorGreater QuantityOperator = ">" 72 | QuantityOperatorLessEq QuantityOperator = "<=" 73 | QuantityOperatorGreaterEq QuantityOperator = ">=" 74 | ) 75 | 76 | type compOptions struct { 77 | path string 78 | } 79 | 80 | func (options *compOptions) Setup(fs *pflag.FlagSet) { 81 | fs.StringVar(&options.path, "kelemetrix-config-path", "hack/kelemetrix.toml", "path to kelemetrix config") 82 | } 83 | 84 | func (options *compOptions) EnableFlag() *bool { return nil } 85 | 86 | type comp struct { 87 | options compOptions 88 | 89 | value Config 90 | } 91 | 92 | func (config *comp) Options() manager.Options { return &config.options } 93 | 94 | func (config *comp) Init() error { 95 | file, err := os.Open(config.options.path) 96 | if err != nil { 97 | return fmt.Errorf("cannot open config file %s: %w", config.options.path, err) 98 | } 99 | 100 | if err := toml.NewDecoder(file).DisallowUnknownFields().Decode(&config.value); err != nil { 101 | return fmt.Errorf("error decoding config file %s: %w", config.options.path, err) 102 | } 103 | 104 | return nil 105 | } 106 | 107 | func (config *comp) Start(ctx context.Context) error { return nil } 108 | func (config *comp) Close(ctx context.Context) error { return nil } 109 | 110 | func (config *comp) Get() *Config { 111 | return &config.value 112 | } 113 | -------------------------------------------------------------------------------- /pkg/kelemetrix/defaults/quantities/mq_latency.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package defaultquantities 16 | 17 | import ( 18 | "time" 19 | 20 | auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" 21 | 22 | "github.com/kubewharf/kelemetry/pkg/audit" 23 | "github.com/kubewharf/kelemetry/pkg/kelemetrix" 24 | "github.com/kubewharf/kelemetry/pkg/manager" 25 | ) 26 | 27 | func init() { 28 | manager.Global.ProvideListImpl( 29 | "kelemetrix-quantity-mq-latency", 30 | manager.Ptr(&kelemetrix.BaseQuantifier[MqLatency]{}), 31 | &manager.List[kelemetrix.Quantifier]{}, 32 | ) 33 | } 34 | 35 | type MqLatency struct{} 36 | 37 | func (MqLatency) Name() string { return "mq_latency" } 38 | func (MqLatency) Type() kelemetrix.MetricType { return kelemetrix.MetricTypeHistogram } 39 | func (MqLatency) DefaultEnable() bool { return true } 40 | 41 | func (MqLatency) Quantify(message *audit.Message) (float64, bool, error) { 42 | return float64(time.Since(message.StageTimestamp.Time).Nanoseconds()), message.Stage == auditv1.StageResponseComplete, nil 43 | } 44 | -------------------------------------------------------------------------------- /pkg/kelemetrix/defaults/quantities/request_count.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package defaultquantities 16 | 17 | import ( 18 | auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" 19 | 20 | "github.com/kubewharf/kelemetry/pkg/audit" 21 | "github.com/kubewharf/kelemetry/pkg/kelemetrix" 22 | "github.com/kubewharf/kelemetry/pkg/manager" 23 | ) 24 | 25 | func init() { 26 | manager.Global.ProvideListImpl( 27 | "kelemetrix-quantity-request-count", 28 | manager.Ptr(&kelemetrix.BaseQuantifier[RequestCount]{}), 29 | &manager.List[kelemetrix.Quantifier]{}, 30 | ) 31 | } 32 | 33 | type RequestCount struct{} 34 | 35 | func (RequestCount) Name() string { return "request_count" } 36 | func (RequestCount) Type() kelemetrix.MetricType { return kelemetrix.MetricTypeCount } 37 | func (RequestCount) DefaultEnable() bool { return true } 38 | 39 | func (RequestCount) Quantify(message *audit.Message) (float64, bool, error) { 40 | if message.Stage != auditv1.StageResponseComplete { 41 | return 0, false, nil 42 | } 43 | return 1, true, nil 44 | } 45 | -------------------------------------------------------------------------------- /pkg/kelemetrix/defaults/quantities/request_latency.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package defaultquantities 16 | 17 | import ( 18 | "fmt" 19 | "net/url" 20 | "strconv" 21 | 22 | auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" 23 | 24 | "github.com/kubewharf/kelemetry/pkg/audit" 25 | k8sconfig "github.com/kubewharf/kelemetry/pkg/k8s/config" 26 | "github.com/kubewharf/kelemetry/pkg/kelemetrix" 27 | "github.com/kubewharf/kelemetry/pkg/manager" 28 | ) 29 | 30 | func init() { 31 | manager.Global.ProvideListImpl( 32 | "kelemetrix-quantity-request-latency", 33 | manager.Ptr(&kelemetrix.BaseQuantifier[RequestLatency]{}), 34 | &manager.List[kelemetrix.Quantifier]{}, 35 | ) 36 | manager.Global.ProvideListImpl( 37 | "kelemetrix-quantity-request-latency-ratio", 38 | manager.Ptr(&kelemetrix.BaseQuantifier[RequestLatencyRatio]{}), 39 | &manager.List[kelemetrix.Quantifier]{}, 40 | ) 41 | } 42 | 43 | type RequestLatency struct{} 44 | 45 | func (RequestLatency) Name() string { return "request_latency" } 46 | func (RequestLatency) Type() kelemetrix.MetricType { return kelemetrix.MetricTypeHistogram } 47 | func (RequestLatency) DefaultEnable() bool { return true } 48 | 49 | func (RequestLatency) Quantify(message *audit.Message) (float64, bool, error) { 50 | if message.Stage != auditv1.StageResponseComplete { 51 | return 0, false, nil 52 | } 53 | return float64(message.StageTimestamp.Time.Sub(message.RequestReceivedTimestamp.Time).Nanoseconds()), true, nil 54 | } 55 | 56 | type RequestLatencyRatio struct { 57 | Config k8sconfig.Config 58 | } 59 | 60 | func (RequestLatencyRatio) Name() string { return "request_latency_ratio" } 61 | func (RequestLatencyRatio) Type() kelemetrix.MetricType { return kelemetrix.MetricTypeSummary } 62 | func (RequestLatencyRatio) DefaultEnable() bool { return true } 63 | 64 | func (r RequestLatencyRatio) Quantify(message *audit.Message) (float64, bool, error) { 65 | if message.Stage != auditv1.StageResponseComplete { 66 | return 0, false, nil 67 | } 68 | 69 | url, err := url.Parse(message.RequestURI) 70 | if err != nil { 71 | return 0, false, fmt.Errorf("cannot parse request URI: %w", err) 72 | } 73 | 74 | latency := message.StageTimestamp.Time.Sub(message.RequestReceivedTimestamp.Time) 75 | requestTimeoutStrings := url.Query()["timeoutSeconds"] 76 | 77 | var requestTimeout float64 78 | if len(requestTimeoutStrings) > 0 { 79 | requestTimeout, err = strconv.ParseFloat(requestTimeoutStrings[0], 64) 80 | if err != nil { 81 | return 0, false, fmt.Errorf("invalid timeoutSeconds value %q", requestTimeoutStrings[0]) 82 | } 83 | } else { 84 | cluster := r.Config.Provide(message.Cluster) 85 | if cluster != nil { 86 | requestTimeout = cluster.DefaultRequestTimeout.Seconds() 87 | } else { 88 | // unknown cluster 89 | requestTimeout = 60. 90 | } 91 | } 92 | 93 | ratio := requestTimeout / latency.Seconds() 94 | return ratio, true, nil 95 | } 96 | -------------------------------------------------------------------------------- /pkg/kelemetrix/registry.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package kelemetrix 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/kubewharf/kelemetry/pkg/audit" 21 | "github.com/kubewharf/kelemetry/pkg/manager" 22 | ) 23 | 24 | func init() { 25 | manager.Global.Provide("kelemetrix-registry", manager.Ptr(NewRegistry())) 26 | } 27 | 28 | type Registry struct { 29 | TagProviderFactories *manager.List[TagProviderFactory] 30 | TagProviders []TagProvider 31 | TagProviderNameIndex map[string]int 32 | 33 | Quantifiers *manager.List[Quantifier] 34 | QuantifierNameIndex map[string]int 35 | } 36 | 37 | func NewRegistry() *Registry { 38 | return &Registry{ 39 | TagProviders: []TagProvider{}, 40 | TagProviderNameIndex: make(map[string]int), 41 | QuantifierNameIndex: make(map[string]int), 42 | } 43 | } 44 | 45 | func NewMockRegistry( 46 | tagProviders []TagProvider, 47 | quantifiers []Quantifier, 48 | ) *Registry { 49 | registry := NewRegistry() 50 | for _, tagProvider := range tagProviders { 51 | registry.addTagProvider(tagProvider) 52 | } 53 | registry.Quantifiers = &manager.List[Quantifier]{ 54 | Impls: quantifiers, 55 | } 56 | for i, quantifier := range quantifiers { 57 | registry.QuantifierNameIndex[quantifier.Name()] = i 58 | } 59 | 60 | return registry 61 | } 62 | 63 | func (registry *Registry) Options() manager.Options { return &manager.NoOptions{} } 64 | 65 | func (registry *Registry) Init() error { 66 | for _, factory := range registry.TagProviderFactories.Impls { 67 | for _, provider := range factory.GetTagProviders() { 68 | registry.addTagProvider(provider) 69 | } 70 | } 71 | 72 | for i, quantifier := range registry.Quantifiers.Impls { 73 | registry.QuantifierNameIndex[quantifier.Name()] = i 74 | } 75 | 76 | return nil 77 | } 78 | 79 | func (registry *Registry) Start(ctx context.Context) error { return nil } 80 | func (registry *Registry) Close(ctx context.Context) error { return nil } 81 | 82 | type TagProviderFactory interface { 83 | GetTagProviders() []TagProvider 84 | } 85 | 86 | type TagProvider interface { 87 | TagNames() []string 88 | 89 | ProvideValues(message *audit.Message, slice []string) 90 | } 91 | 92 | type MetricType uint8 93 | 94 | const ( 95 | MetricTypeCount = MetricType(iota) 96 | MetricTypeHistogram 97 | MetricTypeSummary 98 | ) 99 | 100 | type Quantifier interface { 101 | Name() string 102 | Type() MetricType 103 | Quantify(message *audit.Message) (float64, bool) 104 | } 105 | 106 | func (registry *Registry) addTagProvider(provider TagProvider) { 107 | providerIndex := len(registry.TagProviders) 108 | registry.TagProviders = append(registry.TagProviders, provider) 109 | 110 | for _, name := range provider.TagNames() { 111 | registry.TagProviderNameIndex[name] = providerIndex 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /pkg/metrics/noop/noop.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package metricsnoop 16 | 17 | import ( 18 | "context" 19 | "time" 20 | 21 | "github.com/kubewharf/kelemetry/pkg/manager" 22 | "github.com/kubewharf/kelemetry/pkg/metrics" 23 | ) 24 | 25 | func init() { 26 | manager.Global.ProvideMuxImpl("metrics/noop", manager.Ptr(&noop{}), func(metrics.Client) {}) 27 | } 28 | 29 | type noop struct { 30 | manager.MuxImplBase 31 | } 32 | 33 | var _ metrics.Impl = &noop{} 34 | 35 | func (_ *noop) MuxImplName() (name string, isDefault bool) { return "noop", true } 36 | 37 | func (client *noop) Options() manager.Options { 38 | return &manager.NoOptions{} 39 | } 40 | 41 | func (client *noop) Init() error { return nil } 42 | 43 | func (client *noop) Start(ctx context.Context) error { return nil } 44 | 45 | func (client *noop) Close(ctx context.Context) error { return nil } 46 | 47 | func (client *noop) New(name string, tagNames []string) metrics.MetricImpl { 48 | return metric{} 49 | } 50 | 51 | type metric struct{} 52 | 53 | func (metric metric) Count(value float64, tags []string) {} 54 | 55 | func (metric metric) Histogram(value float64, tags []string) {} 56 | 57 | func (metric metric) Summary(value float64, tags []string) {} 58 | 59 | func (metric metric) Gauge(value float64, tags []string) {} 60 | 61 | func (metric metric) Defer(start time.Time, tags []string) {} 62 | -------------------------------------------------------------------------------- /pkg/util/cache/ttl.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "context" 19 | "sync" 20 | "time" 21 | 22 | "github.com/sirupsen/logrus" 23 | "k8s.io/utils/clock" 24 | 25 | "github.com/kubewharf/kelemetry/pkg/util/channel" 26 | "github.com/kubewharf/kelemetry/pkg/util/shutdown" 27 | ) 28 | 29 | // TtlOnce is a cache where new insertions do not overwrite the old insertion. 30 | type TtlOnce struct { 31 | ttl time.Duration 32 | clock clock.Clock 33 | wakeupCh chan struct{} 34 | 35 | lock sync.RWMutex 36 | cleanupQueue *channel.Deque[cleanupEntry] 37 | data map[string]any 38 | } 39 | 40 | type cleanupEntry struct { 41 | key string 42 | expiry time.Time 43 | } 44 | 45 | func NewTtlOnce(ttl time.Duration, clock clock.Clock) *TtlOnce { 46 | return &TtlOnce{ 47 | ttl: ttl, 48 | clock: clock, 49 | wakeupCh: make(chan struct{}), 50 | cleanupQueue: channel.NewDeque[cleanupEntry](16), 51 | data: map[string]any{}, 52 | } 53 | } 54 | 55 | func (cache *TtlOnce) Add(key string, value any) { 56 | cache.lock.Lock() 57 | defer cache.lock.Unlock() 58 | 59 | if _, exists := cache.data[key]; !exists { 60 | cache.data[key] = value 61 | expiry := cache.clock.Now().Add(cache.ttl) 62 | cache.cleanupQueue.LockedPushBack(cleanupEntry{key: key, expiry: expiry}) 63 | select { 64 | case cache.wakeupCh <- struct{}{}: 65 | default: 66 | } 67 | } 68 | } 69 | 70 | func (cache *TtlOnce) Get(key string) (any, bool) { 71 | cache.lock.RLock() 72 | defer cache.lock.RUnlock() 73 | 74 | value, ok := cache.data[key] 75 | return value, ok 76 | } 77 | 78 | func (cache *TtlOnce) Size() int { 79 | cache.lock.RLock() 80 | defer cache.lock.RUnlock() 81 | 82 | return len(cache.data) 83 | } 84 | 85 | func (cache *TtlOnce) RunCleanupLoop(ctx context.Context, logger logrus.FieldLogger) { 86 | defer shutdown.RecoverPanic(logger) 87 | 88 | for { 89 | wakeup := cache.wakeupCh 90 | var nextExpiryCh <-chan time.Time 91 | if expiry, hasNext := cache.peekExpiry(); hasNext { 92 | nextExpiryCh = cache.clock.After(expiry.Sub(cache.clock.Now()) + time.Second) // +1s to mitigate race conditions 93 | wakeup = nil 94 | } 95 | 96 | select { 97 | case <-ctx.Done(): 98 | return 99 | case <-wakeup: 100 | continue 101 | case <-nextExpiryCh: 102 | cache.doCleanup() 103 | } 104 | } 105 | } 106 | 107 | func (cache *TtlOnce) doCleanup() { 108 | cache.lock.Lock() 109 | defer cache.lock.Unlock() 110 | 111 | for { 112 | if entry, hasEntry := cache.cleanupQueue.LockedPeekFront(); hasEntry { 113 | if entry.expiry.Before(cache.clock.Now()) { 114 | cache.cleanupQueue.LockedPopFront() 115 | delete(cache.data, entry.key) 116 | continue 117 | } 118 | } 119 | 120 | break 121 | } 122 | } 123 | 124 | func (cache *TtlOnce) peekExpiry() (expiry time.Time, found bool) { 125 | cache.lock.RLock() 126 | defer cache.lock.RUnlock() 127 | 128 | if entry, hasEntry := cache.cleanupQueue.LockedPeekFront(); hasEntry { 129 | expiry = entry.expiry 130 | found = true 131 | } 132 | 133 | return expiry, found 134 | } 135 | -------------------------------------------------------------------------------- /pkg/util/channel/channel_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package channel_test 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/stretchr/testify/assert" 21 | 22 | "github.com/kubewharf/kelemetry/pkg/util/channel" 23 | ) 24 | 25 | func TestOverflow(t *testing.T) { 26 | assert := assert.New(t) 27 | 28 | uq := channel.NewUnboundedQueue[int](3) 29 | recv := []any{} 30 | 31 | uq.Send(int(1)) 32 | recv = append(recv, <-uq.Receiver()) 33 | 34 | uq.Send(int(2)) 35 | uq.Send(int(3)) 36 | uq.Send(int(4)) 37 | recv = append(recv, <-uq.Receiver()) 38 | recv = append(recv, <-uq.Receiver()) 39 | recv = append(recv, <-uq.Receiver()) 40 | 41 | uq.Send(int(5)) 42 | uq.Send(int(6)) 43 | uq.Send(int(7)) 44 | uq.Send(int(8)) 45 | uq.Send(int(9)) 46 | recv = append(recv, <-uq.Receiver()) 47 | recv = append(recv, <-uq.Receiver()) 48 | recv = append(recv, <-uq.Receiver()) 49 | recv = append(recv, <-uq.Receiver()) 50 | recv = append(recv, <-uq.Receiver()) 51 | 52 | for i, v := range recv { 53 | assert.Equal(i, v.(int)-1) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pkg/util/errors/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package errors 16 | 17 | import ( 18 | goerrors "errors" 19 | ) 20 | 21 | var ( 22 | New = goerrors.New 23 | Unwrap = goerrors.Unwrap 24 | Is = goerrors.Is 25 | As = goerrors.As 26 | ) 27 | 28 | type labeled struct { 29 | unwrap error 30 | key string 31 | value any 32 | } 33 | 34 | func Label(err error, key string, value any) error { 35 | return labeled{unwrap: err, key: key, value: value} 36 | } 37 | 38 | func (err labeled) Error() string { 39 | return err.unwrap.Error() 40 | } 41 | 42 | func (err labeled) Unwrap() error { 43 | return err.unwrap 44 | } 45 | 46 | func GetNearestLabel(err error, key string) (any, bool) { 47 | if err, ok := err.(labeled); ok && err.key == key { 48 | return err.value, true 49 | } 50 | 51 | if inner := goerrors.Unwrap(err); inner != nil { 52 | return GetNearestLabel(inner, key) 53 | } 54 | 55 | return "", false 56 | } 57 | 58 | func GetDeepestLabel(err error, key string) (any, bool) { 59 | if inner := goerrors.Unwrap(err); inner != nil { 60 | if value, ok := GetDeepestLabel(inner, key); ok { 61 | return value, true 62 | } 63 | } 64 | 65 | if err, ok := err.(labeled); ok && err.key == key { 66 | return err.value, true 67 | } 68 | 69 | return "", false 70 | } 71 | 72 | // GetLabels returns labels from the nearest to the deepest. 73 | func GetLabels(err error, key string) []any { 74 | var out []any 75 | 76 | for err != nil { 77 | if err, ok := err.(labeled); ok { 78 | out = append(out, err.value) 79 | } 80 | 81 | err = Unwrap(err) 82 | } 83 | 84 | return out 85 | } 86 | -------------------------------------------------------------------------------- /pkg/util/errors/errors_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package errors_test 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/stretchr/testify/assert" 21 | 22 | "github.com/kubewharf/kelemetry/pkg/util/errors" 23 | ) 24 | 25 | func TestGetLabels(t *testing.T) { 26 | assert := assert.New(t) 27 | 28 | err := errors.New("base") 29 | 30 | err = errors.Label(err, "foo", "bar") 31 | err = errors.Label(err, "foo", "qux") 32 | 33 | labels := errors.GetLabels(err, "foo") 34 | assert.EqualValues([]any{"qux", "bar"}, labels) 35 | } 36 | -------------------------------------------------------------------------------- /pkg/util/filter/object_filter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package filter 16 | 17 | import ( 18 | "github.com/dlclark/regexp2" 19 | 20 | utilobject "github.com/kubewharf/kelemetry/pkg/util/object" 21 | ) 22 | 23 | type ObjectFilters struct { 24 | Cluster Regex `json:"cluster"` 25 | Group Regex `json:"group"` 26 | Version Regex `json:"version"` 27 | Resource Regex `json:"resource"` 28 | Namespace Regex `json:"namespace"` 29 | Name Regex `json:"name"` 30 | } 31 | 32 | type Regex struct { 33 | // golang regex not support (?!..), use third party regular engine for go 34 | Pattern *regexp2.Regexp 35 | } 36 | 37 | func (regex *Regex) UnmarshalText(text []byte) (err error) { 38 | regex.Pattern, err = regexp2.Compile(string(text), 0) 39 | return err 40 | } 41 | 42 | func (regex *Regex) MatchString(s string) bool { 43 | match, _ := regex.Pattern.MatchString(s) 44 | return match 45 | } 46 | 47 | func (f *ObjectFilters) Check(object utilobject.VersionedKey) bool { 48 | if f.Cluster.Pattern != nil && !f.Cluster.MatchString(object.Cluster) { 49 | return false 50 | } 51 | 52 | if f.Group.Pattern != nil && !f.Group.MatchString(object.Group) { 53 | return false 54 | } 55 | 56 | if f.Version.Pattern != nil && !f.Version.MatchString(object.Version) { 57 | return false 58 | } 59 | 60 | if f.Resource.Pattern != nil && !f.Resource.MatchString(object.Resource) { 61 | return false 62 | } 63 | 64 | if f.Namespace.Pattern != nil && !f.Namespace.MatchString(object.Namespace) { 65 | return false 66 | } 67 | 68 | if f.Name.Pattern != nil && !f.Name.MatchString(object.Name) { 69 | return false 70 | } 71 | 72 | return true 73 | } 74 | 75 | type TagFilters map[string]Regex 76 | 77 | func (f TagFilters) Check(tags map[string]string) bool { 78 | for key, filter := range f { 79 | val := tags[key] 80 | if filter.Pattern != nil && !filter.MatchString(val) { 81 | return false 82 | } 83 | } 84 | 85 | return true 86 | } 87 | -------------------------------------------------------------------------------- /pkg/util/informer/swap_map.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package informerutil 16 | 17 | import ( 18 | "sync" 19 | ) 20 | 21 | // SwapMap is a thread-safe map that provides atomic swapping results, 22 | // optimized to be used as an informer backend with no read operations. 23 | type SwapMap[K comparable, V any] struct { 24 | mutex sync.Mutex // not RWMutex, because there is no read-only access. 25 | data map[K]V 26 | } 27 | 28 | func NewSwapMap[K comparable, V any](capacity int) *SwapMap[K, V] { 29 | return &SwapMap[K, V]{ 30 | data: make(map[K]V, capacity), 31 | } 32 | } 33 | 34 | func (m *SwapMap[K, V]) Swap(key K, newValue V, hasNewValue bool) SwapResult[V] { 35 | return m.SwapIf(key, newValue, hasNewValue, func(_, _ V) bool { return true }) 36 | } 37 | 38 | // SwapIf swaps newValue with m[key] unless both values exist and condition(m[key], newValue) is false. 39 | func (m *SwapMap[K, V]) SwapIf( 40 | key K, 41 | newValue V, 42 | hasNewValue bool, 43 | shouldSwap func(oldValue, newValue V) bool, 44 | ) SwapResult[V] { 45 | m.mutex.Lock() 46 | defer m.mutex.Unlock() 47 | 48 | return m.lockedSwapIf(key, newValue, hasNewValue, shouldSwap) 49 | } 50 | 51 | func (m *SwapMap[K, V]) lockedSwapIf( 52 | key K, 53 | newValue V, 54 | hasNewValue bool, 55 | condition func(oldValue, newValue V) bool, 56 | ) SwapResult[V] { 57 | oldValue, hasOldValue := m.data[key] 58 | if hasOldValue && hasNewValue { 59 | if condition(oldValue, newValue) { 60 | m.data[key] = newValue 61 | return SwapResult[V]{ 62 | Kind: SwapResultKindReplace, 63 | OldValue: oldValue, 64 | NewValue: newValue, 65 | } 66 | } else { 67 | return SwapResult[V]{ 68 | Kind: SwapResultKindNoop, 69 | } 70 | } 71 | } else if hasOldValue { 72 | delete(m.data, key) 73 | return SwapResult[V]{ 74 | Kind: SwapResultKindRemove, 75 | OldValue: oldValue, 76 | } 77 | } else if hasNewValue { 78 | m.data[key] = newValue 79 | return SwapResult[V]{ 80 | Kind: SwapResultKindAdd, 81 | NewValue: newValue, 82 | } 83 | } else { 84 | return SwapResult[V]{ 85 | Kind: SwapResultKindNoop, 86 | } 87 | } 88 | } 89 | 90 | func SwapMapReplace[K comparable, V any, U any](m *SwapMap[K, V], values map[K]U, transformValue func(U) V) map[K]SwapResult[V] { 91 | m.mutex.Lock() 92 | defer m.mutex.Unlock() 93 | 94 | output := map[K]SwapResult[V]{} 95 | 96 | for key, value := range m.data { 97 | if _, retain := values[key]; !retain { 98 | output[key] = SwapResult[V]{ 99 | Kind: SwapResultKindRemove, 100 | OldValue: value, 101 | } 102 | delete(m.data, key) 103 | } 104 | } 105 | 106 | for key, value := range values { 107 | output[key] = m.lockedSwapIf(key, transformValue(value), true, func(_, _ V) bool { return true }) 108 | } 109 | 110 | return output 111 | } 112 | 113 | type SwapResult[V any] struct { 114 | Kind SwapResultKind 115 | // OldValue is the original value if Kind is Remove or Replace 116 | OldValue V 117 | // NewValue is the new value if Kind is Add or Replace 118 | NewValue V 119 | } 120 | 121 | type SwapResultKind uint8 122 | 123 | const ( 124 | SwapResultKindNoop SwapResultKind = iota 125 | SwapResultKindAdd 126 | SwapResultKindRemove 127 | SwapResultKindReplace 128 | ) 129 | -------------------------------------------------------------------------------- /pkg/util/jaeger/constants.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utiljaeger 16 | 17 | var SpanStorageTypesToAddFlag = []string{ 18 | "cassandra", 19 | "elasticsearch", 20 | "kafka", 21 | "grpc", 22 | "badger", 23 | } 24 | -------------------------------------------------------------------------------- /pkg/util/object/key.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utilobject 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | 21 | "github.com/sirupsen/logrus" 22 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | ) 25 | 26 | type Key struct { 27 | Cluster string `json:"cluster"` 28 | Group string `json:"group"` 29 | Resource string `json:"resource"` 30 | Namespace string `json:"namespace"` 31 | Name string `json:"name"` 32 | } 33 | 34 | func (key Key) Clone() Key { 35 | return Key{ 36 | Cluster: strings.Clone(key.Cluster), 37 | Group: strings.Clone(key.Group), 38 | Resource: strings.Clone(key.Resource), 39 | Namespace: strings.Clone(key.Namespace), 40 | Name: strings.Clone(key.Name), 41 | } 42 | } 43 | 44 | func (key Key) GroupResource() schema.GroupResource { 45 | return schema.GroupResource{Group: key.Group, Resource: key.Resource} 46 | } 47 | 48 | func (key Key) String() string { 49 | return fmt.Sprintf("%s/%s/%s/%s/%s", key.Cluster, key.Group, key.Resource, key.Namespace, key.Name) 50 | } 51 | 52 | func (key Key) AsFields(prefix string) logrus.Fields { 53 | return logrus.Fields{ 54 | prefix + "Cluster": key.Cluster, 55 | prefix + "Group": key.Group, 56 | prefix + "Resource": key.Resource, 57 | prefix + "Namespace": key.Namespace, 58 | prefix + "Name": key.Name, 59 | } 60 | } 61 | 62 | func FromMap(tags map[string]string) (key Key, ok bool) { 63 | for mapKey, field := range map[string]*string{ 64 | "cluster": &key.Cluster, 65 | "group": &key.Group, 66 | "resource": &key.Resource, 67 | "namespace": &key.Namespace, 68 | "name": &key.Name, 69 | } { 70 | *field, ok = tags[mapKey] 71 | if !ok { 72 | return key, false 73 | } 74 | } 75 | 76 | return key, true 77 | } 78 | 79 | type VersionedKey struct { 80 | Key 81 | Version string `json:"version"` 82 | } 83 | 84 | func (key VersionedKey) Clone() VersionedKey { 85 | return VersionedKey{ 86 | Key: key.Key.Clone(), 87 | Version: strings.Clone(key.Version), 88 | } 89 | } 90 | 91 | func (key VersionedKey) GroupVersionResource() schema.GroupVersionResource { 92 | return key.GroupResource().WithVersion(key.Version) 93 | } 94 | 95 | func (key VersionedKey) GroupVersion() schema.GroupVersion { 96 | return schema.GroupVersion{Group: key.Group, Version: key.Version} 97 | } 98 | 99 | func FromObject(object metav1.Object, cluster string, gvr schema.GroupVersionResource) VersionedKey { 100 | return VersionedKey{ 101 | Key: Key{ 102 | Cluster: cluster, 103 | Group: gvr.Group, 104 | Resource: gvr.Resource, 105 | Namespace: object.GetNamespace(), 106 | Name: object.GetName(), 107 | }, 108 | Version: gvr.Version, 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /pkg/util/object/rich.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utilobject 16 | 17 | import ( 18 | "strings" 19 | 20 | "github.com/sirupsen/logrus" 21 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 22 | "k8s.io/apimachinery/pkg/runtime/schema" 23 | "k8s.io/apimachinery/pkg/types" 24 | auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" 25 | ) 26 | 27 | type Rich struct { 28 | VersionedKey 29 | 30 | Uid types.UID 31 | 32 | Raw *unstructured.Unstructured `json:"-"` 33 | } 34 | 35 | // Returns a new Rich reference with all strings cloned again 36 | // to ensure this object does not reference more data than it needs. 37 | // 38 | // Does not copy the Raw field. 39 | func (ref Rich) Clone() Rich { 40 | return Rich{ 41 | VersionedKey: ref.VersionedKey.Clone(), 42 | Uid: types.UID(strings.Clone(string(ref.Uid))), 43 | } 44 | } 45 | 46 | func (ref Rich) String() string { 47 | return ref.Key.String() 48 | } 49 | 50 | func (ref Rich) AsFields(prefix string) logrus.Fields { 51 | fields := ref.Key.AsFields(prefix) 52 | fields[prefix+"Uid"] = ref.Uid 53 | return fields 54 | } 55 | 56 | func RichFromUnstructured( 57 | uns *unstructured.Unstructured, 58 | cluster string, 59 | gvr schema.GroupVersionResource, 60 | ) Rich { 61 | return Rich{ 62 | VersionedKey: FromObject(uns, cluster, gvr), 63 | Uid: uns.GetUID(), 64 | Raw: uns, 65 | } 66 | } 67 | 68 | func RichFromAudit(object *auditv1.ObjectReference, cluster string) Rich { 69 | return Rich{ 70 | VersionedKey: VersionedKey{ 71 | Key: Key{ 72 | Cluster: cluster, 73 | Group: object.APIGroup, 74 | Resource: object.Resource, 75 | Namespace: object.Namespace, 76 | Name: object.Name, 77 | }, 78 | Version: object.APIVersion, 79 | }, 80 | Uid: object.UID, 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /pkg/util/reflect/reflect.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package reflectutil 16 | 17 | import "reflect" 18 | 19 | func TypeOf[T any]() reflect.Type { 20 | array := [0]T{} 21 | arrayTy := reflect.TypeOf(array) 22 | return arrayTy.Elem() 23 | } 24 | 25 | func ZeroOf[T any]() (_ T) { return } 26 | 27 | func Identity[T any](t T) T { return t } 28 | -------------------------------------------------------------------------------- /pkg/util/shutdown/shutdown.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package shutdown 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "os/signal" 22 | "sync" 23 | "sync/atomic" 24 | "syscall" 25 | 26 | "github.com/sirupsen/logrus" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | ) 29 | 30 | type DeferFunc struct { 31 | name string 32 | fn func(context.Context) error 33 | } 34 | 35 | type DeferList struct { 36 | list []DeferFunc 37 | wasClosed int32 38 | mutex sync.Mutex 39 | } 40 | 41 | func NewDeferList() *DeferList { 42 | return &DeferList{list: make([]DeferFunc, 0)} 43 | } 44 | 45 | func (list *DeferList) DeferContextWithLock(name string, fn func(ctx context.Context) error) { 46 | list.mutex.Lock() 47 | defer list.mutex.Unlock() 48 | list.DeferContext(name, fn) 49 | } 50 | 51 | func (list *DeferList) DeferWithLock(name string, fn func() error) { 52 | list.mutex.Lock() 53 | defer list.mutex.Unlock() 54 | list.Defer(name, fn) 55 | } 56 | 57 | func (list *DeferList) Defer(name string, fn func() error) { 58 | list.DeferContext(name, func(ctx context.Context) error { return fn() }) 59 | } 60 | 61 | func (list *DeferList) DeferContext(name string, fn func(context.Context) error) { 62 | list.list = append(list.list, DeferFunc{name: name, fn: fn}) 63 | } 64 | 65 | func (list *DeferList) LockedRun(ctx context.Context, logger logrus.FieldLogger) (string, error) { 66 | list.mutex.Lock() 67 | defer list.mutex.Unlock() 68 | return list.Run(ctx, logger) 69 | } 70 | 71 | // Run runs the defer list. Should be called from the Close function of components. 72 | // Returns a nonempty string containing the defer message and the error that occurred 73 | // upon error. Returns empty string and nil error upon success. 74 | func (list *DeferList) Run(ctx context.Context, logger logrus.FieldLogger) (string, error) { 75 | wasClosed := atomic.SwapInt32(&list.wasClosed, 1) 76 | if wasClosed > 0 { 77 | return "", nil 78 | } 79 | 80 | for i := len(list.list) - 1; i >= 0; i-- { 81 | entry := &list.list[i] 82 | logger.Infof("Shutdown: %s", entry.name) 83 | if err := entry.fn(ctx); err != nil { 84 | return entry.name, err 85 | } 86 | } 87 | 88 | return "", nil 89 | } 90 | 91 | func (list *DeferList) RunWithChannel(ctx context.Context, logger logrus.FieldLogger, ch chan<- error) { 92 | if name, err := list.Run(ctx, logger); err != nil { 93 | ch <- fmt.Errorf("%s: %w", name, err) 94 | } else { 95 | ch <- nil 96 | } 97 | } 98 | 99 | func RecoverPanic(logger logrus.FieldLogger) { 100 | utilruntime.HandleCrash(func(err any) { 101 | if logger != nil { 102 | logger.WithField("error", err).Error() 103 | } 104 | }) 105 | } 106 | 107 | type ShutdownTrigger struct { 108 | Trigger context.CancelFunc 109 | } 110 | 111 | func ContextWithTrigger(ctx context.Context) (context.Context, *ShutdownTrigger) { 112 | ctx, cancelFunc := context.WithCancel(ctx) 113 | 114 | return ctx, &ShutdownTrigger{ 115 | Trigger: cancelFunc, 116 | } 117 | } 118 | 119 | func (trigger *ShutdownTrigger) SetupSignalHandler() { 120 | go func() { 121 | defer RecoverPanic(nil) 122 | c := make(chan os.Signal, 2) 123 | signal.Notify(c, os.Interrupt, syscall.SIGTERM) 124 | <-c 125 | trigger.Trigger() 126 | }() 127 | } 128 | -------------------------------------------------------------------------------- /pkg/util/zconstants/zconstants.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // zconstants are special span/event tags that provide context 16 | // for span transformation in the frontend storage plugin. 17 | package zconstants 18 | 19 | import ( 20 | "time" 21 | ) 22 | 23 | // All tags with this prefix are not rendered. 24 | const Prefix = "zzz-" 25 | 26 | // The value replaces the OperationName. 27 | const SpanName = Prefix + "kelemetryName" 28 | 29 | // Indicates that the current span is a pseudospan that can be folded or flattened. 30 | // The value is the folding type. 31 | const PseudoType = Prefix + "pseudoType" 32 | 33 | // Indicates that the current span is not pseudo. 34 | // Used to optimize trace listing. 35 | // 36 | // This constant is used as both tag key and value. 37 | const NotPseudo = Prefix + "notPseudo" 38 | 39 | type PseudoTypeValue string 40 | 41 | const ( 42 | // Root span in an object trace. 43 | PseudoTypeObject PseudoTypeValue = "object" 44 | // Indicate that another trace shall be included. 45 | PseudoTypeLink PseudoTypeValue = "link" 46 | // A virtual span synthesized in the frontend when link class is nonempty. 47 | PseudoTypeLinkClass PseudoTypeValue = "linkClass" 48 | ) 49 | 50 | // If value is not "object", identifies that the span represents an actual event (rather than as a pseudospan). 51 | const TraceSource = Prefix + "traceSource" 52 | 53 | const ( 54 | TraceSourceObject = "object" 55 | 56 | TraceSourceAudit = "audit" 57 | TraceSourceEvent = "event" 58 | ) 59 | 60 | func KnownPseudoTraceSources() []string { 61 | return []string{TraceSourceObject} 62 | } 63 | 64 | func KnownNonPseudoTraceSources() []string { 65 | return []string{ 66 | TraceSourceAudit, 67 | TraceSourceEvent, 68 | } 69 | } 70 | 71 | // Classifies the type of a log line. 72 | // Logs without this attribute will not have special treatment. 73 | const LogTypeAttr = Prefix + "logType" 74 | 75 | type LogType string 76 | 77 | const ( 78 | LogTypeRealError LogType = "realError" 79 | LogTypeRealVerbose LogType = "realVerbose" 80 | LogTypeKelemetryError LogType = "kelemetryError" 81 | LogTypeObjectSnapshot LogType = "audit/objectSnapshot" 82 | LogTypeObjectDiff LogType = "audit/objectDiff" 83 | LogTypeEventMessage LogType = "event/message" 84 | ) 85 | 86 | // DummyDuration is the span duration used when the span is instantaneous. 87 | const DummyDuration = time.Second 88 | -------------------------------------------------------------------------------- /scan/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.21-alpine3.18 AS prom2json 2 | RUN GO111MODULE=on go install github.com/prometheus/prom2json/cmd/prom2json@v1.3.3 3 | 4 | FROM alpine:3 5 | 6 | RUN apk add --no-cache curl jq 7 | RUN curl -L -o /usr/local/bin/kubectl \ 8 | "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ 9 | chmod +x /usr/local/bin/kubectl 10 | COPY --from=prom2json /go/bin/prom2json /usr/local/bin/prom2json 11 | 12 | ADD main.sh /usr/local/bin/scan 13 | ENTRYPOINT ["scan"] 14 | -------------------------------------------------------------------------------- /scan/main.sh: -------------------------------------------------------------------------------- 1 | #!//bin/ash 2 | 3 | set -euo pipefail 4 | 5 | kubectl get pod -o json -n ${RELEASE_NAMESPACE} \ 6 | -l app.kubernetes.io/name=kelemetry \ 7 | -l app.kubernetes.io/instance=${RELEASE_NAME} \ 8 | | jq -c '.items[]' >pods.jsonl 9 | 10 | metrics.jsonl 43 | 44 | webhook_requests=$(jq -sr 'map(select(.name == "audit_webhook_request_count")) | map(.value) | add' metrics.jsonl) 45 | echo Received $webhook_requests webhook requests 46 | if [ $webhook_requests -eq 0 ]; then 47 | echo "[!]" Check if audit webhook was configured correctly 48 | fi 49 | 50 | consumed_audits=$(jq -sr 'map(select(.name == "audit_consumer_event_count" and .labels.hasTrace == "true")) | map(.value) | add' metrics.jsonl) 51 | echo Consumed $consumed_audits audit events with an audit span 52 | 53 | handled_events=$(jq -sr 'map(select(.name == "event_handle_count")) | map(.value) | add' metrics.jsonl) 54 | ok_events=$(jq -sr 'map(select(.name == "event_handle_count" and .labels.error == "")) | map(.value) | add' metrics.jsonl) 55 | event_errs=$(jq -sr ' 56 | map(select(.name == "event_handle_count" and .value > 0 and .labels.error != "" and .labels.error != "BeforeRestart" and .labels.error != "Filtered")) 57 | | group_by(.labels.error) 58 | | map({error: .[0].labels.error, value: map(.value) | add}) 59 | | sort_by(-.value) 60 | | map(.error + " (" + (.value | tostring) + ")") 61 | | join(", ") 62 | ' metrics.jsonl) 63 | 64 | echo -n Handled $handled_events k8s events, $ok_events have spans 65 | if [ ! -z "$event_errs" ]; then 66 | echo , with errors: "$event_errs" 67 | else 68 | echo 69 | fi 70 | 71 | sent_spans=$(jq -sr 'map(select(.name == "aggregator_send_count")) | map(.value) | add' metrics.jsonl) 72 | echo Sent $sent_spans non-pseudo spans 73 | 74 | jq -sr \ 75 | 'map( 76 | select((.name == "jaeger_collector_spans_received_total" or .name == "jaeger_collector_spans_rejected_total") and .value > 0) 77 | | { 78 | svc: .labels.svc, 79 | action: (if .name == "jaeger_collector_spans_received_total" then "received" else "rejected" end), 80 | value: .value, 81 | } 82 | ) 83 | | group_by([.svc, .action]) 84 | | map({svc: .[0].svc, action: .[0].action, value: map(.value) | add}) 85 | | sort_by(-.value) 86 | | map("jaeger-collector " + .action + " " + (.value | tostring) + " " + .svc + " spans") 87 | | join("\n") 88 | ' metrics.jsonl 89 | -------------------------------------------------------------------------------- /tools/imports.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Kelemetry Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Dummy package for dependencies only used as `go run` 16 | // to prevent go.mod formatting tools from automatically removing them. 17 | // 18 | // This package should not be imported by the binary package. 19 | package kelemetry_tools 20 | 21 | import ( 22 | _ "github.com/daixiang0/gci/cmd/gci" 23 | _ "github.com/itchyny/gojq/cli" 24 | _ "github.com/segmentio/golines" 25 | _ "golang.org/x/tools/cmd/goimports" 26 | _ "mvdan.cc/gofumpt" 27 | ) 28 | --------------------------------------------------------------------------------