├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── dependabot.yml └── workflows │ ├── build.yml │ ├── integration.yml │ ├── release.yml │ └── scorecard.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── config └── config.go ├── connector.go ├── couchbase └── event.go ├── dcp_event_handler.go ├── docker-compose.yml ├── example ├── README.md ├── default-completion │ ├── Dockerfile │ ├── config.yml │ ├── go.mod │ ├── go.sum │ └── main.go ├── default-mapper │ ├── Dockerfile │ ├── config.yml │ ├── go.mod │ ├── go.sum │ └── main.go ├── grafana │ ├── Dockerfile │ ├── config.yml │ ├── couchbase │ │ ├── Dockerfile │ │ └── configure.sh │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ ├── grafana │ │ ├── grafana-1.png │ │ ├── grafana-2.png │ │ ├── grafana-3.png │ │ └── provisioning │ │ │ ├── dashboards │ │ │ ├── dashboard.json │ │ │ └── dashboards.yaml │ │ │ └── datasources │ │ │ └── datasource.yaml │ ├── main.go │ └── prometheus │ │ └── prometheus.yml ├── simple-logger │ ├── Dockerfile │ ├── config.yml │ ├── go.mod │ ├── go.sum │ └── main.go ├── simple-rejection-log-sink-response-handler │ ├── Dockerfile │ ├── config.yml │ ├── go.mod │ ├── go.sum │ └── main.go ├── simple │ ├── Dockerfile │ ├── config.yml │ ├── go.mod │ ├── go.sum │ └── main.go └── struct-config │ ├── Dockerfile │ ├── go.mod │ ├── go.sum │ └── main.go ├── go.mod ├── go.sum ├── kafka ├── client.go ├── client_test.go ├── message │ └── message.go ├── metadata │ └── kafka_metadata.go ├── producer │ ├── completion.go │ ├── producer.go │ └── producer_batch.go ├── rejection_log_sink_response_handler.go └── sink_response_handler.go ├── mapper.go ├── metric └── collector.go └── test ├── couchbase ├── Dockerfile └── configure.sh └── integration ├── Dockerfile ├── config.yml ├── go.mod ├── go.sum └── integration_test.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Do the following '...' 16 | 2. Run the project with '....' 17 | 3. See error 18 | 19 | **Expected behavior** 20 | A clear and concise description of what you expected to happen. 21 | 22 | **Screenshots** 23 | If applicable, add screenshots to help explain your problem. 24 | 25 | **Version (please complete the following information):** 26 | - OS: [e.g. macOS] 27 | - Golang version [e.g. 1.17] 28 | - Couchbase/Kafka version 29 | 30 | **Additional context** 31 | Add any other context about the problem here. 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | open-pull-requests-limit: 10 8 | commit-message: 9 | prefix: "chore: bump" 10 | allow: 11 | - dependency-name: github.com/Trendyol/go-dcp 12 | - dependency-name: github.com/segmentio/kafka-go 13 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master] 8 | types: [opened, reopened, synchronize] 9 | 10 | concurrency: 11 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | build: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v4 20 | with: 21 | fetch-depth: 1 22 | 23 | - name: Set up Go 24 | uses: actions/setup-go@v4 25 | with: 26 | go-version: '1.20' 27 | 28 | - name: Lint 29 | run: | 30 | make init 31 | make lint 32 | 33 | - name: Install dependencies 34 | run: go get . 35 | 36 | - name: Build 37 | run: go build -v ./... 38 | 39 | - name: Test 40 | run: go test -v ./... 41 | 42 | 43 | security-gates: 44 | uses: Trendyol/security-actions/.github/workflows/security-gates.yml@master 45 | permissions: 46 | actions: read 47 | contents: read 48 | security-events: write 49 | -------------------------------------------------------------------------------- /.github/workflows/integration.yml: -------------------------------------------------------------------------------- 1 | name: Integration 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master] 8 | types: [opened, reopened, synchronize] 9 | 10 | concurrency: 11 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | integration: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v4 20 | with: 21 | fetch-depth: 1 22 | 23 | - name: Set up Go 24 | uses: actions/setup-go@v4 25 | with: 26 | go-version: '1.20' 27 | 28 | - name: Start containers 29 | run: make compose 30 | 31 | - name: Integration Test 32 | run: go test -v test/integration/integration_test.go 33 | env: 34 | INPUT_PUBLISH: false 35 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 36 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v4 14 | with: 15 | fetch-depth: 0 16 | 17 | - run: git tag ${{ github.event.inputs.tag }} 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v4 21 | with: 22 | go-version: '1.20' 23 | 24 | - name: Run GoReleaser 25 | uses: goreleaser/goreleaser-action@v4 26 | with: 27 | version: latest 28 | args: release --clean 29 | env: 30 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 31 | -------------------------------------------------------------------------------- /.github/workflows/scorecard.yml: -------------------------------------------------------------------------------- 1 | 2 | name: Scorecard supply-chain security 3 | 4 | on: 5 | branch_protection_rule: 6 | schedule: 7 | - cron: '29 23 * * 3' 8 | push: 9 | branches: [ "main", "master"] 10 | pull_request: 11 | branches: ["main", "master"] 12 | 13 | permissions: read-all 14 | 15 | jobs: 16 | visibility-check: 17 | # Bu job, deponun public/private olduğunu belirler 18 | outputs: 19 | visibility: ${{ steps.drv.outputs.visibility }} 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Determine repository visibility 23 | id: drv 24 | run: | 25 | visibility=$(gh api /repos/$GITHUB_REPOSITORY --jq '.visibility') 26 | echo "visibility=$visibility" >> $GITHUB_OUTPUT 27 | env: 28 | GH_TOKEN: ${{ github.token }} 29 | 30 | analysis: 31 | if: ${{ needs.visibility-check.outputs.visibility == 'public' }} 32 | needs: visibility-check 33 | runs-on: ubuntu-latest 34 | permissions: 35 | security-events: write 36 | id-token: write 37 | steps: 38 | - name: "Checkout code" 39 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 40 | with: 41 | persist-credentials: false 42 | 43 | - name: "Run analysis" 44 | uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 45 | with: 46 | results_file: results.sarif 47 | results_format: sarif 48 | publish_results: true 49 | 50 | - name: "Upload artifact" 51 | uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 52 | with: 53 | name: SARIF file 54 | path: results.sarif 55 | retention-days: 5 56 | 57 | - name: "Upload to code-scanning" 58 | uses: github/codeql-action/upload-sarif@v3 59 | with: 60 | sarif_file: results.sarif 61 | 62 | 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | .DS_store 15 | # Dependency directories (remove the comment below to include it) 16 | # vendor/ 17 | 18 | .idea 19 | checkpoints.json 20 | main.exe -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | lll: 3 | line-length: 140 4 | funlen: 5 | lines: 70 6 | 7 | linters: 8 | disable-all: true 9 | enable: 10 | - bodyclose 11 | - errcheck 12 | - dupl 13 | - exhaustive 14 | - funlen 15 | - goconst 16 | - gocritic 17 | - gocyclo 18 | - gosimple 19 | - govet 20 | - gosec 21 | - ineffassign 22 | - lll 23 | - misspell 24 | - nakedret 25 | - gofumpt 26 | - staticcheck 27 | - stylecheck 28 | - typecheck 29 | - unconvert 30 | - unparam 31 | - unused 32 | - whitespace 33 | - prealloc 34 | 35 | service: 36 | golangci-lint-version: 1.55.2 # use the fixed version to not introduce new linters unexpectedly 37 | prepare: 38 | - echo "here I can run custom commands, but no preparation needed for this repo" -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: go-dcp-kafka 2 | 3 | release: 4 | github: 5 | name: go-dcp-kafka 6 | owner: Trendyol 7 | 8 | before: 9 | hooks: 10 | - go mod tidy 11 | 12 | builds: 13 | - skip: true 14 | 15 | changelog: 16 | sort: asc 17 | use: github 18 | filters: 19 | exclude: 20 | - '^test:' 21 | - '^docs:' 22 | - '^chore:' 23 | - 'merge conflict' 24 | - Merge pull request 25 | - Merge remote-tracking branch 26 | - Merge branch 27 | - go mod tidy -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contribution Guidelines 2 | 3 | Thank you for your interest in go-dcp-kafka! 4 | 5 | This project welcomes contributions and suggestions. Most contributions require you to signoff on your commits. Please 6 | follow the instructions provided below. 7 | 8 | Contributions come in many forms: submitting issues, writing code, participating in discussions and community calls. 9 | 10 | This document provides the guidelines for how to contribute to the project. 11 | 12 | ### Issues 13 | 14 | This section describes the guidelines for submitting issues 15 | 16 | **Issue Types** 17 | 18 | There are 2 types of issues: 19 | 20 | - Issue/Bug: You've found a bug with the code, and want to report it, or create an issue to track the bug. 21 | - Issue/Feature: You have something on your mind, which requires input form others in a discussion, before it eventually 22 | manifests as a proposal. 23 | 24 | ### Before You File 25 | 26 | Before you file an issue, make sure you've checked the following: 27 | 28 | 1. Check for existing issues 29 | Before you create a new issue, please do a search in open issues to see if the issue or feature request has already 30 | been filed. 31 | 32 | If you find your issue already exists, make relevant comments and add your reaction. Use a reaction: 33 | 👍 up-vote 34 | 👎 down-vote 35 | 36 | 2. For bugs 37 | Check it's not an environment issue. For example, if your configurations correct or network connections is alive. 38 | 39 | ### Contributing to go-dcp-kafka 40 | 41 | Pull Requests 42 | All contributions come through pull requests. To submit a proposed change, we recommend following this workflow: 43 | 44 | - Make sure there's an issue (bug or feature) raised, which sets the expectations for the contribution you are about to 45 | make. 46 | - Fork the relevant repo and create a new branch 47 | - Create your change 48 | - Code changes require tests 49 | - Update relevant documentation for the change 50 | - Commit sign-off and open a PR 51 | - Wait for the CI process to finish and make sure all checks are green 52 | - A maintainer of the project will be assigned, and you can expect a review within a few days 53 | 54 | ### Use work-in-progress PRs for early feedback 55 | 56 | A good way to communicate before investing too much time is to create a "Work-in-progress" PR and share it with your 57 | reviewers. The standard way of doing this is to add a "[WIP]" prefix in your PR's title and assign the do-not-merge 58 | label. This will let people looking at your PR know that it is not well baked yet. 59 | 60 | ### Developer Certificate of Origin: Signing your work 61 | 62 | **Every commit needs to be signed** 63 | 64 | The Developer Certificate of Origin (DCO) is a lightweight way for contributors to certify that they wrote or otherwise 65 | have the right to submit the code they are contributing to the project. Here is the full text of the DCO, reformatted 66 | for readability: 67 | 68 | By making a contribution to this project, I certify that: 69 | 70 | (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or 71 | 72 | (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or 73 | 74 | (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. 75 | 76 | (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. 77 | 78 | Contributors sign-off that they adhere to these requirements by adding a Signed-off-by line to commit messages. 79 | 80 | This is my commit message 81 | 82 | Signed-off-by: Random X Developer 83 | 84 | Git even has a -s command line option to append this automatically to your commit message: 85 | 86 | $ git commit -s -m 'This is my commit message' 87 | 88 | Each Pull Request is checked whether or not commits in a Pull Request do contain a valid Signed-off-by line. 89 | 90 | I didn't sign my commit, now what?! 91 | No worries - You can easily replay your changes, sign them and force push them! 92 | 93 | git checkout 94 | git commit --amend --no-edit --signoff 95 | git push --force-with-lease 96 | 97 | ### Use of Third-party code 98 | 99 | - Third-party code must include licenses. 100 | 101 | **Thank You!** - Your contributions to open source, large or small, make projects like this possible. Thank you for 102 | taking the time to contribute. 103 | 104 | ### Code of Conduct 105 | 106 | This project has adopted the Contributor Covenant Code of Conduct -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20-alpine as builder 2 | 3 | WORKDIR /project 4 | 5 | COPY . . 6 | 7 | WORKDIR /project/example 8 | 9 | RUN go mod download 10 | RUN CGO_ENABLED=0 go build -a -o example main.go 11 | 12 | FROM alpine:3.17.0 13 | 14 | WORKDIR /app 15 | 16 | RUN apk --no-cache add ca-certificates 17 | 18 | USER nobody 19 | COPY --from=builder --chown=nobody:nobody /project/example/example . 20 | COPY --from=builder --chown=nobody:nobody /project/example/config.yml ./config.yml 21 | 22 | ENTRYPOINT ["./example"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Trendyol 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: default 2 | 3 | default: init 4 | 5 | init: 6 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2 7 | go install golang.org/x/tools/go/analysis/passes/fieldalignment/cmd/fieldalignment@v0.15.0 8 | 9 | clean: 10 | rm -rf ./build 11 | 12 | linter: 13 | fieldalignment -fix ./... 14 | golangci-lint run -c .golangci.yml --timeout=5m -v --fix 15 | 16 | lint: 17 | golangci-lint run -c .golangci.yml --timeout=5m -v 18 | 19 | test: 20 | go test ./... -bench . -benchmem 21 | 22 | compose: 23 | docker compose up --wait --build --force-recreate --remove-orphans 24 | 25 | tidy: 26 | go mod tidy 27 | cd example/default-mapper && go mod tidy && cd ../.. 28 | cd example/default-completion && go mod tidy && cd ../.. 29 | cd example/simple && go mod tidy && cd ../.. 30 | cd example/simple-logger && go mod tidy && cd ../.. 31 | cd example/struct-config && go mod tidy && cd ../.. 32 | cd example/grafana && go mod tidy && cd ../.. 33 | cd example/simple-rejection-log-sink-response-handler && go mod tidy && cd ../.. 34 | cd test/integration && go mod tidy && cd ../.. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/Trendyol/go-dcp-kafka/badge)](https://scorecard.dev/viewer/?uri=github.com/Trendyol/go-dcp-kafka) 3 | # Go Dcp Kafka 4 | 5 | [![Go Reference](https://pkg.go.dev/badge/github.com/Trendyol/go-dcp-kafka.svg)](https://pkg.go.dev/github.com/Trendyol/go-dcp-kafka) [![Go Report Card](https://goreportcard.com/badge/github.com/Trendyol/go-dcp-kafka)](https://goreportcard.com/report/github.com/Trendyol/go-dcp-kafka) 6 | 7 | Go implementation of the [Kafka Connect Couchbase](https://github.com/couchbase/kafka-connect-couchbase). 8 | 9 | **Go Dcp Kafka** streams documents from Couchbase Database Change Protocol (DCP) and publishes Kafka 10 | events in near real-time. 11 | 12 | ## Features 13 | 14 | * **Less resource usage** and **higher throughput**(see [Benchmarks](#benchmarks)). 15 | * **Custom Kafka key and headers** implementation(see [Example](#example)). 16 | * Sending **multiple Kafka events for a DCP event**(see [Example](#example)). 17 | * Handling different DCP events such as **expiration, deletion and mutation**(see [Example](#example)). 18 | * **Kafka compression** support(Gzip, Snappy, Lz4, Zstd). 19 | * **Kafka producer acknowledges** support(fire-and-forget, wait for the leader, wait for the full ISR). 20 | * Metadata can be saved to **Couchbase or Kafka**. 21 | * **Managing batch configurations** such as maximum batch size, batch bytes, batch ticker durations. 22 | * **Scale up and down** by custom membership algorithms(Couchbase, KubernetesHa, Kubernetes StatefulSet or 23 | Static, see [examples](https://github.com/Trendyol/go-dcp#examples)). 24 | * **Easily manageable configurations**. 25 | 26 | ## Benchmarks 27 | 28 | The benchmark was made with the **1,001,006** Couchbase document, because it is possible to more clearly observe the 29 | difference in the batch structure between the two packages. **Default configurations** for Java Kafka Connect Couchbase 30 | used for both connectors. 31 | 32 | | Package | Time to Process Events | Average CPU Usage(Core) | Average Memory Usage | 33 | |:------------------------------------|:----------------------:|:-----------------------:|:--------------------:| 34 | | **Go Dcp Kafka**(1.20) | **12s** | **0.383** | **428MB** 35 | | Java Kafka Connect Couchbase(JDK11) | 19s | 1.5 | 932MB 36 | 37 | ## Example 38 | 39 | [Struct Config](example/struct-config/main.go) 40 | 41 | ```go 42 | func mapper(event couchbase.Event) []message.KafkaMessage { 43 | // return nil if you wish to discard the event 44 | return []message.KafkaMessage{ 45 | { 46 | Headers: nil, 47 | Key: event.Key, 48 | Value: event.Value, 49 | }, 50 | } 51 | } 52 | 53 | func main() { 54 | c, err := dcpkafka.NewConnector(&config.Connector{ 55 | Dcp: dcpConfig.Dcp{ 56 | Hosts: []string{"localhost:8091"}, 57 | Username: "user", 58 | Password: "password", 59 | BucketName: "dcp-test", 60 | Dcp: dcpConfig.ExternalDcp{ 61 | Group: dcpConfig.DCPGroup{ 62 | Name: "groupName", 63 | Membership: dcpConfig.DCPGroupMembership{ 64 | RebalanceDelay: 3 * time.Second, 65 | }, 66 | }, 67 | }, 68 | Metadata: dcpConfig.Metadata{ 69 | Config: map[string]string{ 70 | "bucket": "checkpoint-bucket-name", 71 | "scope": "_default", 72 | "collection": "_default", 73 | }, 74 | Type: "couchbase", 75 | }, 76 | Debug: true}, 77 | Kafka: config.Kafka{ 78 | CollectionTopicMapping: map[string]string{"_default": "topic"}, 79 | Brokers: []string{"localhost:9092"}, 80 | }, 81 | }, mapper) 82 | if err != nil { 83 | panic(err) 84 | } 85 | 86 | defer c.Close() 87 | c.Start() 88 | } 89 | ``` 90 | 91 | [File Config](example/simple/main.go) 92 | 93 | [File Config](example/default-mapper/main.go) 94 | 95 | ## Configuration 96 | 97 | ### Dcp Configuration 98 | 99 | Check out on [go-dcp](https://github.com/Trendyol/go-dcp#configuration) 100 | 101 | ### Kafka Specific Configuration 102 | 103 | | Variable | Type | Required | Default | Description | 104 | |-------------------------------------|-------------------|----------|-------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 105 | | `kafka.collectionTopicMapping` | map[string]string | yes | | Defines which Couchbase collection events will be sent to which topic,:warning: **If topic information is entered in the mapper, it will OVERWRITE this config**. | 106 | | `kafka.brokers` | []string | yes | | Broker ip and port information | 107 | | `kafka.producerBatchSize` | integer | no | 2000 | Maximum message count for batch, if exceed flush will be triggered. | 108 | | `kafka.producerBatchBytes` | 64 bit integer | no | 1mb | Maximum size(byte) for batch, if exceed flush will be triggered. `1mb` is default. | 109 | | `kafka.producerMaxAttempts` | int | no | math.MaxInt | Limit on how many attempts will be made to deliver a message. | 110 | | `kafka.producerBatchTickerDuration` | time.Duration | no | 10s | Batch is being flushed automatically at specific time intervals for long waiting messages in batch. | 111 | | `kafka.readTimeout` | time.Duration | no | 30s | segmentio/kafka-go - Timeout for read operations | 112 | | `kafka.writeTimeout` | time.Duration | no | 30s | segmentio/kafka-go - Timeout for write operations | 113 | | `kafka.compression` | integer | no | 0 | Compression can be used if message size is large, CPU usage may be affected. 0=None, 1=Gzip, 2=Snappy, 3=Lz4, 4=Zstd | 114 | | `kafka.balancer` | string | no | Hash | Define balancer strategy. Available fields: Hash, LeastBytes, RoundRobin, ReferenceHash, CRC32Balancer, Murmur2Balancer. | 115 | | `kafka.requiredAcks` | integer | no | 1 | segmentio/kafka-go - Number of acknowledges from partition replicas required before receiving a response to a produce request. 0=fire-and-forget, do not wait for acknowledgements from the, 1=wait for the leader to acknowledge the writes, -1=wait for the full ISR to acknowledge the writes | 116 | | `kafka.secureConnection` | bool | no | false | Enable secure Kafka. | 117 | | `kafka.rootCAPath` | string | no | *not set | Define root CA path. | 118 | | `kafka.interCAPath` | string | no | *not set | Define inter CA path. | 119 | | `kafka.scramUsername` | string | no | *not set | Define scram username. | 120 | | `kafka.scramPassword` | string | no | *not set | Define scram password. | 121 | | `kafka.metadataTTL` | time.Duration | no | 60s | TTL for the metadata cached by segmentio, increase it to reduce network requests. For more detail please check [docs](https://pkg.go.dev/github.com/segmentio/kafka-go#Transport.MetadataTTL). | 122 | | `kafka.metadataTopics` | []string | no | | Topic names for the metadata cached by segmentio, define topics here that the connector may produce. In large Kafka clusters, this will reduce memory usage. For more detail please check [docs](https://pkg.go.dev/github.com/segmentio/kafka-go#Transport.MetadataTopics). | 123 | | `kafka.clientID` | string | no | | Unique identifier that the transport communicates to the brokers when it sends requests. For more detail please check [docs](https://pkg.go.dev/github.com/segmentio/kafka-go#Transport.ClientID). | 124 | | `kafka.allowAutoTopicCreation` | bool | no | false | Create topic if missing. For more detail please check [docs](https://pkg.go.dev/github.com/segmentio/kafka-go#Writer.AllowAutoTopicCreation). | 125 | | `kafka.rejectionLog.topic` | string | no | | Rejection topic name. | 126 | | `kafka.rejectionLog.includeValue` | boolean | no | false | Includes rejection log source info. `false` is default. | 127 | 128 | ### Kafka Metadata Configuration(Use it if you want to store the checkpoint data in Kafka) 129 | 130 | | Variable | Type | Description | 131 | |---------------------|-------------------|------------------------------------------------------------------------------------| 132 | | `metadata.type` | string | Metadata storing types. `kafka`,`file` or `couchbase`. | 133 | | `metadata.readOnly` | bool | Set this for debugging state purposes. | 134 | | `metadata.config` | map[string]string | Set key-values of config. `topic`,`partition`,`replicationFactor` for `kafka` type | 135 | 136 | ## Exposed metrics 137 | 138 | | Metric Name | Description | Labels | Value Type | 139 | |-------------------------------------------------------|----------------------------------------|--------|------------| 140 | | cbgo_kafka_connector_latency_ms_current | Time to adding to the batch. | N/A | Gauge | 141 | | cbgo_kafka_connector_batch_produce_latency_ms_current | Time to produce messages in the batch. | N/A | Gauge | 142 | 143 | You can also use all DCP-related metrics explained [here](https://github.com/Trendyol/go-dcp#exposed-metrics). 144 | All DCP-related metrics are automatically injected. It means you don't need to do anything. 145 | 146 | ## Breaking Changes 147 | 148 | | Date taking effect | Date announced | Change | How to check | 149 | |--------------------| ---- |---- |-----------------| 150 | | November 11, 2023 | November 11, 2023 | Creating connector via builder | Compile project | 151 | 152 | ## Grafana Metric Dashboard 153 | 154 | [Grafana & Prometheus Example](example/grafana) 155 | 156 | ## Contributing 157 | 158 | Go Dcp Kafka is always open for direct contributions. For more information please check 159 | our [Contribution Guideline document](./CONTRIBUTING.md). 160 | 161 | ## License 162 | 163 | Released under the [MIT License](LICENSE). 164 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "math" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/Trendyol/go-dcp/config" 10 | "github.com/Trendyol/go-dcp/helpers" 11 | "github.com/Trendyol/go-dcp/logger" 12 | 13 | "github.com/segmentio/kafka-go" 14 | ) 15 | 16 | type Kafka struct { 17 | ProducerBatchBytes any `yaml:"producerBatchBytes"` 18 | CollectionTopicMapping map[string]string `yaml:"collectionTopicMapping"` 19 | InterCAPath string `yaml:"interCAPath"` 20 | InterCA string `yaml:"interCA"` 21 | ScramUsername string `yaml:"scramUsername"` 22 | ScramPassword string `yaml:"scramPassword"` 23 | RootCAPath string `yaml:"rootCAPath"` 24 | RootCA string `yaml:"rootCA"` 25 | ClientID string `yaml:"clientID"` 26 | Balancer string `yaml:"balancer"` 27 | Brokers []string `yaml:"brokers"` 28 | MetadataTopics []string `yaml:"metadataTopics"` 29 | RejectionLog RejectionLog `yaml:"rejectionLog"` 30 | ProducerMaxAttempts int `yaml:"producerMaxAttempts"` 31 | ReadTimeout time.Duration `yaml:"readTimeout"` 32 | WriteTimeout time.Duration `yaml:"writeTimeout"` 33 | RequiredAcks int `yaml:"requiredAcks"` 34 | ProducerBatchSize int `yaml:"producerBatchSize"` 35 | MetadataTTL time.Duration `yaml:"metadataTTL"` 36 | ProducerBatchTickerDuration time.Duration `yaml:"producerBatchTickerDuration"` 37 | Compression int8 `yaml:"compression"` 38 | SecureConnection bool `yaml:"secureConnection"` 39 | AllowAutoTopicCreation bool `yaml:"allowAutoTopicCreation"` 40 | } 41 | 42 | type RejectionLog struct { 43 | Topic string `yaml:"topic"` 44 | IncludeValue bool `yaml:"includeValue"` 45 | } 46 | 47 | func (k *Kafka) GetBalancer() kafka.Balancer { 48 | switch k.Balancer { 49 | case "", "Hash": 50 | return &kafka.Hash{} 51 | case "LeastBytes": 52 | return &kafka.LeastBytes{} 53 | case "RoundRobin": 54 | return &kafka.RoundRobin{} 55 | case "ReferenceHash": 56 | return &kafka.ReferenceHash{} 57 | case "CRC32Balancer": 58 | return kafka.CRC32Balancer{} 59 | case "Murmur2Balancer": 60 | return kafka.Murmur2Balancer{} 61 | default: 62 | err := errors.New("invalid kafka balancer method, given: " + k.Balancer) 63 | logger.Log.Error("error while get kafka balancer, err: %v", err) 64 | panic(err) 65 | } 66 | } 67 | 68 | func (k *Kafka) GetCompression() int8 { 69 | if k.Compression < 0 || k.Compression > 4 { 70 | err := errors.New("invalid kafka compression method, given: " + strconv.Itoa(int(k.Compression))) 71 | logger.Log.Error("error while get kafka compression, err: %v", err) 72 | panic(err) 73 | } 74 | return k.Compression 75 | } 76 | 77 | type Connector struct { 78 | Kafka Kafka `yaml:"kafka" mapstructure:"kafka"` 79 | Dcp config.Dcp `yaml:",inline" mapstructure:",squash"` 80 | } 81 | 82 | func (c *Connector) ApplyDefaults() { 83 | if c.Kafka.ReadTimeout == 0 { 84 | c.Kafka.ReadTimeout = 30 * time.Second 85 | } 86 | 87 | if c.Kafka.WriteTimeout == 0 { 88 | c.Kafka.WriteTimeout = 30 * time.Second 89 | } 90 | 91 | if c.Kafka.ProducerBatchTickerDuration == 0 { 92 | c.Kafka.ProducerBatchTickerDuration = 10 * time.Second 93 | } 94 | 95 | if c.Kafka.ProducerBatchSize == 0 { 96 | c.Kafka.ProducerBatchSize = 2000 97 | } 98 | 99 | if c.Kafka.ProducerBatchBytes == nil { 100 | c.Kafka.ProducerBatchBytes = helpers.ResolveUnionIntOrStringValue("1mb") 101 | } 102 | 103 | if c.Kafka.RequiredAcks == 0 { 104 | c.Kafka.RequiredAcks = 1 105 | } 106 | 107 | if c.Kafka.MetadataTTL == 0 { 108 | c.Kafka.MetadataTTL = 60 * time.Second 109 | } 110 | 111 | if c.Kafka.ProducerMaxAttempts == 0 { 112 | c.Kafka.ProducerMaxAttempts = math.MaxInt 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /connector.go: -------------------------------------------------------------------------------- 1 | package dcpkafka 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "os" 9 | "regexp" 10 | "strings" 11 | 12 | jsoniter "github.com/json-iterator/go" 13 | 14 | dcpCouchbase "github.com/Trendyol/go-dcp/couchbase" 15 | 16 | "github.com/Trendyol/go-dcp/helpers" 17 | 18 | "github.com/sirupsen/logrus" 19 | 20 | "github.com/Trendyol/go-dcp" 21 | 22 | "github.com/Trendyol/go-dcp-kafka/config" 23 | "github.com/Trendyol/go-dcp-kafka/couchbase" 24 | "github.com/Trendyol/go-dcp-kafka/kafka" 25 | "github.com/Trendyol/go-dcp-kafka/kafka/metadata" 26 | "github.com/Trendyol/go-dcp-kafka/kafka/producer" 27 | "github.com/Trendyol/go-dcp-kafka/metric" 28 | dcpConfig "github.com/Trendyol/go-dcp/config" 29 | "github.com/Trendyol/go-dcp/logger" 30 | "github.com/Trendyol/go-dcp/models" 31 | sKafka "github.com/segmentio/kafka-go" 32 | "gopkg.in/yaml.v3" 33 | ) 34 | 35 | var MetadataTypeKafka = "kafka" 36 | 37 | type Connector interface { 38 | Start() 39 | Close() 40 | GetDcpClient() dcpCouchbase.Client 41 | } 42 | 43 | type connector struct { 44 | dcp dcp.Dcp 45 | mapper Mapper 46 | producer producer.Producer 47 | config *config.Connector 48 | } 49 | 50 | func (c *connector) Start() { 51 | go func() { 52 | <-c.dcp.WaitUntilReady() 53 | c.producer.StartBatch() 54 | }() 55 | c.dcp.Start() 56 | } 57 | 58 | func (c *connector) Close() { 59 | c.dcp.Close() 60 | err := c.producer.Close() 61 | if err != nil { 62 | logger.Log.Error("error | %v", err) 63 | } 64 | } 65 | 66 | func (c *connector) GetDcpClient() dcpCouchbase.Client { 67 | return c.dcp.GetClient() 68 | } 69 | 70 | func (c *connector) produce(ctx *models.ListenerContext) { 71 | var e couchbase.Event 72 | switch event := ctx.Event.(type) { 73 | case models.DcpMutation: 74 | e = couchbase.NewMutateEvent( 75 | event.Key, event.Value, 76 | event.CollectionName, event.EventTime, event.Cas, event.VbID, event.SeqNo, event.RevNo, 77 | ) 78 | case models.DcpExpiration: 79 | e = couchbase.NewExpireEvent( 80 | event.Key, nil, 81 | event.CollectionName, event.EventTime, event.Cas, event.VbID, event.SeqNo, event.RevNo, 82 | ) 83 | case models.DcpDeletion: 84 | e = couchbase.NewDeleteEvent( 85 | event.Key, nil, 86 | event.CollectionName, event.EventTime, event.Cas, event.VbID, event.SeqNo, event.RevNo, 87 | ) 88 | default: 89 | return 90 | } 91 | 92 | kafkaMessages := c.mapper(e) 93 | 94 | if len(kafkaMessages) == 0 { 95 | ctx.Ack() 96 | return 97 | } 98 | 99 | messages := make([]sKafka.Message, 0, len(kafkaMessages)) 100 | for _, message := range kafkaMessages { 101 | messages = append(messages, sKafka.Message{ 102 | Topic: c.getTopicName(e.CollectionName, message.Topic), 103 | Key: message.Key, 104 | Value: message.Value, 105 | Headers: message.Headers, 106 | }) 107 | } 108 | 109 | batchSizeLimit := c.config.Kafka.ProducerBatchSize 110 | if len(messages) > batchSizeLimit { 111 | chunks := helpers.ChunkSliceWithSize[sKafka.Message](messages, batchSizeLimit) 112 | lastChunkIndex := len(chunks) - 1 113 | for idx, chunk := range chunks { 114 | c.producer.Produce(ctx, e.EventTime, chunk, idx == lastChunkIndex) 115 | } 116 | } else { 117 | c.producer.Produce(ctx, e.EventTime, messages, true) 118 | } 119 | } 120 | 121 | func (c *connector) getTopicName(collectionName string, messageTopic string) string { 122 | if messageTopic != "" { 123 | return messageTopic 124 | } 125 | 126 | topic := c.config.Kafka.CollectionTopicMapping[collectionName] 127 | if topic == "" { 128 | err := fmt.Errorf( 129 | "there is no topic mapping for collection: %s on your configuration", 130 | collectionName, 131 | ) 132 | logger.Log.Error("error while get topic name, err: %v", err) 133 | panic(err) 134 | } 135 | return topic 136 | } 137 | 138 | func newConnector(cfg any, mapper Mapper, sinkResponseHandler kafka.SinkResponseHandler, 139 | completionHandler func(messages []sKafka.Message, err error), 140 | ) (Connector, error) { 141 | c, err := newConfig(cfg) 142 | if err != nil { 143 | return nil, err 144 | } 145 | c.ApplyDefaults() 146 | 147 | connector := &connector{ 148 | mapper: mapper, 149 | config: c, 150 | } 151 | 152 | dcpClient, err := dcp.NewDcp(&c.Dcp, connector.produce) 153 | if err != nil { 154 | logger.Log.Error("dcp error: %v", err) 155 | return nil, err 156 | } 157 | 158 | copyOfConfig := c.Kafka 159 | printConfiguration(copyOfConfig) 160 | 161 | conf := dcpClient.GetConfig() 162 | conf.Checkpoint.Type = "manual" 163 | 164 | kafkaClient, err := createKafkaClient(c) 165 | if err != nil { 166 | return nil, err 167 | } 168 | 169 | if conf.Metadata.Type == MetadataTypeKafka { 170 | setKafkaMetadata(kafkaClient, conf, dcpClient) 171 | } 172 | 173 | connector.dcp = dcpClient 174 | 175 | connector.producer, err = producer.NewProducer(kafkaClient, c, dcpClient.Commit, sinkResponseHandler, completionHandler) 176 | if err != nil { 177 | logger.Log.Error("kafka error: %v", err) 178 | return nil, err 179 | } 180 | 181 | connector.dcp.SetEventHandler(&DcpEventHandler{ 182 | producerBatch: connector.producer.ProducerBatch, 183 | }) 184 | 185 | initializeMetricCollector(connector, dcpClient) 186 | 187 | return connector, nil 188 | } 189 | 190 | func newConfig(cf any) (*config.Connector, error) { 191 | switch v := cf.(type) { 192 | case *config.Connector: 193 | return v, nil 194 | case config.Connector: 195 | return &v, nil 196 | case string: 197 | return newConnectorConfigFromPath(v) 198 | default: 199 | return nil, errors.New("invalid config") 200 | } 201 | } 202 | 203 | func createKafkaClient(cc *config.Connector) (kafka.Client, error) { 204 | kafkaClient := kafka.NewClient(cc) 205 | 206 | topics := make([]string, 0, len(cc.Kafka.CollectionTopicMapping)) 207 | for _, topic := range cc.Kafka.CollectionTopicMapping { 208 | topics = append(topics, topic) 209 | } 210 | 211 | if !cc.Kafka.AllowAutoTopicCreation { 212 | if err := kafkaClient.CheckTopics(topics); err != nil { 213 | logger.Log.Error("collection topic mapping error: %v", err) 214 | return nil, err 215 | } 216 | } 217 | 218 | return kafkaClient, nil 219 | } 220 | 221 | func setKafkaMetadata(kafkaClient kafka.Client, dcpConfig *dcpConfig.Dcp, dcp dcp.Dcp) { 222 | kafkaMetadata := metadata.NewKafkaMetadata(kafkaClient, dcpConfig.Metadata.Config) 223 | dcp.SetMetadata(kafkaMetadata) 224 | } 225 | 226 | func initializeMetricCollector(connector *connector, dcp dcp.Dcp) { 227 | metricCollector := metric.NewMetricCollector(connector.producer) 228 | dcp.SetMetricCollectors(metricCollector) 229 | } 230 | 231 | func newConnectorConfigFromPath(path string) (*config.Connector, error) { 232 | file, err := os.ReadFile(path) 233 | if err != nil { 234 | return nil, err 235 | } 236 | envPattern := regexp.MustCompile(`\${([^}]+)}`) 237 | matches := envPattern.FindAllStringSubmatch(string(file), -1) 238 | for _, match := range matches { 239 | envVar := match[1] 240 | if value, exists := os.LookupEnv(envVar); exists { 241 | updatedFile := strings.ReplaceAll(string(file), "${"+envVar+"}", value) 242 | file = []byte(updatedFile) 243 | } 244 | } 245 | var c config.Connector 246 | err = yaml.Unmarshal(file, &c) 247 | if err != nil { 248 | return nil, err 249 | } 250 | c.ApplyDefaults() 251 | return &c, nil 252 | } 253 | 254 | type ConnectorBuilder struct { 255 | mapper Mapper 256 | config any 257 | sinkResponseHandler kafka.SinkResponseHandler 258 | completionHandler func(messages []sKafka.Message, err error) 259 | } 260 | 261 | func NewConnectorBuilder(config any) *ConnectorBuilder { 262 | return &ConnectorBuilder{ 263 | config: config, 264 | mapper: DefaultMapper, 265 | sinkResponseHandler: nil, 266 | completionHandler: nil, 267 | } 268 | } 269 | 270 | func (c *ConnectorBuilder) SetMapper(mapper Mapper) *ConnectorBuilder { 271 | c.mapper = mapper 272 | return c 273 | } 274 | 275 | func (c *ConnectorBuilder) SetSinkResponseHandler(sinkResponseHandler kafka.SinkResponseHandler) *ConnectorBuilder { 276 | c.sinkResponseHandler = sinkResponseHandler 277 | return c 278 | } 279 | 280 | func (c *ConnectorBuilder) Build() (Connector, error) { 281 | return newConnector(c.config, c.mapper, c.sinkResponseHandler, c.completionHandler) 282 | } 283 | 284 | func (c *ConnectorBuilder) SetLogger(l *logrus.Logger) *ConnectorBuilder { 285 | logger.Log = &logger.Loggers{ 286 | Logrus: l, 287 | } 288 | return c 289 | } 290 | 291 | func printConfiguration(config config.Kafka) { 292 | config.ScramPassword = "*****" 293 | configJSON, _ := jsoniter.Marshal(config) 294 | 295 | dst := &bytes.Buffer{} 296 | if err := json.Compact(dst, configJSON); err != nil { 297 | logger.Log.Error("error while print kafka configuration, err: %v", err) 298 | panic(err) 299 | } 300 | 301 | logger.Log.Info("using kafka config: %v", dst.String()) 302 | } 303 | 304 | func (c *ConnectorBuilder) SetCompletionHandler(f func(messages []sKafka.Message, err error)) *ConnectorBuilder { 305 | c.completionHandler = f 306 | return c 307 | } 308 | -------------------------------------------------------------------------------- /couchbase/event.go: -------------------------------------------------------------------------------- 1 | package couchbase 2 | 3 | import "time" 4 | 5 | type Event struct { 6 | CollectionName string 7 | EventTime time.Time 8 | Key []byte 9 | Value []byte 10 | Cas uint64 11 | VbID uint16 12 | IsDeleted bool 13 | IsExpired bool 14 | IsMutated bool 15 | SeqNo uint64 16 | RevNo uint64 17 | } 18 | 19 | func NewDeleteEvent( 20 | key []byte, value []byte, 21 | collectionName string, eventTime time.Time, cas uint64, vbID uint16, seqNo uint64, revNo uint64, 22 | ) Event { 23 | return Event{ 24 | Key: key, 25 | Value: value, 26 | IsDeleted: true, 27 | CollectionName: collectionName, 28 | EventTime: eventTime, 29 | Cas: cas, 30 | VbID: vbID, 31 | SeqNo: seqNo, 32 | RevNo: revNo, 33 | } 34 | } 35 | 36 | func NewExpireEvent( 37 | key []byte, value []byte, 38 | collectionName string, eventTime time.Time, cas uint64, vbID uint16, seqNo uint64, revNo uint64, 39 | ) Event { 40 | return Event{ 41 | Key: key, 42 | Value: value, 43 | IsExpired: true, 44 | CollectionName: collectionName, 45 | EventTime: eventTime, 46 | Cas: cas, 47 | VbID: vbID, 48 | SeqNo: seqNo, 49 | RevNo: revNo, 50 | } 51 | } 52 | 53 | func NewMutateEvent( 54 | key []byte, value []byte, 55 | collectionName string, eventTime time.Time, cas uint64, vbID uint16, seqNo uint64, revNo uint64, 56 | ) Event { 57 | return Event{ 58 | Key: key, 59 | Value: value, 60 | IsMutated: true, 61 | CollectionName: collectionName, 62 | EventTime: eventTime, 63 | Cas: cas, 64 | VbID: vbID, 65 | SeqNo: seqNo, 66 | RevNo: revNo, 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /dcp_event_handler.go: -------------------------------------------------------------------------------- 1 | package dcpkafka 2 | 3 | import "github.com/Trendyol/go-dcp-kafka/kafka/producer" 4 | 5 | type DcpEventHandler struct { 6 | producerBatch *producer.Batch 7 | } 8 | 9 | func (h *DcpEventHandler) BeforeRebalanceStart() { 10 | } 11 | 12 | func (h *DcpEventHandler) AfterRebalanceStart() { 13 | } 14 | 15 | func (h *DcpEventHandler) BeforeRebalanceEnd() { 16 | } 17 | 18 | func (h *DcpEventHandler) AfterRebalanceEnd() { 19 | } 20 | 21 | func (h *DcpEventHandler) BeforeStreamStart() { 22 | h.producerBatch.PrepareEndRebalancing() 23 | } 24 | 25 | func (h *DcpEventHandler) AfterStreamStart() { 26 | } 27 | 28 | func (h *DcpEventHandler) BeforeStreamStop() { 29 | h.producerBatch.PrepareStartRebalancing() 30 | } 31 | 32 | func (h *DcpEventHandler) AfterStreamStop() { 33 | } 34 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | couchbase: 4 | build: 5 | context: "test/couchbase" 6 | ports: 7 | - "8091:8091" 8 | - "11210:11210" 9 | healthcheck: 10 | test: [ "CMD", "curl", "-f", "http://user:password@localhost:8091/pools/default/buckets/dcp-test" ] 11 | interval: 2s 12 | timeout: 3s 13 | retries: 60 14 | zookeeper: 15 | image: confluentinc/cp-zookeeper:7.6.1 16 | container_name: zookeeper 17 | environment: 18 | ZOOKEEPER_CLIENT_PORT: 2181 19 | ZOOKEEPER_TICK_TIME: 2000 20 | broker: 21 | image: confluentinc/cp-kafka:7.6.1 22 | container_name: broker 23 | ports: 24 | - "9092:9092" 25 | depends_on: 26 | - zookeeper 27 | environment: 28 | KAFKA_BROKER_ID: 1 29 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 30 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT 31 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,PLAINTEXT_INTERNAL://broker:29092 32 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 33 | KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 34 | KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 35 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: true 36 | kowl: 37 | image: quay.io/cloudhut/kowl:master-59f68da 38 | container_name: kowl 39 | depends_on: 40 | - broker 41 | ports: 42 | - "8081:8080" 43 | environment: 44 | KAFKA_BROKERS: 'broker:29092' 45 | healthcheck: 46 | test: wget --no-verbose --tries=1 --spider http://localhost:8080/api/topics/topicname/partitions || exit 1 47 | interval: 2s 48 | timeout: 3s 49 | retries: 60 50 | redpanda-topic-create: 51 | image: docker.redpanda.com/redpandadata/redpanda:v24.2.8 52 | container_name: redpanda-topic-create 53 | depends_on: 54 | - kowl 55 | entrypoint: [ "bash", "-c", "sleep 10 && rpk topic create topicname -c max.message.bytes=1048576 --brokers broker:29092" ] -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | # How to Create a New Connector 2 | 3 | If you want to create a new connector that streams mutations from Couchbase to Kafka, you can use the `go-dcp-kafka` library. 4 | This library provides a simple and flexible way to implement connectors that can be customized to your needs. 5 | 6 | ## Step 1: Installing the Library 7 | 8 | To use the `go-dcp-kafka` library, you first need to install it. You can do this using the `go get` command: 9 | 10 | ``` 11 | $ go get github.com/Trendyol/go-dcp-kafka 12 | 13 | ``` 14 | 15 | ## Step 2: Implementing the Mapper 16 | 17 | The mapper is a function that takes a `couchbase.Event` as input and returns a slice of `message.KafkaMessage`. 18 | The mapper is responsible for converting the Couchbase mutations to Kafka events that will be sent to Kafka. 19 | 20 | Here's an example mapper implementation: 21 | 22 | ```go 23 | package main 24 | 25 | import ( 26 | "github.com/Trendyol/go-dcp-kafka/couchbase" 27 | "github.com/Trendyol/go-dcp-kafka/kafka/message" 28 | ) 29 | 30 | func mapper(event couchbase.Event) []message.KafkaMessage { 31 | // here you can filter and transform events 32 | return []message.KafkaMessage{ 33 | { 34 | Headers: nil, 35 | Key: event.Key, 36 | Value: event.Value, 37 | }, 38 | } 39 | } 40 | ``` 41 | 42 | ## Step 3: Implementing the SinkResponseHandler 43 | 44 | This function is called after the event is published and takes `message.KafkaMessage` as a parameter. 45 | Here's an example SinkResponseHandler implementation: 46 | 47 | ```go 48 | type sinkResponseHandler struct { 49 | } 50 | 51 | func (s *sinkResponseHandler) OnSuccess(ctx *kafka.SinkResponseHandlerContext) { 52 | fmt.Printf("OnSuccess %v\n", string(ctx.Message.Value)) 53 | } 54 | 55 | func (s *sinkResponseHandler) OnError(ctx *kafka.SinkResponseHandlerContext) { 56 | fmt.Printf("OnError %v\n", string(ctx.Message.Value)) 57 | } 58 | ``` 59 | 60 | ## Step 4: Configuring the Connector 61 | 62 | The configuration for the connector is provided via a YAML file. Here's an example [configuration](https://github.com/Trendyol/go-dcp-kafka/blob/master/example/config.yml): 63 | 64 | You can find explanation of [configurations](https://github.com/Trendyol/go-dcp#configuration) 65 | 66 | You can pass this configuration file to the connector by providing the path to the file when creating the connector: 67 | ```go 68 | connector, err := dcpkafka.NewConnectorBuilder("config.yml"). 69 | SetMapper(mapper). 70 | SetSinkResponseHandler(&sinkResponseHandler{}). // if you want to add callback func 71 | Build() 72 | 73 | if err != nil { 74 | panic(err) 75 | } 76 | ``` 77 | 78 | ## Step 5: Starting and Closing the Connector 79 | 80 | Once you have implemented the mapper and configured the connector, you can start/stop the connector: 81 | 82 | ```go 83 | defer connector.Close() 84 | connector.Start() 85 | ``` 86 | 87 | This will start the connector, which will continuously listen for Couchbase mutations and stream events to Kafka according to the configured mapper. 88 | 89 | ## Monitoring 90 | 91 | The connector features an API that allows for management and observation, and it also exposes multiple metrics to facilitate these tasks. 92 | 93 | You can find explanation [here](https://github.com/Trendyol/go-dcp#monitoring) -------------------------------------------------------------------------------- /example/default-completion/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20-alpine as builder 2 | 3 | WORKDIR /project 4 | 5 | COPY go.mod go.sum ./ 6 | COPY main.go ./ 7 | COPY config.yml ./config.yml 8 | 9 | RUN go mod download 10 | RUN CGO_ENABLED=0 go build -a -o example main.go 11 | 12 | FROM alpine:3.17.0 13 | 14 | WORKDIR /app 15 | 16 | RUN apk --no-cache add ca-certificates 17 | 18 | USER nobody 19 | COPY --from=builder --chown=nobody:nobody /project/example . 20 | COPY --from=builder --chown=nobody:nobody /project/config.yml ./config.yml 21 | 22 | ENTRYPOINT ["./example"] -------------------------------------------------------------------------------- /example/default-completion/config.yml: -------------------------------------------------------------------------------- 1 | hosts: 2 | - localhost:8091 3 | username: user 4 | password: password 5 | bucketName: dcp-test 6 | scopeName: _default 7 | collectionNames: 8 | - _default 9 | metadata: 10 | type: couchbase 11 | config: 12 | bucket: checkpoint-bucket-name 13 | scope: _default 14 | collection: _default 15 | dcp: 16 | group: 17 | name: groupName 18 | kafka: 19 | collectionTopicMapping: 20 | _default: topicname 21 | brokers: 22 | - localhost:9092 23 | # SSL configurations 24 | # 25 | # secureConnection: true 26 | # Config support env variable "$HOME/example/..." 27 | # rootCAPath: "example/stretch-kafka/rootCA.pem" 28 | # interCAPath: "example/stretch-kafka/interCA.pem" 29 | # scramUsername: "username" 30 | # scramPassword: "password" -------------------------------------------------------------------------------- /example/default-completion/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.20 4 | 5 | replace github.com/Trendyol/go-dcp-kafka => ./../.. 6 | 7 | require github.com/Trendyol/go-dcp-kafka v0.0.0 8 | 9 | require ( 10 | github.com/Trendyol/go-dcp v1.2.6 // indirect 11 | github.com/andybalholm/brotli v1.1.1 // indirect 12 | github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect 13 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect 14 | github.com/beorn7/perks v1.0.1 // indirect 15 | github.com/bytedance/sonic v1.12.8 // indirect 16 | github.com/bytedance/sonic/loader v0.2.2 // indirect 17 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 18 | github.com/cloudwego/base64x v0.1.5 // indirect 19 | github.com/couchbase/gocbcore/v10 v10.5.2 // indirect 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 22 | github.com/go-logr/logr v1.4.1 // indirect 23 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 24 | github.com/go-openapi/jsonreference v0.20.2 // indirect 25 | github.com/go-openapi/swag v0.22.3 // indirect 26 | github.com/gofiber/fiber/v2 v2.52.5 // indirect 27 | github.com/gogo/protobuf v1.3.2 // indirect 28 | github.com/golang/protobuf v1.5.4 // indirect 29 | github.com/golang/snappy v0.0.4 // indirect 30 | github.com/google/gnostic-models v0.6.8 // indirect 31 | github.com/google/gofuzz v1.2.0 // indirect 32 | github.com/google/uuid v1.6.0 // indirect 33 | github.com/josharian/intern v1.0.0 // indirect 34 | github.com/json-iterator/go v1.1.12 // indirect 35 | github.com/klauspost/compress v1.17.11 // indirect 36 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 37 | github.com/mailru/easyjson v0.7.7 // indirect 38 | github.com/mattn/go-colorable v0.1.13 // indirect 39 | github.com/mattn/go-isatty v0.0.20 // indirect 40 | github.com/mattn/go-runewidth v0.0.15 // indirect 41 | github.com/mhmtszr/concurrent-swiss-map v1.0.8 // indirect 42 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 43 | github.com/modern-go/reflect2 v1.0.2 // indirect 44 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 45 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 46 | github.com/prometheus/client_golang v1.20.5 // indirect 47 | github.com/prometheus/client_model v0.6.1 // indirect 48 | github.com/prometheus/common v0.58.0 // indirect 49 | github.com/prometheus/procfs v0.15.1 // indirect 50 | github.com/rivo/uniseg v0.4.7 // indirect 51 | github.com/segmentio/kafka-go v0.4.47 // indirect 52 | github.com/sirupsen/logrus v1.9.3 // indirect 53 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 54 | github.com/valyala/bytebufferpool v1.0.0 // indirect 55 | github.com/valyala/fasthttp v1.57.0 // indirect 56 | github.com/valyala/tcplisten v1.0.0 // indirect 57 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 58 | github.com/xdg-go/scram v1.1.2 // indirect 59 | github.com/xdg-go/stringprep v1.0.4 // indirect 60 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect 61 | golang.org/x/net v0.33.0 // indirect 62 | golang.org/x/oauth2 v0.22.0 // indirect 63 | golang.org/x/sync v0.10.0 // indirect 64 | golang.org/x/sys v0.28.0 // indirect 65 | golang.org/x/term v0.27.0 // indirect 66 | golang.org/x/text v0.21.0 // indirect 67 | golang.org/x/time v0.3.0 // indirect 68 | google.golang.org/protobuf v1.36.4 // indirect 69 | gopkg.in/inf.v0 v0.9.1 // indirect 70 | gopkg.in/yaml.v2 v2.4.0 // indirect 71 | gopkg.in/yaml.v3 v3.0.1 // indirect 72 | k8s.io/api v0.29.4 // indirect 73 | k8s.io/apimachinery v0.29.4 // indirect 74 | k8s.io/client-go v0.29.4 // indirect 75 | k8s.io/klog/v2 v2.110.1 // indirect 76 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect 77 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 78 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 79 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 80 | sigs.k8s.io/yaml v1.3.0 // indirect 81 | ) 82 | -------------------------------------------------------------------------------- /example/default-completion/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/Trendyol/go-dcp-kafka" 5 | "github.com/Trendyol/go-dcp-kafka/kafka/producer" 6 | ) 7 | 8 | func main() { 9 | c, err := dcpkafka.NewConnectorBuilder("config.yml"). 10 | SetMapper(dcpkafka.DefaultMapper). 11 | SetCompletionHandler(producer.DefaultCompletion). 12 | Build() 13 | if err != nil { 14 | panic(err) 15 | } 16 | 17 | defer c.Close() 18 | c.Start() 19 | } 20 | -------------------------------------------------------------------------------- /example/default-mapper/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20-alpine as builder 2 | 3 | WORKDIR /project 4 | 5 | COPY go.mod go.sum ./ 6 | COPY main.go ./ 7 | COPY config.yml ./config.yml 8 | 9 | RUN go mod download 10 | RUN CGO_ENABLED=0 go build -a -o example main.go 11 | 12 | FROM alpine:3.17.0 13 | 14 | WORKDIR /app 15 | 16 | RUN apk --no-cache add ca-certificates 17 | 18 | USER nobody 19 | COPY --from=builder --chown=nobody:nobody /project/example . 20 | COPY --from=builder --chown=nobody:nobody /project/config.yml ./config.yml 21 | 22 | ENTRYPOINT ["./example"] -------------------------------------------------------------------------------- /example/default-mapper/config.yml: -------------------------------------------------------------------------------- 1 | hosts: 2 | - localhost:8091 3 | username: user 4 | password: password 5 | bucketName: dcp-test 6 | scopeName: _default 7 | collectionNames: 8 | - _default 9 | metadata: 10 | type: couchbase 11 | config: 12 | bucket: checkpoint-bucket-name 13 | scope: _default 14 | collection: _default 15 | dcp: 16 | group: 17 | name: groupName 18 | kafka: 19 | collectionTopicMapping: 20 | _default: topicname 21 | brokers: 22 | - localhost:9092 23 | # SSL configurations 24 | # 25 | # secureConnection: true 26 | # Config support env variable "$HOME/example/..." 27 | # rootCAPath: "example/stretch-kafka/rootCA.pem" 28 | # interCAPath: "example/stretch-kafka/interCA.pem" 29 | # scramUsername: "username" 30 | # scramPassword: "password" -------------------------------------------------------------------------------- /example/default-mapper/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.20 4 | 5 | replace github.com/Trendyol/go-dcp-kafka => ./../.. 6 | 7 | require github.com/Trendyol/go-dcp-kafka v0.0.0 8 | 9 | require ( 10 | github.com/Trendyol/go-dcp v1.2.6 // indirect 11 | github.com/andybalholm/brotli v1.1.1 // indirect 12 | github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect 13 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect 14 | github.com/beorn7/perks v1.0.1 // indirect 15 | github.com/bytedance/sonic v1.12.8 // indirect 16 | github.com/bytedance/sonic/loader v0.2.2 // indirect 17 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 18 | github.com/cloudwego/base64x v0.1.5 // indirect 19 | github.com/couchbase/gocbcore/v10 v10.5.2 // indirect 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 22 | github.com/go-logr/logr v1.4.1 // indirect 23 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 24 | github.com/go-openapi/jsonreference v0.20.2 // indirect 25 | github.com/go-openapi/swag v0.22.3 // indirect 26 | github.com/gofiber/fiber/v2 v2.52.5 // indirect 27 | github.com/gogo/protobuf v1.3.2 // indirect 28 | github.com/golang/protobuf v1.5.4 // indirect 29 | github.com/golang/snappy v0.0.4 // indirect 30 | github.com/google/gnostic-models v0.6.8 // indirect 31 | github.com/google/gofuzz v1.2.0 // indirect 32 | github.com/google/uuid v1.6.0 // indirect 33 | github.com/josharian/intern v1.0.0 // indirect 34 | github.com/json-iterator/go v1.1.12 // indirect 35 | github.com/klauspost/compress v1.17.11 // indirect 36 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 37 | github.com/mailru/easyjson v0.7.7 // indirect 38 | github.com/mattn/go-colorable v0.1.13 // indirect 39 | github.com/mattn/go-isatty v0.0.20 // indirect 40 | github.com/mattn/go-runewidth v0.0.15 // indirect 41 | github.com/mhmtszr/concurrent-swiss-map v1.0.8 // indirect 42 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 43 | github.com/modern-go/reflect2 v1.0.2 // indirect 44 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 45 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 46 | github.com/prometheus/client_golang v1.20.5 // indirect 47 | github.com/prometheus/client_model v0.6.1 // indirect 48 | github.com/prometheus/common v0.58.0 // indirect 49 | github.com/prometheus/procfs v0.15.1 // indirect 50 | github.com/rivo/uniseg v0.4.7 // indirect 51 | github.com/segmentio/kafka-go v0.4.47 // indirect 52 | github.com/sirupsen/logrus v1.9.3 // indirect 53 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 54 | github.com/valyala/bytebufferpool v1.0.0 // indirect 55 | github.com/valyala/fasthttp v1.57.0 // indirect 56 | github.com/valyala/tcplisten v1.0.0 // indirect 57 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 58 | github.com/xdg-go/scram v1.1.2 // indirect 59 | github.com/xdg-go/stringprep v1.0.4 // indirect 60 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect 61 | golang.org/x/net v0.33.0 // indirect 62 | golang.org/x/oauth2 v0.22.0 // indirect 63 | golang.org/x/sync v0.10.0 // indirect 64 | golang.org/x/sys v0.28.0 // indirect 65 | golang.org/x/term v0.27.0 // indirect 66 | golang.org/x/text v0.21.0 // indirect 67 | golang.org/x/time v0.3.0 // indirect 68 | google.golang.org/protobuf v1.36.4 // indirect 69 | gopkg.in/inf.v0 v0.9.1 // indirect 70 | gopkg.in/yaml.v2 v2.4.0 // indirect 71 | gopkg.in/yaml.v3 v3.0.1 // indirect 72 | k8s.io/api v0.29.4 // indirect 73 | k8s.io/apimachinery v0.29.4 // indirect 74 | k8s.io/client-go v0.29.4 // indirect 75 | k8s.io/klog/v2 v2.110.1 // indirect 76 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect 77 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 78 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 79 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 80 | sigs.k8s.io/yaml v1.3.0 // indirect 81 | ) 82 | -------------------------------------------------------------------------------- /example/default-mapper/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/Trendyol/go-dcp-kafka" 5 | ) 6 | 7 | func main() { 8 | c, err := dcpkafka.NewConnectorBuilder("config.yml").SetMapper(dcpkafka.DefaultMapper).Build() 9 | if err != nil { 10 | panic(err) 11 | } 12 | 13 | defer c.Close() 14 | c.Start() 15 | } 16 | -------------------------------------------------------------------------------- /example/grafana/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20-alpine AS builder 2 | 3 | WORKDIR /project 4 | 5 | COPY go.mod go.sum ./ 6 | COPY main.go ./ 7 | COPY config.yml ./config.yml 8 | 9 | RUN go mod download 10 | RUN CGO_ENABLED=0 go build -a -o example main.go 11 | 12 | FROM alpine:3.17.1 13 | 14 | WORKDIR /app 15 | 16 | RUN apk --no-cache add ca-certificates 17 | 18 | USER nobody 19 | COPY --from=builder --chown=nobody:nobody /project/example . 20 | COPY --from=builder --chown=nobody:nobody /project/config.yml ./config.yml 21 | 22 | ENTRYPOINT ["./example"] -------------------------------------------------------------------------------- /example/grafana/config.yml: -------------------------------------------------------------------------------- 1 | hosts: 2 | - couchbase:8091 3 | username: user 4 | password: password 5 | bucketName: dcp-test 6 | dcp: 7 | group: 8 | name: group-test 9 | metadata: 10 | type: couchbase 11 | config: 12 | bucket: dcp-test 13 | scope: _default 14 | collection: _default 15 | kafka: 16 | collectionTopicMapping: 17 | _default: topic-test 18 | brokers: 19 | - redpanda:9092 20 | -------------------------------------------------------------------------------- /example/grafana/couchbase/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM couchbase:7.6.3 2 | 3 | COPY configure.sh /opt/couchbase 4 | RUN chmod +x /opt/couchbase/configure.sh 5 | 6 | CMD ["/opt/couchbase/configure.sh"] 7 | -------------------------------------------------------------------------------- /example/grafana/couchbase/configure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Start Couchbase server in background 4 | /entrypoint.sh couchbase-server & 5 | 6 | # Wait for Couchbase to start 7 | sleep 10 8 | 9 | # Setup cluster 10 | couchbase-cli cluster-init -c 127.0.0.1:8091 \ 11 | --cluster-username user \ 12 | --cluster-password password \ 13 | --services data,index,query,fts,eventing,analytics \ 14 | --cluster-ramsize 1024 \ 15 | --cluster-index-ramsize 256 \ 16 | --cluster-fts-ramsize 256 \ 17 | --cluster-eventing-ramsize 256 \ 18 | --cluster-analytics-ramsize 1024 19 | 20 | # Create a bucket 21 | couchbase-cli bucket-create -c 127.0.0.1:8091 \ 22 | --username user \ 23 | --password password \ 24 | --bucket dcp-test \ 25 | --bucket-type couchbase \ 26 | --bucket-ramsize 256 27 | 28 | # Keep the container running 29 | tail -f /dev/null 30 | -------------------------------------------------------------------------------- /example/grafana/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.9' 2 | 3 | volumes: 4 | prometheus_data: { } 5 | grafana_data: { } 6 | 7 | services: 8 | go-dcp-kafka: 9 | build: . 10 | ports: 11 | - "8080:8080" 12 | expose: 13 | - 8080 14 | depends_on: 15 | - couchbase 16 | - redpanda 17 | 18 | couchbase: 19 | build: ./couchbase 20 | ports: 21 | - "8091-8094:8091-8094" 22 | - "11210:11210" 23 | expose: 24 | - 8091 25 | environment: 26 | - COUCHBASE_ADMINISTRATOR_USERNAME=user 27 | - COUCHBASE_ADMINISTRATOR_PASSWORD=password 28 | 29 | redpanda: 30 | image: docker.redpanda.com/redpandadata/redpanda 31 | container_name: redpanda 32 | command: 33 | - redpanda 34 | - start 35 | - --smp 36 | - '1' 37 | - --reserve-memory 38 | - 0M 39 | - --overprovisioned 40 | - --node-id 41 | - '0' 42 | - --kafka-addr 43 | - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092 44 | - --advertise-kafka-addr 45 | - PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092 46 | - --pandaproxy-addr 47 | - PLAINTEXT://0.0.0.0:28082,OUTSIDE://0.0.0.0:8082 48 | - --advertise-pandaproxy-addr 49 | - PLAINTEXT://redpanda:28082,OUTSIDE://localhost:8082 50 | ports: 51 | - 8081:8081 52 | - 8082:8082 53 | - 9092:9092 54 | - 28082:28082 55 | - 29092:29092 56 | redpanda-topic-create: 57 | image: docker.redpanda.com/redpandadata/redpanda 58 | container_name: redpanda-topic-create 59 | entrypoint: [ "bash", "-c", "sleep 5 && rpk topic create topic-test --brokers redpanda:29092" ] 60 | depends_on: 61 | - redpanda 62 | restart: "no" 63 | 64 | prometheus: 65 | image: prom/prometheus:v2.53.1 66 | container_name: prometheus 67 | volumes: 68 | - ./prometheus:/etc/prometheus 69 | - prometheus_data:/prometheus 70 | ports: 71 | - "9090:9090" 72 | expose: 73 | - 9090 74 | 75 | grafana: 76 | image: grafana/grafana:10.0.0 77 | container_name: grafana 78 | volumes: 79 | - grafana_data:/var/lib/grafana 80 | - ./grafana/provisioning:/etc/grafana/provisioning 81 | environment: 82 | - GF_AUTH_DISABLE_LOGIN_FORM=true 83 | - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin 84 | - GF_AUTH_ANONYMOUS_ENABLED=true 85 | ports: 86 | - "3000:3000" 87 | expose: 88 | - 3000 89 | depends_on: 90 | - prometheus 91 | -------------------------------------------------------------------------------- /example/grafana/go.mod: -------------------------------------------------------------------------------- 1 | module grafana 2 | 3 | go 1.20 4 | 5 | replace github.com/Trendyol/go-dcp-kafka => ../../. 6 | 7 | require ( 8 | github.com/Trendyol/go-dcp-kafka v0.0.0 9 | github.com/couchbase/gocb/v2 v2.9.1 10 | ) 11 | 12 | require ( 13 | github.com/Trendyol/go-dcp v1.2.6 // indirect 14 | github.com/andybalholm/brotli v1.1.1 // indirect 15 | github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect 16 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect 17 | github.com/beorn7/perks v1.0.1 // indirect 18 | github.com/bytedance/sonic v1.12.8 // indirect 19 | github.com/bytedance/sonic/loader v0.2.2 // indirect 20 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 21 | github.com/cloudwego/base64x v0.1.5 // indirect 22 | github.com/couchbase/gocbcore/v10 v10.5.2 // indirect 23 | github.com/couchbase/gocbcoreps v0.1.3 // indirect 24 | github.com/couchbase/goprotostellar v1.0.2 // indirect 25 | github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20240607131231-fb385523de28 // indirect 26 | github.com/davecgh/go-spew v1.1.1 // indirect 27 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 28 | github.com/go-logr/logr v1.4.1 // indirect 29 | github.com/go-logr/stdr v1.2.2 // indirect 30 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 31 | github.com/go-openapi/jsonreference v0.20.2 // indirect 32 | github.com/go-openapi/swag v0.22.3 // indirect 33 | github.com/gofiber/fiber/v2 v2.52.5 // indirect 34 | github.com/gogo/protobuf v1.3.2 // indirect 35 | github.com/golang/protobuf v1.5.4 // indirect 36 | github.com/golang/snappy v0.0.4 // indirect 37 | github.com/google/gnostic-models v0.6.8 // indirect 38 | github.com/google/gofuzz v1.2.0 // indirect 39 | github.com/google/uuid v1.6.0 // indirect 40 | github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect 41 | github.com/josharian/intern v1.0.0 // indirect 42 | github.com/json-iterator/go v1.1.12 // indirect 43 | github.com/klauspost/compress v1.17.11 // indirect 44 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 45 | github.com/mailru/easyjson v0.7.7 // indirect 46 | github.com/mattn/go-colorable v0.1.13 // indirect 47 | github.com/mattn/go-isatty v0.0.20 // indirect 48 | github.com/mattn/go-runewidth v0.0.15 // indirect 49 | github.com/mhmtszr/concurrent-swiss-map v1.0.8 // indirect 50 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 51 | github.com/modern-go/reflect2 v1.0.2 // indirect 52 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 53 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 54 | github.com/prometheus/client_golang v1.20.5 // indirect 55 | github.com/prometheus/client_model v0.6.1 // indirect 56 | github.com/prometheus/common v0.58.0 // indirect 57 | github.com/prometheus/procfs v0.15.1 // indirect 58 | github.com/rivo/uniseg v0.4.7 // indirect 59 | github.com/segmentio/kafka-go v0.4.47 // indirect 60 | github.com/sirupsen/logrus v1.9.3 // indirect 61 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 62 | github.com/valyala/bytebufferpool v1.0.0 // indirect 63 | github.com/valyala/fasthttp v1.57.0 // indirect 64 | github.com/valyala/tcplisten v1.0.0 // indirect 65 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 66 | github.com/xdg-go/scram v1.1.2 // indirect 67 | github.com/xdg-go/stringprep v1.0.4 // indirect 68 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect 69 | go.opentelemetry.io/otel v1.24.0 // indirect 70 | go.opentelemetry.io/otel/metric v1.24.0 // indirect 71 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 72 | go.uber.org/multierr v1.11.0 // indirect 73 | go.uber.org/zap v1.27.0 // indirect 74 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect 75 | golang.org/x/net v0.33.0 // indirect 76 | golang.org/x/oauth2 v0.22.0 // indirect 77 | golang.org/x/sync v0.10.0 // indirect 78 | golang.org/x/sys v0.28.0 // indirect 79 | golang.org/x/term v0.27.0 // indirect 80 | golang.org/x/text v0.21.0 // indirect 81 | golang.org/x/time v0.3.0 // indirect 82 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 83 | google.golang.org/grpc v1.66.0 // indirect 84 | google.golang.org/protobuf v1.36.4 // indirect 85 | gopkg.in/inf.v0 v0.9.1 // indirect 86 | gopkg.in/yaml.v2 v2.4.0 // indirect 87 | gopkg.in/yaml.v3 v3.0.1 // indirect 88 | k8s.io/api v0.29.4 // indirect 89 | k8s.io/apimachinery v0.29.4 // indirect 90 | k8s.io/client-go v0.29.4 // indirect 91 | k8s.io/klog/v2 v2.110.1 // indirect 92 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect 93 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 94 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 95 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 96 | sigs.k8s.io/yaml v1.3.0 // indirect 97 | ) 98 | -------------------------------------------------------------------------------- /example/grafana/grafana/grafana-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Trendyol/go-dcp-kafka/170bec8a64b8ad530550962fa34ff28e5c657f22/example/grafana/grafana/grafana-1.png -------------------------------------------------------------------------------- /example/grafana/grafana/grafana-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Trendyol/go-dcp-kafka/170bec8a64b8ad530550962fa34ff28e5c657f22/example/grafana/grafana/grafana-2.png -------------------------------------------------------------------------------- /example/grafana/grafana/grafana-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Trendyol/go-dcp-kafka/170bec8a64b8ad530550962fa34ff28e5c657f22/example/grafana/grafana/grafana-3.png -------------------------------------------------------------------------------- /example/grafana/grafana/provisioning/dashboards/dashboards.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: "Dashboard provider" 5 | orgId: 1 6 | type: file 7 | disableDeletion: false 8 | updateIntervalSeconds: 10 9 | allowUiUpdates: true 10 | options: 11 | path: /etc/grafana/provisioning 12 | foldersFromFilesStructure: true 13 | -------------------------------------------------------------------------------- /example/grafana/grafana/provisioning/datasources/datasource.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: go-dcp-kafka 5 | type: prometheus 6 | access: proxy 7 | orgId: 1 8 | url: http://prometheus:9090 9 | basicAuth: false 10 | isDefault: true 11 | editable: true 12 | -------------------------------------------------------------------------------- /example/grafana/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | dcpkafka "github.com/Trendyol/go-dcp-kafka" 6 | "github.com/Trendyol/go-dcp-kafka/couchbase" 7 | "github.com/Trendyol/go-dcp-kafka/kafka/message" 8 | "github.com/couchbase/gocb/v2" 9 | "log" 10 | "time" 11 | ) 12 | 13 | func mapper(event couchbase.Event) []message.KafkaMessage { 14 | log.Println("Captured: ", string(event.Key)) 15 | return []message.KafkaMessage{ 16 | { 17 | Headers: nil, 18 | Key: event.Key, 19 | Value: event.Value, 20 | }, 21 | } 22 | } 23 | 24 | func main() { 25 | time.Sleep(15 * time.Second) //wait for couchbase container initialize 26 | 27 | go seedCouchbaseBucket() 28 | 29 | connector, err := dcpkafka.NewConnectorBuilder("config.yml"). 30 | SetMapper(mapper). 31 | Build() 32 | 33 | if err != nil { 34 | panic(err) 35 | } 36 | 37 | defer connector.Close() 38 | connector.Start() 39 | } 40 | 41 | func seedCouchbaseBucket() { 42 | cluster, err := gocb.Connect("couchbase://couchbase", gocb.ClusterOptions{ 43 | Username: "user", 44 | Password: "password", 45 | }) 46 | if err != nil { 47 | log.Fatal(err) 48 | } 49 | 50 | bucket := cluster.Bucket("dcp-test") 51 | err = bucket.WaitUntilReady(5*time.Second, nil) 52 | if err != nil { 53 | log.Fatal(err) 54 | } 55 | 56 | collection := bucket.DefaultCollection() 57 | 58 | counter := 0 59 | for { 60 | counter++ 61 | documentID := fmt.Sprintf("doc-%d", counter) 62 | document := map[string]interface{}{ 63 | "counter": counter, 64 | "message": "Hello Couchbase", 65 | "time": time.Now().Format(time.RFC3339), 66 | } 67 | _, err := collection.Upsert(documentID, document, nil) 68 | if err != nil { 69 | log.Println("Error inserting document:", err) 70 | } else { 71 | log.Println("Inserted document:", documentID) 72 | } 73 | time.Sleep(1 * time.Second) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /example/grafana/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 5s 3 | 4 | scrape_configs: 5 | - job_name: "go-dcp-kafka" 6 | static_configs: 7 | - targets: ["go-dcp-kafka:8080"] 8 | -------------------------------------------------------------------------------- /example/simple-logger/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20-alpine as builder 2 | 3 | WORKDIR /project 4 | 5 | COPY go.mod go.sum ./ 6 | COPY main.go ./ 7 | COPY config.yml ./config.yml 8 | 9 | RUN go mod download 10 | RUN CGO_ENABLED=0 go build -a -o example main.go 11 | 12 | FROM alpine:3.17.0 13 | 14 | WORKDIR /app 15 | 16 | RUN apk --no-cache add ca-certificates 17 | 18 | USER nobody 19 | COPY --from=builder --chown=nobody:nobody /project/example . 20 | COPY --from=builder --chown=nobody:nobody /project/config.yml ./config.yml 21 | 22 | ENTRYPOINT ["./example"] -------------------------------------------------------------------------------- /example/simple-logger/config.yml: -------------------------------------------------------------------------------- 1 | hosts: 2 | - localhost:8091 3 | username: user 4 | password: password 5 | bucketName: dcp-test 6 | scopeName: _default 7 | collectionNames: 8 | - _default 9 | metadata: 10 | type: couchbase 11 | config: 12 | bucket: checkpoint-bucket-name 13 | scope: _default 14 | collection: _default 15 | dcp: 16 | group: 17 | name: groupName 18 | kafka: 19 | collectionTopicMapping: 20 | _default: topicname 21 | brokers: 22 | - localhost:9092 23 | # SSL configurations 24 | # 25 | # secureConnection: true 26 | # Config support env variable "$HOME/example/..." 27 | # rootCAPath: "example/stretch-kafka/rootCA.pem" 28 | # interCAPath: "example/stretch-kafka/interCA.pem" 29 | # scramUsername: "username" 30 | # scramPassword: "password" 31 | -------------------------------------------------------------------------------- /example/simple-logger/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.20 4 | 5 | replace github.com/Trendyol/go-dcp-kafka => ./../.. 6 | 7 | require ( 8 | github.com/Trendyol/go-dcp-kafka v0.0.0 9 | github.com/sirupsen/logrus v1.9.3 10 | ) 11 | 12 | require ( 13 | github.com/Trendyol/go-dcp v1.2.6 // indirect 14 | github.com/andybalholm/brotli v1.1.1 // indirect 15 | github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect 16 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect 17 | github.com/beorn7/perks v1.0.1 // indirect 18 | github.com/bytedance/sonic v1.12.8 // indirect 19 | github.com/bytedance/sonic/loader v0.2.2 // indirect 20 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 21 | github.com/cloudwego/base64x v0.1.5 // indirect 22 | github.com/couchbase/gocbcore/v10 v10.5.2 // indirect 23 | github.com/davecgh/go-spew v1.1.1 // indirect 24 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 25 | github.com/go-logr/logr v1.4.1 // indirect 26 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 27 | github.com/go-openapi/jsonreference v0.20.2 // indirect 28 | github.com/go-openapi/swag v0.22.3 // indirect 29 | github.com/gofiber/fiber/v2 v2.52.5 // indirect 30 | github.com/gogo/protobuf v1.3.2 // indirect 31 | github.com/golang/protobuf v1.5.4 // indirect 32 | github.com/golang/snappy v0.0.4 // indirect 33 | github.com/google/gnostic-models v0.6.8 // indirect 34 | github.com/google/gofuzz v1.2.0 // indirect 35 | github.com/google/uuid v1.6.0 // indirect 36 | github.com/josharian/intern v1.0.0 // indirect 37 | github.com/json-iterator/go v1.1.12 // indirect 38 | github.com/klauspost/compress v1.17.11 // indirect 39 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 40 | github.com/mailru/easyjson v0.7.7 // indirect 41 | github.com/mattn/go-colorable v0.1.13 // indirect 42 | github.com/mattn/go-isatty v0.0.20 // indirect 43 | github.com/mattn/go-runewidth v0.0.15 // indirect 44 | github.com/mhmtszr/concurrent-swiss-map v1.0.8 // indirect 45 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 46 | github.com/modern-go/reflect2 v1.0.2 // indirect 47 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 48 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 49 | github.com/prometheus/client_golang v1.20.5 // indirect 50 | github.com/prometheus/client_model v0.6.1 // indirect 51 | github.com/prometheus/common v0.58.0 // indirect 52 | github.com/prometheus/procfs v0.15.1 // indirect 53 | github.com/rivo/uniseg v0.4.7 // indirect 54 | github.com/segmentio/kafka-go v0.4.47 // indirect 55 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 56 | github.com/valyala/bytebufferpool v1.0.0 // indirect 57 | github.com/valyala/fasthttp v1.57.0 // indirect 58 | github.com/valyala/tcplisten v1.0.0 // indirect 59 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 60 | github.com/xdg-go/scram v1.1.2 // indirect 61 | github.com/xdg-go/stringprep v1.0.4 // indirect 62 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect 63 | golang.org/x/net v0.33.0 // indirect 64 | golang.org/x/oauth2 v0.22.0 // indirect 65 | golang.org/x/sync v0.10.0 // indirect 66 | golang.org/x/sys v0.28.0 // indirect 67 | golang.org/x/term v0.27.0 // indirect 68 | golang.org/x/text v0.21.0 // indirect 69 | golang.org/x/time v0.3.0 // indirect 70 | google.golang.org/protobuf v1.36.4 // indirect 71 | gopkg.in/inf.v0 v0.9.1 // indirect 72 | gopkg.in/yaml.v2 v2.4.0 // indirect 73 | gopkg.in/yaml.v3 v3.0.1 // indirect 74 | k8s.io/api v0.29.4 // indirect 75 | k8s.io/apimachinery v0.29.4 // indirect 76 | k8s.io/client-go v0.29.4 // indirect 77 | k8s.io/klog/v2 v2.110.1 // indirect 78 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect 79 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 80 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 81 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 82 | sigs.k8s.io/yaml v1.3.0 // indirect 83 | ) 84 | -------------------------------------------------------------------------------- /example/simple-logger/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | dcpkafka "github.com/Trendyol/go-dcp-kafka" 5 | "github.com/Trendyol/go-dcp-kafka/couchbase" 6 | "github.com/Trendyol/go-dcp-kafka/kafka/message" 7 | "github.com/sirupsen/logrus" 8 | ) 9 | 10 | func mapper(event couchbase.Event) []message.KafkaMessage { 11 | // return nil if you wish to discard the event 12 | return []message.KafkaMessage{ 13 | { 14 | Headers: nil, 15 | Key: event.Key, 16 | Value: event.Value, 17 | }, 18 | } 19 | } 20 | 21 | func main() { 22 | logger := createLogger() 23 | 24 | c, err := dcpkafka.NewConnectorBuilder("config.yml"). 25 | SetMapper(mapper). 26 | SetLogger(logger). 27 | Build() 28 | if err != nil { 29 | panic(err) 30 | } 31 | 32 | defer c.Close() 33 | c.Start() 34 | } 35 | 36 | func createLogger() *logrus.Logger { 37 | logger := logrus.New() 38 | logger.SetLevel(logrus.InfoLevel) 39 | 40 | formatter := &logrus.JSONFormatter{ 41 | FieldMap: logrus.FieldMap{ 42 | logrus.FieldKeyMsg: "ms", 43 | }, 44 | } 45 | 46 | logger.SetFormatter(formatter) 47 | return logger 48 | } 49 | -------------------------------------------------------------------------------- /example/simple-rejection-log-sink-response-handler/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20-alpine as builder 2 | 3 | WORKDIR /project 4 | 5 | COPY go.mod go.sum ./ 6 | COPY main.go ./ 7 | COPY config.yml ./config.yml 8 | 9 | RUN go mod download 10 | RUN CGO_ENABLED=0 go build -a -o example main.go 11 | 12 | FROM alpine:3.17.0 13 | 14 | WORKDIR /app 15 | 16 | RUN apk --no-cache add ca-certificates 17 | 18 | USER nobody 19 | COPY --from=builder --chown=nobody:nobody /project/example . 20 | COPY --from=builder --chown=nobody:nobody /project/config.yml ./config.yml 21 | 22 | ENTRYPOINT ["./example"] -------------------------------------------------------------------------------- /example/simple-rejection-log-sink-response-handler/config.yml: -------------------------------------------------------------------------------- 1 | hosts: 2 | - localhost:8091 3 | username: user 4 | password: password 5 | bucketName: dcp-test 6 | scopeName: _default 7 | collectionNames: 8 | - _default 9 | metadata: 10 | type: couchbase 11 | config: 12 | bucket: checkpoint-bucket-name 13 | scope: _default 14 | collection: _default 15 | dcp: 16 | group: 17 | name: groupName 18 | kafka: 19 | collectionTopicMapping: 20 | _default: topicname 21 | brokers: 22 | - localhost:9092 23 | # SSL configurations 24 | # 25 | # secureConnection: true 26 | # Config support env variable "$HOME/example/..." 27 | # rootCAPath: "example/stretch-kafka/rootCA.pem" 28 | # interCAPath: "example/stretch-kafka/interCA.pem" 29 | # scramUsername: "username" 30 | # scramPassword: "password" 31 | rejectionLog: 32 | topic: "rejection-topic" 33 | includeValue: true -------------------------------------------------------------------------------- /example/simple-rejection-log-sink-response-handler/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.20 4 | 5 | replace github.com/Trendyol/go-dcp-kafka => ./../.. 6 | 7 | require github.com/Trendyol/go-dcp-kafka v0.0.0 8 | 9 | require ( 10 | github.com/Trendyol/go-dcp v1.2.6 // indirect 11 | github.com/andybalholm/brotli v1.1.1 // indirect 12 | github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect 13 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect 14 | github.com/beorn7/perks v1.0.1 // indirect 15 | github.com/bytedance/sonic v1.12.8 // indirect 16 | github.com/bytedance/sonic/loader v0.2.2 // indirect 17 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 18 | github.com/cloudwego/base64x v0.1.5 // indirect 19 | github.com/couchbase/gocbcore/v10 v10.5.2 // indirect 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 22 | github.com/go-logr/logr v1.4.1 // indirect 23 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 24 | github.com/go-openapi/jsonreference v0.20.2 // indirect 25 | github.com/go-openapi/swag v0.22.3 // indirect 26 | github.com/gofiber/fiber/v2 v2.52.5 // indirect 27 | github.com/gogo/protobuf v1.3.2 // indirect 28 | github.com/golang/protobuf v1.5.4 // indirect 29 | github.com/golang/snappy v0.0.4 // indirect 30 | github.com/google/gnostic-models v0.6.8 // indirect 31 | github.com/google/gofuzz v1.2.0 // indirect 32 | github.com/google/uuid v1.6.0 // indirect 33 | github.com/josharian/intern v1.0.0 // indirect 34 | github.com/json-iterator/go v1.1.12 // indirect 35 | github.com/klauspost/compress v1.17.11 // indirect 36 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 37 | github.com/mailru/easyjson v0.7.7 // indirect 38 | github.com/mattn/go-colorable v0.1.13 // indirect 39 | github.com/mattn/go-isatty v0.0.20 // indirect 40 | github.com/mattn/go-runewidth v0.0.15 // indirect 41 | github.com/mhmtszr/concurrent-swiss-map v1.0.8 // indirect 42 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 43 | github.com/modern-go/reflect2 v1.0.2 // indirect 44 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 45 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 46 | github.com/prometheus/client_golang v1.20.5 // indirect 47 | github.com/prometheus/client_model v0.6.1 // indirect 48 | github.com/prometheus/common v0.58.0 // indirect 49 | github.com/prometheus/procfs v0.15.1 // indirect 50 | github.com/rivo/uniseg v0.4.7 // indirect 51 | github.com/segmentio/kafka-go v0.4.47 // indirect 52 | github.com/sirupsen/logrus v1.9.3 // indirect 53 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 54 | github.com/valyala/bytebufferpool v1.0.0 // indirect 55 | github.com/valyala/fasthttp v1.57.0 // indirect 56 | github.com/valyala/tcplisten v1.0.0 // indirect 57 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 58 | github.com/xdg-go/scram v1.1.2 // indirect 59 | github.com/xdg-go/stringprep v1.0.4 // indirect 60 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect 61 | golang.org/x/net v0.33.0 // indirect 62 | golang.org/x/oauth2 v0.22.0 // indirect 63 | golang.org/x/sync v0.10.0 // indirect 64 | golang.org/x/sys v0.28.0 // indirect 65 | golang.org/x/term v0.27.0 // indirect 66 | golang.org/x/text v0.21.0 // indirect 67 | golang.org/x/time v0.3.0 // indirect 68 | google.golang.org/protobuf v1.36.4 // indirect 69 | gopkg.in/inf.v0 v0.9.1 // indirect 70 | gopkg.in/yaml.v2 v2.4.0 // indirect 71 | gopkg.in/yaml.v3 v3.0.1 // indirect 72 | k8s.io/api v0.29.4 // indirect 73 | k8s.io/apimachinery v0.29.4 // indirect 74 | k8s.io/client-go v0.29.4 // indirect 75 | k8s.io/klog/v2 v2.110.1 // indirect 76 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect 77 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 78 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 79 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 80 | sigs.k8s.io/yaml v1.3.0 // indirect 81 | ) 82 | -------------------------------------------------------------------------------- /example/simple-rejection-log-sink-response-handler/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | dcpkafka "github.com/Trendyol/go-dcp-kafka" 5 | "github.com/Trendyol/go-dcp-kafka/couchbase" 6 | "github.com/Trendyol/go-dcp-kafka/kafka" 7 | "github.com/Trendyol/go-dcp-kafka/kafka/message" 8 | ) 9 | 10 | func mapper(event couchbase.Event) []message.KafkaMessage { 11 | // return nil if you wish to discard the event 12 | return []message.KafkaMessage{ 13 | { 14 | Headers: nil, 15 | Key: event.Key, 16 | Value: event.Value, 17 | }, 18 | } 19 | } 20 | 21 | func main() { 22 | c, err := dcpkafka.NewConnectorBuilder("config.yml"). 23 | SetMapper(mapper). 24 | SetSinkResponseHandler(kafka.NewRejectionLogSinkResponseHandler()). 25 | Build() 26 | if err != nil { 27 | panic(err) 28 | } 29 | 30 | defer c.Close() 31 | c.Start() 32 | } 33 | -------------------------------------------------------------------------------- /example/simple/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20-alpine as builder 2 | 3 | WORKDIR /project 4 | 5 | COPY go.mod go.sum ./ 6 | COPY main.go ./ 7 | COPY config.yml ./config.yml 8 | 9 | RUN go mod download 10 | RUN CGO_ENABLED=0 go build -a -o example main.go 11 | 12 | FROM alpine:3.17.0 13 | 14 | WORKDIR /app 15 | 16 | RUN apk --no-cache add ca-certificates 17 | 18 | USER nobody 19 | COPY --from=builder --chown=nobody:nobody /project/example . 20 | COPY --from=builder --chown=nobody:nobody /project/config.yml ./config.yml 21 | 22 | ENTRYPOINT ["./example"] -------------------------------------------------------------------------------- /example/simple/config.yml: -------------------------------------------------------------------------------- 1 | hosts: 2 | - localhost:8091 3 | username: user 4 | password: password 5 | bucketName: dcp-test 6 | scopeName: _default 7 | collectionNames: 8 | - _default 9 | metadata: 10 | type: couchbase 11 | config: 12 | bucket: checkpoint-bucket-name 13 | scope: _default 14 | collection: _default 15 | dcp: 16 | group: 17 | name: groupName 18 | kafka: 19 | collectionTopicMapping: 20 | _default: topicname 21 | brokers: 22 | - localhost:9092 23 | # SSL configurations 24 | # 25 | # secureConnection: true 26 | # Config support env variable "$HOME/example/..." 27 | # rootCAPath: "example/stretch-kafka/rootCA.pem" 28 | # interCAPath: "example/stretch-kafka/interCA.pem" 29 | # scramUsername: "username" 30 | # scramPassword: "password" -------------------------------------------------------------------------------- /example/simple/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.20 4 | 5 | replace github.com/Trendyol/go-dcp-kafka => ./../.. 6 | 7 | require github.com/Trendyol/go-dcp-kafka v0.0.0 8 | 9 | require ( 10 | github.com/Trendyol/go-dcp v1.2.6 // indirect 11 | github.com/andybalholm/brotli v1.1.1 // indirect 12 | github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect 13 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect 14 | github.com/beorn7/perks v1.0.1 // indirect 15 | github.com/bytedance/sonic v1.12.8 // indirect 16 | github.com/bytedance/sonic/loader v0.2.2 // indirect 17 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 18 | github.com/cloudwego/base64x v0.1.5 // indirect 19 | github.com/couchbase/gocbcore/v10 v10.5.2 // indirect 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 22 | github.com/go-logr/logr v1.4.1 // indirect 23 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 24 | github.com/go-openapi/jsonreference v0.20.2 // indirect 25 | github.com/go-openapi/swag v0.22.3 // indirect 26 | github.com/gofiber/fiber/v2 v2.52.5 // indirect 27 | github.com/gogo/protobuf v1.3.2 // indirect 28 | github.com/golang/protobuf v1.5.4 // indirect 29 | github.com/golang/snappy v0.0.4 // indirect 30 | github.com/google/gnostic-models v0.6.8 // indirect 31 | github.com/google/gofuzz v1.2.0 // indirect 32 | github.com/google/uuid v1.6.0 // indirect 33 | github.com/josharian/intern v1.0.0 // indirect 34 | github.com/json-iterator/go v1.1.12 // indirect 35 | github.com/klauspost/compress v1.17.11 // indirect 36 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 37 | github.com/mailru/easyjson v0.7.7 // indirect 38 | github.com/mattn/go-colorable v0.1.13 // indirect 39 | github.com/mattn/go-isatty v0.0.20 // indirect 40 | github.com/mattn/go-runewidth v0.0.15 // indirect 41 | github.com/mhmtszr/concurrent-swiss-map v1.0.8 // indirect 42 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 43 | github.com/modern-go/reflect2 v1.0.2 // indirect 44 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 45 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 46 | github.com/prometheus/client_golang v1.20.5 // indirect 47 | github.com/prometheus/client_model v0.6.1 // indirect 48 | github.com/prometheus/common v0.58.0 // indirect 49 | github.com/prometheus/procfs v0.15.1 // indirect 50 | github.com/rivo/uniseg v0.4.7 // indirect 51 | github.com/segmentio/kafka-go v0.4.47 // indirect 52 | github.com/sirupsen/logrus v1.9.3 // indirect 53 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 54 | github.com/valyala/bytebufferpool v1.0.0 // indirect 55 | github.com/valyala/fasthttp v1.57.0 // indirect 56 | github.com/valyala/tcplisten v1.0.0 // indirect 57 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 58 | github.com/xdg-go/scram v1.1.2 // indirect 59 | github.com/xdg-go/stringprep v1.0.4 // indirect 60 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect 61 | golang.org/x/net v0.33.0 // indirect 62 | golang.org/x/oauth2 v0.22.0 // indirect 63 | golang.org/x/sync v0.10.0 // indirect 64 | golang.org/x/sys v0.28.0 // indirect 65 | golang.org/x/term v0.27.0 // indirect 66 | golang.org/x/text v0.21.0 // indirect 67 | golang.org/x/time v0.3.0 // indirect 68 | google.golang.org/protobuf v1.36.4 // indirect 69 | gopkg.in/inf.v0 v0.9.1 // indirect 70 | gopkg.in/yaml.v2 v2.4.0 // indirect 71 | gopkg.in/yaml.v3 v3.0.1 // indirect 72 | k8s.io/api v0.29.4 // indirect 73 | k8s.io/apimachinery v0.29.4 // indirect 74 | k8s.io/client-go v0.29.4 // indirect 75 | k8s.io/klog/v2 v2.110.1 // indirect 76 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect 77 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 78 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 79 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 80 | sigs.k8s.io/yaml v1.3.0 // indirect 81 | ) 82 | -------------------------------------------------------------------------------- /example/simple/go.sum: -------------------------------------------------------------------------------- 1 | github.com/Trendyol/go-dcp v1.2.6 h1:4DlFXHYlN7b66jEFzRfP34s9smwBYjs4IsQCD+y/5pM= 2 | github.com/Trendyol/go-dcp v1.2.6/go.mod h1:t2vNISMXxbYPA6jNHBUNqdk0jzGuVMQKkZnCrSiQ1ik= 3 | github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= 4 | github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= 5 | github.com/ansrivas/fiberprometheus/v2 v2.7.0 h1:09XiSzG0J7aZp7RviklngdWdDbSybKjhuWAstp003Gg= 6 | github.com/ansrivas/fiberprometheus/v2 v2.7.0/go.mod h1:hSJdO65lfnWW70Qn9uGdXXsUUSkckbhuw5r/KesygpU= 7 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= 8 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= 9 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 10 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 11 | github.com/bytedance/sonic v1.12.8 h1:4xYRVRlXIgvSZ4e8iVTlMF5szgpXd4AfvuWgA8I8lgs= 12 | github.com/bytedance/sonic v1.12.8/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8= 13 | github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= 14 | github.com/bytedance/sonic/loader v0.2.2 h1:jxAJuN9fOot/cyz5Q6dUuMJF5OqQ6+5GfA8FjjQ0R4o= 15 | github.com/bytedance/sonic/loader v0.2.2/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= 16 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 17 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 18 | github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= 19 | github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= 20 | github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= 21 | github.com/couchbase/gocbcore/v10 v10.5.2 h1:DHK042E1RfhPBR3b14CITl5XHRsLjH3hpERuwUc5UIg= 22 | github.com/couchbase/gocbcore/v10 v10.5.2/go.mod h1:rulbgUK70EuyRUiLQ0LhQAfSI/Rl+jWws8tTbHzvB6M= 23 | github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4= 24 | github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY= 25 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 26 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 27 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 28 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 29 | github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= 30 | github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= 31 | github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= 32 | github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 33 | github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= 34 | github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 35 | github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= 36 | github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= 37 | github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= 38 | github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= 39 | github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= 40 | github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= 41 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= 42 | github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= 43 | github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= 44 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 45 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 46 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= 47 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 48 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 49 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 50 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 51 | github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= 52 | github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= 53 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 54 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 55 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 56 | github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= 57 | github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 58 | github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= 59 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 60 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 61 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 62 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 63 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 64 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 65 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 66 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 67 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 68 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 69 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 70 | github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= 71 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 72 | github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= 73 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 74 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 75 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 76 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 77 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 78 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 79 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 80 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 81 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 82 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 83 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 84 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 85 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 86 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 87 | github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= 88 | github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 89 | github.com/mhmtszr/concurrent-swiss-map v1.0.8 h1:GDSxgVrXsPFsraUJaPMm7ptYulj8qnWPgnwXcWbJNxo= 90 | github.com/mhmtszr/concurrent-swiss-map v1.0.8/go.mod h1:F6QETL48Qn7jEJ3ZPt7EqRZjAAZu7lRQeQGIzXuUIDc= 91 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 92 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 93 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 94 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 95 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 96 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 97 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 98 | github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= 99 | github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= 100 | github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= 101 | github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= 102 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 103 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 104 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 105 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 106 | github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= 107 | github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= 108 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 109 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 110 | github.com/prometheus/common v0.58.0 h1:N+N8vY4/23r6iYfD3UQZUoJPnUYAo7v6LG5XZxjZTXo= 111 | github.com/prometheus/common v0.58.0/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= 112 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 113 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 114 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 115 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 116 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 117 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 118 | github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0= 119 | github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= 120 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 121 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 122 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 123 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 124 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 125 | github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= 126 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 127 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 128 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 129 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 130 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 131 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 132 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 133 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 134 | github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU= 135 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= 136 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= 137 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 138 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 139 | github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg= 140 | github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE= 141 | github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= 142 | github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= 143 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 144 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 145 | github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= 146 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 147 | github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= 148 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 149 | github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= 150 | github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= 151 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 152 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 153 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 154 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= 155 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= 156 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 157 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 158 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 159 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 160 | golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= 161 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 162 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 163 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 164 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 165 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 166 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 167 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 168 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 169 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 170 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 171 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 172 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 173 | golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= 174 | golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= 175 | golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= 176 | golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= 177 | golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= 178 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 179 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 180 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 181 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 182 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 183 | golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= 184 | golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 185 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 186 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 187 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 188 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 189 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 190 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 191 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 192 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 193 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 194 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 195 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 196 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 197 | golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 198 | golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= 199 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 200 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 201 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 202 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 203 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 204 | golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= 205 | golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= 206 | golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= 207 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 208 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 209 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 210 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 211 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 212 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 213 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 214 | golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= 215 | golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= 216 | golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= 217 | golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 218 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 219 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 220 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 221 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 222 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 223 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 224 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= 225 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 226 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 227 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 228 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 229 | google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= 230 | google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 231 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 232 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 233 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 234 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 235 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 236 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 237 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 238 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 239 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 240 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 241 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 242 | k8s.io/api v0.29.4 h1:WEnF/XdxuCxdG3ayHNRR8yH3cI1B/llkWBma6bq4R3w= 243 | k8s.io/api v0.29.4/go.mod h1:DetSv0t4FBTcEpfA84NJV3g9a7+rSzlUHk5ADAYHUv0= 244 | k8s.io/apimachinery v0.29.4 h1:RaFdJiDmuKs/8cm1M6Dh1Kvyh59YQFDcFuFTSmXes6Q= 245 | k8s.io/apimachinery v0.29.4/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= 246 | k8s.io/client-go v0.29.4 h1:79ytIedxVfyXV8rpH3jCBW0u+un0fxHDwX5F9K8dPR8= 247 | k8s.io/client-go v0.29.4/go.mod h1:kC1thZQ4zQWYwldsfI088BbK6RkxK+aF5ebV8y9Q4tk= 248 | k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= 249 | k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= 250 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= 251 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= 252 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= 253 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 254 | nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= 255 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= 256 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= 257 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= 258 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= 259 | sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= 260 | sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= 261 | -------------------------------------------------------------------------------- /example/simple/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | dcpkafka "github.com/Trendyol/go-dcp-kafka" 6 | "github.com/Trendyol/go-dcp-kafka/couchbase" 7 | "github.com/Trendyol/go-dcp-kafka/kafka" 8 | "github.com/Trendyol/go-dcp-kafka/kafka/message" 9 | ) 10 | 11 | func mapper(event couchbase.Event) []message.KafkaMessage { 12 | // return nil if you wish to discard the event 13 | return []message.KafkaMessage{ 14 | { 15 | Headers: nil, 16 | Key: event.Key, 17 | Value: event.Value, 18 | }, 19 | } 20 | } 21 | 22 | type sinkResponseHandler struct { 23 | } 24 | 25 | func (s *sinkResponseHandler) OnInit(ctx *kafka.SinkResponseHandlerInitContext) {} 26 | 27 | func (s *sinkResponseHandler) OnSuccess(ctx *kafka.SinkResponseHandlerContext) { 28 | fmt.Printf("OnSuccess Key: %v, Len: %v\n", string(ctx.Message.Key), len(ctx.Message.Value)) 29 | } 30 | 31 | func (s *sinkResponseHandler) OnError(ctx *kafka.SinkResponseHandlerContext) { 32 | fmt.Printf("OnError Key: %v, Len: %v, Err: %v\n", string(ctx.Message.Key), len(ctx.Message.Value), ctx.Err) 33 | } 34 | 35 | func main() { 36 | c, err := dcpkafka.NewConnectorBuilder("config.yml"). 37 | SetMapper(mapper). 38 | SetSinkResponseHandler(&sinkResponseHandler{}). 39 | Build() 40 | if err != nil { 41 | panic(err) 42 | } 43 | 44 | defer c.Close() 45 | c.Start() 46 | } 47 | -------------------------------------------------------------------------------- /example/struct-config/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20-alpine as builder 2 | 3 | WORKDIR /project 4 | 5 | COPY go.mod go.sum ./ 6 | COPY main.go ./ 7 | COPY config.yml ./config.yml 8 | 9 | RUN go mod download 10 | RUN CGO_ENABLED=0 go build -a -o example main.go 11 | 12 | FROM alpine:3.17.0 13 | 14 | WORKDIR /app 15 | 16 | RUN apk --no-cache add ca-certificates 17 | 18 | USER nobody 19 | COPY --from=builder --chown=nobody:nobody /project/example . 20 | COPY --from=builder --chown=nobody:nobody /project/config.yml ./config.yml 21 | 22 | ENTRYPOINT ["./example"] -------------------------------------------------------------------------------- /example/struct-config/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.20 4 | 5 | replace github.com/Trendyol/go-dcp-kafka => ./../.. 6 | 7 | require ( 8 | github.com/Trendyol/go-dcp v1.2.6 9 | github.com/Trendyol/go-dcp-kafka v0.0.0 10 | ) 11 | 12 | require ( 13 | github.com/andybalholm/brotli v1.1.1 // indirect 14 | github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect 15 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect 16 | github.com/beorn7/perks v1.0.1 // indirect 17 | github.com/bytedance/sonic v1.12.8 // indirect 18 | github.com/bytedance/sonic/loader v0.2.2 // indirect 19 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 20 | github.com/cloudwego/base64x v0.1.5 // indirect 21 | github.com/couchbase/gocbcore/v10 v10.5.2 // indirect 22 | github.com/davecgh/go-spew v1.1.1 // indirect 23 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 24 | github.com/go-logr/logr v1.4.1 // indirect 25 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 26 | github.com/go-openapi/jsonreference v0.20.2 // indirect 27 | github.com/go-openapi/swag v0.22.3 // indirect 28 | github.com/gofiber/fiber/v2 v2.52.5 // indirect 29 | github.com/gogo/protobuf v1.3.2 // indirect 30 | github.com/golang/protobuf v1.5.4 // indirect 31 | github.com/golang/snappy v0.0.4 // indirect 32 | github.com/google/gnostic-models v0.6.8 // indirect 33 | github.com/google/gofuzz v1.2.0 // indirect 34 | github.com/google/uuid v1.6.0 // indirect 35 | github.com/josharian/intern v1.0.0 // indirect 36 | github.com/json-iterator/go v1.1.12 // indirect 37 | github.com/klauspost/compress v1.17.11 // indirect 38 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 39 | github.com/mailru/easyjson v0.7.7 // indirect 40 | github.com/mattn/go-colorable v0.1.13 // indirect 41 | github.com/mattn/go-isatty v0.0.20 // indirect 42 | github.com/mattn/go-runewidth v0.0.15 // indirect 43 | github.com/mhmtszr/concurrent-swiss-map v1.0.8 // indirect 44 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 45 | github.com/modern-go/reflect2 v1.0.2 // indirect 46 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 47 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 48 | github.com/prometheus/client_golang v1.20.5 // indirect 49 | github.com/prometheus/client_model v0.6.1 // indirect 50 | github.com/prometheus/common v0.58.0 // indirect 51 | github.com/prometheus/procfs v0.15.1 // indirect 52 | github.com/rivo/uniseg v0.4.7 // indirect 53 | github.com/segmentio/kafka-go v0.4.47 // indirect 54 | github.com/sirupsen/logrus v1.9.3 // indirect 55 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 56 | github.com/valyala/bytebufferpool v1.0.0 // indirect 57 | github.com/valyala/fasthttp v1.57.0 // indirect 58 | github.com/valyala/tcplisten v1.0.0 // indirect 59 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 60 | github.com/xdg-go/scram v1.1.2 // indirect 61 | github.com/xdg-go/stringprep v1.0.4 // indirect 62 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect 63 | golang.org/x/net v0.33.0 // indirect 64 | golang.org/x/oauth2 v0.22.0 // indirect 65 | golang.org/x/sync v0.10.0 // indirect 66 | golang.org/x/sys v0.28.0 // indirect 67 | golang.org/x/term v0.27.0 // indirect 68 | golang.org/x/text v0.21.0 // indirect 69 | golang.org/x/time v0.3.0 // indirect 70 | google.golang.org/protobuf v1.36.4 // indirect 71 | gopkg.in/inf.v0 v0.9.1 // indirect 72 | gopkg.in/yaml.v2 v2.4.0 // indirect 73 | gopkg.in/yaml.v3 v3.0.1 // indirect 74 | k8s.io/api v0.29.4 // indirect 75 | k8s.io/apimachinery v0.29.4 // indirect 76 | k8s.io/client-go v0.29.4 // indirect 77 | k8s.io/klog/v2 v2.110.1 // indirect 78 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect 79 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 80 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 81 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 82 | sigs.k8s.io/yaml v1.3.0 // indirect 83 | ) 84 | -------------------------------------------------------------------------------- /example/struct-config/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/Trendyol/go-dcp-kafka" 5 | "github.com/Trendyol/go-dcp-kafka/config" 6 | "github.com/Trendyol/go-dcp-kafka/couchbase" 7 | "github.com/Trendyol/go-dcp-kafka/kafka/message" 8 | dcpConfig "github.com/Trendyol/go-dcp/config" 9 | "time" 10 | ) 11 | 12 | func mapper(event couchbase.Event) []message.KafkaMessage { 13 | // return nil if you wish to discard the event 14 | return []message.KafkaMessage{ 15 | { 16 | Headers: nil, 17 | Key: event.Key, 18 | Value: event.Value, 19 | }, 20 | } 21 | } 22 | 23 | func main() { 24 | c, err := dcpkafka.NewConnectorBuilder(&config.Connector{ 25 | Dcp: dcpConfig.Dcp{ 26 | Hosts: []string{"localhost:8091"}, 27 | Username: "user", 28 | Password: "password", 29 | BucketName: "dcp-test", 30 | Dcp: dcpConfig.ExternalDcp{ 31 | Group: dcpConfig.DCPGroup{ 32 | Name: "groupName", 33 | Membership: dcpConfig.DCPGroupMembership{ 34 | RebalanceDelay: 3 * time.Second, 35 | }, 36 | }, 37 | }, 38 | Metadata: dcpConfig.Metadata{ 39 | Config: map[string]string{ 40 | "bucket": "checkpoint-bucket-name", 41 | "scope": "_default", 42 | "collection": "_default", 43 | }, 44 | Type: "couchbase", 45 | }, 46 | Debug: true}, 47 | Kafka: config.Kafka{ 48 | CollectionTopicMapping: map[string]string{"_default": "topic"}, 49 | Brokers: []string{"localhost:9092"}, 50 | }, 51 | }).SetMapper(mapper). 52 | Build() 53 | if err != nil { 54 | panic(err) 55 | } 56 | 57 | defer c.Close() 58 | c.Start() 59 | } 60 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Trendyol/go-dcp-kafka 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/Trendyol/go-dcp v1.2.6 7 | github.com/json-iterator/go v1.1.12 8 | github.com/prometheus/client_golang v1.20.5 9 | github.com/segmentio/kafka-go v0.4.47 10 | github.com/sirupsen/logrus v1.9.3 11 | gopkg.in/yaml.v3 v3.0.1 12 | ) 13 | 14 | require ( 15 | github.com/andybalholm/brotli v1.1.1 // indirect 16 | github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect 17 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect 18 | github.com/beorn7/perks v1.0.1 // indirect 19 | github.com/bytedance/sonic v1.12.8 // indirect 20 | github.com/bytedance/sonic/loader v0.2.2 // indirect 21 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 22 | github.com/cloudwego/base64x v0.1.5 // indirect 23 | github.com/couchbase/gocbcore/v10 v10.5.2 // indirect 24 | github.com/davecgh/go-spew v1.1.1 // indirect 25 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 26 | github.com/go-logr/logr v1.4.1 // indirect 27 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 28 | github.com/go-openapi/jsonreference v0.20.2 // indirect 29 | github.com/go-openapi/swag v0.22.3 // indirect 30 | github.com/gofiber/fiber/v2 v2.52.5 // indirect 31 | github.com/gogo/protobuf v1.3.2 // indirect 32 | github.com/golang/protobuf v1.5.4 // indirect 33 | github.com/golang/snappy v0.0.4 // indirect 34 | github.com/google/gnostic-models v0.6.8 // indirect 35 | github.com/google/gofuzz v1.2.0 // indirect 36 | github.com/google/uuid v1.6.0 // indirect 37 | github.com/josharian/intern v1.0.0 // indirect 38 | github.com/klauspost/compress v1.17.11 // indirect 39 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 40 | github.com/mailru/easyjson v0.7.7 // indirect 41 | github.com/mattn/go-colorable v0.1.13 // indirect 42 | github.com/mattn/go-isatty v0.0.20 // indirect 43 | github.com/mattn/go-runewidth v0.0.15 // indirect 44 | github.com/mhmtszr/concurrent-swiss-map v1.0.8 // indirect 45 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 46 | github.com/modern-go/reflect2 v1.0.2 // indirect 47 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 48 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 49 | github.com/prometheus/client_model v0.6.1 // indirect 50 | github.com/prometheus/common v0.58.0 // indirect 51 | github.com/prometheus/procfs v0.15.1 // indirect 52 | github.com/rivo/uniseg v0.4.7 // indirect 53 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 54 | github.com/valyala/bytebufferpool v1.0.0 // indirect 55 | github.com/valyala/fasthttp v1.57.0 // indirect 56 | github.com/valyala/tcplisten v1.0.0 // indirect 57 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 58 | github.com/xdg-go/scram v1.1.2 // indirect 59 | github.com/xdg-go/stringprep v1.0.4 // indirect 60 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect 61 | golang.org/x/net v0.33.0 // indirect 62 | golang.org/x/oauth2 v0.22.0 // indirect 63 | golang.org/x/sync v0.10.0 // indirect 64 | golang.org/x/sys v0.28.0 // indirect 65 | golang.org/x/term v0.27.0 // indirect 66 | golang.org/x/text v0.21.0 // indirect 67 | golang.org/x/time v0.3.0 // indirect 68 | google.golang.org/protobuf v1.36.4 // indirect 69 | gopkg.in/inf.v0 v0.9.1 // indirect 70 | gopkg.in/yaml.v2 v2.4.0 // indirect 71 | k8s.io/api v0.29.4 // indirect 72 | k8s.io/apimachinery v0.29.4 // indirect 73 | k8s.io/client-go v0.29.4 // indirect 74 | k8s.io/klog/v2 v2.110.1 // indirect 75 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect 76 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 77 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 78 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 79 | sigs.k8s.io/yaml v1.3.0 // indirect 80 | ) 81 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Trendyol/go-dcp v1.2.6 h1:4DlFXHYlN7b66jEFzRfP34s9smwBYjs4IsQCD+y/5pM= 2 | github.com/Trendyol/go-dcp v1.2.6/go.mod h1:t2vNISMXxbYPA6jNHBUNqdk0jzGuVMQKkZnCrSiQ1ik= 3 | github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= 4 | github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= 5 | github.com/ansrivas/fiberprometheus/v2 v2.7.0 h1:09XiSzG0J7aZp7RviklngdWdDbSybKjhuWAstp003Gg= 6 | github.com/ansrivas/fiberprometheus/v2 v2.7.0/go.mod h1:hSJdO65lfnWW70Qn9uGdXXsUUSkckbhuw5r/KesygpU= 7 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= 8 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= 9 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 10 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 11 | github.com/bytedance/sonic v1.12.8 h1:4xYRVRlXIgvSZ4e8iVTlMF5szgpXd4AfvuWgA8I8lgs= 12 | github.com/bytedance/sonic v1.12.8/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8= 13 | github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= 14 | github.com/bytedance/sonic/loader v0.2.2 h1:jxAJuN9fOot/cyz5Q6dUuMJF5OqQ6+5GfA8FjjQ0R4o= 15 | github.com/bytedance/sonic/loader v0.2.2/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= 16 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 17 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 18 | github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= 19 | github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= 20 | github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= 21 | github.com/couchbase/gocbcore/v10 v10.5.2 h1:DHK042E1RfhPBR3b14CITl5XHRsLjH3hpERuwUc5UIg= 22 | github.com/couchbase/gocbcore/v10 v10.5.2/go.mod h1:rulbgUK70EuyRUiLQ0LhQAfSI/Rl+jWws8tTbHzvB6M= 23 | github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4= 24 | github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY= 25 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 26 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 27 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 28 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 29 | github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= 30 | github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= 31 | github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= 32 | github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 33 | github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= 34 | github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 35 | github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= 36 | github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= 37 | github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= 38 | github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= 39 | github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= 40 | github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= 41 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= 42 | github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= 43 | github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= 44 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 45 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 46 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= 47 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 48 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 49 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 50 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 51 | github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= 52 | github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= 53 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 54 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 55 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 56 | github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= 57 | github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 58 | github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= 59 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 60 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 61 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 62 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 63 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 64 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 65 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 66 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 67 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 68 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 69 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 70 | github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= 71 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 72 | github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= 73 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 74 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 75 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 76 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 77 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 78 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 79 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 80 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 81 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 82 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 83 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 84 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 85 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 86 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 87 | github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= 88 | github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 89 | github.com/mhmtszr/concurrent-swiss-map v1.0.8 h1:GDSxgVrXsPFsraUJaPMm7ptYulj8qnWPgnwXcWbJNxo= 90 | github.com/mhmtszr/concurrent-swiss-map v1.0.8/go.mod h1:F6QETL48Qn7jEJ3ZPt7EqRZjAAZu7lRQeQGIzXuUIDc= 91 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 92 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 93 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 94 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 95 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 96 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 97 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 98 | github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= 99 | github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= 100 | github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= 101 | github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= 102 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 103 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 104 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 105 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 106 | github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= 107 | github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= 108 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 109 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 110 | github.com/prometheus/common v0.58.0 h1:N+N8vY4/23r6iYfD3UQZUoJPnUYAo7v6LG5XZxjZTXo= 111 | github.com/prometheus/common v0.58.0/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= 112 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 113 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 114 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 115 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 116 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 117 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 118 | github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0= 119 | github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= 120 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 121 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 122 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 123 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 124 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 125 | github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= 126 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 127 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 128 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 129 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 130 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 131 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 132 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 133 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 134 | github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU= 135 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= 136 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= 137 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 138 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 139 | github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg= 140 | github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE= 141 | github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= 142 | github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= 143 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 144 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 145 | github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= 146 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 147 | github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= 148 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 149 | github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= 150 | github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= 151 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 152 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 153 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 154 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= 155 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= 156 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 157 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 158 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 159 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 160 | golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= 161 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 162 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 163 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 164 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 165 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 166 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 167 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 168 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 169 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 170 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 171 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 172 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 173 | golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= 174 | golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= 175 | golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= 176 | golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= 177 | golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= 178 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 179 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 180 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 181 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 182 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 183 | golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= 184 | golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 185 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 186 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 187 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 188 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 189 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 190 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 191 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 192 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 193 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 194 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 195 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 196 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 197 | golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 198 | golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= 199 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 200 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 201 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 202 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 203 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 204 | golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= 205 | golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= 206 | golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= 207 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 208 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 209 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 210 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 211 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 212 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 213 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 214 | golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= 215 | golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= 216 | golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= 217 | golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 218 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 219 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 220 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 221 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 222 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 223 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 224 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= 225 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 226 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 227 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 228 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 229 | google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= 230 | google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 231 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 232 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 233 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 234 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 235 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 236 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 237 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 238 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 239 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 240 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 241 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 242 | k8s.io/api v0.29.4 h1:WEnF/XdxuCxdG3ayHNRR8yH3cI1B/llkWBma6bq4R3w= 243 | k8s.io/api v0.29.4/go.mod h1:DetSv0t4FBTcEpfA84NJV3g9a7+rSzlUHk5ADAYHUv0= 244 | k8s.io/apimachinery v0.29.4 h1:RaFdJiDmuKs/8cm1M6Dh1Kvyh59YQFDcFuFTSmXes6Q= 245 | k8s.io/apimachinery v0.29.4/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= 246 | k8s.io/client-go v0.29.4 h1:79ytIedxVfyXV8rpH3jCBW0u+un0fxHDwX5F9K8dPR8= 247 | k8s.io/client-go v0.29.4/go.mod h1:kC1thZQ4zQWYwldsfI088BbK6RkxK+aF5ebV8y9Q4tk= 248 | k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= 249 | k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= 250 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= 251 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= 252 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= 253 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 254 | nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= 255 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= 256 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= 257 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= 258 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= 259 | sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= 260 | sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= 261 | -------------------------------------------------------------------------------- /kafka/client.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "errors" 8 | "fmt" 9 | "math" 10 | "net" 11 | "os" 12 | "time" 13 | 14 | "github.com/segmentio/kafka-go/sasl" 15 | 16 | "github.com/Trendyol/go-dcp-kafka/config" 17 | "github.com/Trendyol/go-dcp/logger" 18 | "github.com/segmentio/kafka-go" 19 | "github.com/segmentio/kafka-go/sasl/scram" 20 | ) 21 | 22 | type Client interface { 23 | GetEndOffsets(topic string, partitions []int) ([]kafka.PartitionOffsets, error) 24 | GetPartitions(topic string) ([]int, error) 25 | CreateCompactedTopic(topic string, partition int, replicationFactor int) error 26 | Producer(completionHandler func(messages []kafka.Message, err error)) *kafka.Writer 27 | Consumer(topic string, partition int, startOffset int64) *kafka.Reader 28 | CheckTopicIsCompacted(topic string) error 29 | CheckTopics(topics []string) error 30 | } 31 | 32 | type client struct { 33 | addr net.Addr 34 | kafkaClient *kafka.Client 35 | config *config.Connector 36 | transport *kafka.Transport 37 | dialer *kafka.Dialer 38 | } 39 | 40 | type tlsContent struct { 41 | config *tls.Config 42 | sasl sasl.Mechanism 43 | } 44 | 45 | func newTLSContent( 46 | scramUsername, 47 | scramPassword, 48 | rootCAPath, 49 | interCAPath, 50 | rootCA, 51 | interCA string, 52 | ) (*tlsContent, error) { 53 | mechanism, err := scram.Mechanism(scram.SHA512, scramUsername, scramPassword) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | certCount := 0 59 | caCertPool := x509.NewCertPool() 60 | 61 | if rootCAPath != "" { 62 | caCert, err := os.ReadFile(os.ExpandEnv(rootCAPath)) 63 | if err != nil { 64 | logger.Log.Error("an error occurred while reading ca.pem file! Error: %s", err.Error()) 65 | return nil, err 66 | } 67 | caCertPool.AppendCertsFromPEM(caCert) 68 | certCount++ 69 | } 70 | 71 | if interCAPath != "" { 72 | intCert, err := os.ReadFile(os.ExpandEnv(interCAPath)) 73 | if err != nil { 74 | logger.Log.Error("an error occurred while reading int.pem file! Error: %s", err.Error()) 75 | return nil, err 76 | } 77 | caCertPool.AppendCertsFromPEM(intCert) 78 | certCount++ 79 | } 80 | 81 | if rootCA != "" { 82 | caCertPool.AppendCertsFromPEM([]byte(rootCA)) 83 | certCount++ 84 | } 85 | 86 | if interCA != "" { 87 | caCertPool.AppendCertsFromPEM([]byte(interCA)) 88 | certCount++ 89 | } 90 | 91 | if certCount == 0 { 92 | err := errors.New("certPool is empty") 93 | logger.Log.Error("an error occurred while creating tls content! Error: %s", err.Error()) 94 | return nil, err 95 | } 96 | 97 | return &tlsContent{ 98 | config: &tls.Config{ 99 | RootCAs: caCertPool, 100 | MinVersion: tls.VersionTLS12, 101 | }, 102 | sasl: mechanism, 103 | }, nil 104 | } 105 | 106 | func (c *client) GetEndOffsets(topic string, partitions []int) ([]kafka.PartitionOffsets, error) { 107 | offsetRequests := make([]kafka.OffsetRequest, 0, len(partitions)) 108 | for _, partition := range partitions { 109 | offsetRequests = append(offsetRequests, kafka.LastOffsetOf(partition)) 110 | } 111 | 112 | request := &kafka.ListOffsetsRequest{ 113 | Addr: c.addr, 114 | Topics: map[string][]kafka.OffsetRequest{ 115 | topic: offsetRequests, 116 | }, 117 | IsolationLevel: kafka.ReadUncommitted, 118 | } 119 | 120 | response, err := c.kafkaClient.ListOffsets(context.Background(), request) 121 | if err != nil { 122 | return nil, err 123 | } 124 | 125 | return response.Topics[topic], nil 126 | } 127 | 128 | func (c *client) CheckTopicIsCompacted(topic string) error { 129 | response, err := c.kafkaClient.DescribeConfigs(context.Background(), &kafka.DescribeConfigsRequest{ 130 | Addr: c.addr, 131 | Resources: []kafka.DescribeConfigRequestResource{ 132 | { 133 | ResourceType: kafka.ResourceTypeTopic, 134 | ResourceName: topic, 135 | ConfigNames: []string{"cleanup.policy"}, 136 | }, 137 | }, 138 | IncludeSynonyms: false, 139 | IncludeDocumentation: false, 140 | }) 141 | if err != nil { 142 | return err 143 | } 144 | 145 | for _, resource := range response.Resources { 146 | if resource.Error != nil { 147 | return resource.Error 148 | } 149 | 150 | if resource.ResourceType == int8(kafka.ResourceTypeTopic) { 151 | for _, entity := range resource.ConfigEntries { 152 | if entity.ConfigName == "cleanup.policy" && entity.ConfigValue == "compact" { 153 | return nil 154 | } 155 | } 156 | } 157 | } 158 | 159 | return errors.New("topic is not compacted") 160 | } 161 | 162 | func (c *client) GetPartitions(topic string) ([]int, error) { 163 | response, err := c.kafkaClient.Metadata(context.Background(), &kafka.MetadataRequest{ 164 | Topics: []string{topic}, 165 | Addr: c.addr, 166 | }) 167 | if err != nil { 168 | return nil, err 169 | } 170 | 171 | var partitions []int 172 | 173 | for _, responseTopic := range response.Topics { 174 | if responseTopic.Name == topic { 175 | for _, partition := range responseTopic.Partitions { 176 | partitions = append(partitions, partition.ID) 177 | } 178 | } 179 | } 180 | 181 | return partitions, nil 182 | } 183 | 184 | func (c *client) CheckTopics(topics []string) error { 185 | response, err := c.kafkaClient.Metadata(context.Background(), &kafka.MetadataRequest{ 186 | Topics: topics, 187 | Addr: c.addr, 188 | }) 189 | if err != nil { 190 | return err 191 | } 192 | 193 | for _, responseTopic := range response.Topics { 194 | if responseTopic.Error != nil { 195 | return fmt.Errorf("topic=%s, err=%v", responseTopic.Name, responseTopic.Error) 196 | } 197 | } 198 | 199 | return nil 200 | } 201 | 202 | func (c *client) Producer(completionHandler func(messages []kafka.Message, err error)) *kafka.Writer { 203 | return &kafka.Writer{ 204 | Addr: kafka.TCP(c.config.Kafka.Brokers...), 205 | Balancer: c.config.Kafka.GetBalancer(), 206 | BatchSize: c.config.Kafka.ProducerBatchSize, 207 | BatchBytes: math.MaxInt, 208 | BatchTimeout: time.Nanosecond, 209 | MaxAttempts: c.config.Kafka.ProducerMaxAttempts, 210 | ReadTimeout: c.config.Kafka.ReadTimeout, 211 | WriteTimeout: c.config.Kafka.WriteTimeout, 212 | RequiredAcks: kafka.RequiredAcks(c.config.Kafka.RequiredAcks), 213 | Compression: kafka.Compression(c.config.Kafka.GetCompression()), 214 | Transport: c.transport, 215 | AllowAutoTopicCreation: c.config.Kafka.AllowAutoTopicCreation, 216 | Completion: completionHandler, 217 | } 218 | } 219 | 220 | func (c *client) Consumer(topic string, partition int, startOffset int64) *kafka.Reader { 221 | readerConfig := kafka.ReaderConfig{ 222 | Brokers: c.config.Kafka.Brokers, 223 | Topic: topic, 224 | Partition: partition, 225 | StartOffset: startOffset, 226 | } 227 | 228 | if c.dialer != nil { 229 | readerConfig.Dialer = c.dialer 230 | } 231 | 232 | return kafka.NewReader(readerConfig) 233 | } 234 | 235 | func (c *client) CreateCompactedTopic(topic string, partition int, replicationFactor int) error { 236 | response, err := c.kafkaClient.CreateTopics(context.Background(), &kafka.CreateTopicsRequest{ 237 | Topics: []kafka.TopicConfig{ 238 | { 239 | Topic: topic, 240 | NumPartitions: partition, 241 | ReplicationFactor: replicationFactor, 242 | ConfigEntries: []kafka.ConfigEntry{{ 243 | ConfigName: "cleanup.policy", 244 | ConfigValue: "compact", 245 | }, { 246 | ConfigName: "min.insync.replicas", 247 | ConfigValue: "2", 248 | }, { 249 | ConfigName: "retention.ms", 250 | ConfigValue: "86400000", 251 | }, { 252 | ConfigName: "max.message.bytes", 253 | ConfigValue: "1048588", 254 | }, { 255 | ConfigName: "retention.bytes", 256 | ConfigValue: "-1", 257 | }, { 258 | ConfigName: "segment.bytes", 259 | ConfigValue: "2097152", // 2MB 260 | }}, 261 | }, 262 | }, 263 | }) 264 | if err != nil { 265 | return err 266 | } 267 | 268 | for _, topicError := range response.Errors { 269 | if topicError != nil { 270 | return topicError 271 | } 272 | } 273 | 274 | return nil 275 | } 276 | 277 | func NewClient(config *config.Connector) Client { 278 | addr := kafka.TCP(config.Kafka.Brokers...) 279 | 280 | newClient := &client{ 281 | addr: addr, 282 | kafkaClient: &kafka.Client{ 283 | Addr: addr, 284 | }, 285 | config: config, 286 | } 287 | 288 | newClient.transport = &kafka.Transport{ 289 | MetadataTTL: config.Kafka.MetadataTTL, 290 | MetadataTopics: config.Kafka.MetadataTopics, 291 | ClientID: config.Kafka.ClientID, 292 | } 293 | 294 | if config.Kafka.SecureConnection { 295 | tlsContent, err := newTLSContent( 296 | config.Kafka.ScramUsername, 297 | config.Kafka.ScramPassword, 298 | config.Kafka.RootCAPath, 299 | config.Kafka.InterCAPath, 300 | config.Kafka.RootCA, 301 | config.Kafka.InterCA, 302 | ) 303 | if err != nil { 304 | logger.Log.Error("error while creating new tls content, err: %v", err) 305 | panic(err) 306 | } 307 | 308 | newClient.transport.TLS = tlsContent.config 309 | newClient.transport.SASL = tlsContent.sasl 310 | 311 | newClient.dialer = &kafka.Dialer{ 312 | Timeout: 10 * time.Second, 313 | DualStack: true, 314 | TLS: tlsContent.config, 315 | SASLMechanism: tlsContent.sasl, 316 | } 317 | } 318 | newClient.kafkaClient.Transport = newClient.transport 319 | return newClient 320 | } 321 | -------------------------------------------------------------------------------- /kafka/client_test.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "reflect" 7 | "testing" 8 | "time" 9 | 10 | "github.com/Trendyol/go-dcp-kafka/config" 11 | "github.com/Trendyol/go-dcp/logger" 12 | "github.com/segmentio/kafka-go" 13 | ) 14 | 15 | type MockLogger struct { 16 | debugMessages []string 17 | errorMessages []string 18 | } 19 | 20 | func TestProducer(t *testing.T) { 21 | mockLogger := &MockLogger{} 22 | logger.Log = mockLogger 23 | 24 | mockConfig := config.Connector{ 25 | Kafka: config.Kafka{ 26 | Brokers: []string{"localhost:9092"}, 27 | ProducerBatchSize: 10, 28 | ProducerMaxAttempts: 5, 29 | ReadTimeout: time.Second, 30 | WriteTimeout: time.Second, 31 | RequiredAcks: 1, 32 | AllowAutoTopicCreation: true, 33 | }, 34 | } 35 | 36 | client := &client{config: &mockConfig} 37 | 38 | completionHandler := func(messages []kafka.Message, err error) { 39 | if err != nil { 40 | logger.Log.Error("message produce failed: %v", err) 41 | return 42 | } 43 | for _, msg := range messages { 44 | logger.Log.Debug("message produce success, message: %s", msg.WriterData) 45 | } 46 | } 47 | 48 | writer := client.Producer(completionHandler) 49 | 50 | t.Run("Completion callback should log success message", func(t *testing.T) { 51 | messages := []kafka.Message{ 52 | {WriterData: "message 1"}, 53 | } 54 | 55 | writer.Completion(messages, nil) 56 | 57 | expected := "message produce success, message: message 1" 58 | if !contains(mockLogger.debugMessages, expected) { 59 | t.Errorf("Expected log message: %s, but got: %v", expected, mockLogger.debugMessages) 60 | } 61 | }) 62 | 63 | t.Run("Completion callback should log error message", func(t *testing.T) { 64 | writer.Completion(nil, errors.New("some error occurred")) 65 | 66 | expected := "message produce failed: some error occurred" 67 | if !contains(mockLogger.errorMessages, expected) { 68 | t.Errorf("Expected error log message: %s, but got: %v", expected, mockLogger.errorMessages) 69 | } 70 | }) 71 | } 72 | 73 | func contains(list []string, item string) bool { 74 | for _, v := range list { 75 | if reflect.DeepEqual(v, item) { 76 | return true 77 | } 78 | } 79 | return false 80 | } 81 | 82 | func (m *MockLogger) Trace(message string, args ...interface{}) { panic("implement me") } 83 | func (m *MockLogger) Info(message string, args ...interface{}) { panic("implement me") } 84 | func (m *MockLogger) Warn(message string, args ...interface{}) { panic("implement me") } 85 | func (m *MockLogger) Log(level string, message string, args ...interface{}) { panic("implement me") } 86 | func (m *MockLogger) Debug(format string, args ...interface{}) { 87 | m.debugMessages = append(m.debugMessages, fmt.Sprintf(format, args...)) 88 | } 89 | 90 | func (m *MockLogger) Error(format string, args ...interface{}) { 91 | m.errorMessages = append(m.errorMessages, fmt.Sprintf(format, args...)) 92 | } 93 | -------------------------------------------------------------------------------- /kafka/message/message.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | import "github.com/segmentio/kafka-go" 4 | 5 | type KafkaMessage struct { 6 | Topic string 7 | Headers []kafka.Header 8 | Key []byte 9 | Value []byte 10 | } 11 | -------------------------------------------------------------------------------- /kafka/metadata/kafka_metadata.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "strconv" 7 | "sync" 8 | 9 | "github.com/Trendyol/go-dcp/wrapper" 10 | 11 | "github.com/Trendyol/go-dcp/metadata" 12 | "github.com/Trendyol/go-dcp/models" 13 | 14 | gKafka "github.com/Trendyol/go-dcp-kafka/kafka" 15 | "github.com/Trendyol/go-dcp/logger" 16 | "github.com/json-iterator/go" 17 | "github.com/segmentio/kafka-go" 18 | ) 19 | 20 | type kafkaMetadata struct { 21 | kafkaClient gKafka.Client 22 | writer *kafka.Writer 23 | topic string 24 | } 25 | 26 | func (s *kafkaMetadata) Save(state map[uint16]*models.CheckpointDocument, dirtyOffsets map[uint16]bool, _ string) error { 27 | messages := make([]kafka.Message, 0, len(state)) 28 | for vbID, document := range state { 29 | if !dirtyOffsets[vbID] { 30 | continue 31 | } 32 | 33 | value, err := jsoniter.Marshal(document) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | messages = append(messages, kafka.Message{ 39 | Topic: s.topic, 40 | Key: []byte(strconv.Itoa(int(vbID))), 41 | Value: value, 42 | }) 43 | } 44 | 45 | return s.writer.WriteMessages(context.Background(), messages...) 46 | } 47 | 48 | func (s *kafkaMetadata) Load( //nolint:funlen 49 | vbIDs []uint16, 50 | bucketUUID string, 51 | ) (*wrapper.ConcurrentSwissMap[uint16, *models.CheckpointDocument], bool, error) { 52 | partitions, err := s.kafkaClient.GetPartitions(s.topic) 53 | if err != nil { 54 | return nil, false, err 55 | } 56 | 57 | endOffsets, err := s.kafkaClient.GetEndOffsets(s.topic, partitions) 58 | if err != nil { 59 | return nil, false, err 60 | } 61 | 62 | ch := make(chan kafka.Message) 63 | wg := &sync.WaitGroup{} 64 | wg.Add(len(endOffsets)) 65 | 66 | for _, endOffset := range endOffsets { 67 | consumer := s.kafkaClient.Consumer(s.topic, endOffset.Partition, endOffset.FirstOffset) 68 | 69 | if endOffset.FirstOffset == -1 && endOffset.LastOffset == 0 { 70 | wg.Done() 71 | continue 72 | } 73 | 74 | go func(consumer *kafka.Reader, lastOffset int64) { 75 | for { 76 | m, err := consumer.ReadMessage(context.Background()) 77 | if err != nil { 78 | break 79 | } 80 | 81 | ch <- m 82 | if m.Offset+1 >= lastOffset { 83 | break 84 | } 85 | } 86 | 87 | if err := consumer.Close(); err != nil { 88 | logger.Log.Error("failed to close consumer %v", err) 89 | } 90 | 91 | wg.Done() 92 | }(consumer, endOffset.LastOffset) 93 | } 94 | 95 | state := wrapper.CreateConcurrentSwissMap[uint16, *models.CheckpointDocument](1024) 96 | exist := false 97 | 98 | go func() { 99 | for m := range ch { 100 | var doc *models.CheckpointDocument 101 | 102 | err = jsoniter.Unmarshal(m.Value, &doc) 103 | if err != nil { 104 | doc = models.NewEmptyCheckpointDocument(bucketUUID) 105 | } else { 106 | exist = true 107 | } 108 | 109 | vbID, err := strconv.ParseUint(string(m.Key), 0, 16) 110 | if err == nil { 111 | state.Store(uint16(vbID), doc) 112 | } else { 113 | logger.Log.Error("cannot load checkpoint, vbID: %d %v", vbID, err) 114 | panic(err) 115 | } 116 | } 117 | }() 118 | 119 | wg.Wait() 120 | 121 | for _, vbID := range vbIDs { 122 | _, ok := state.Load(vbID) 123 | if !ok { 124 | state.Store(vbID, models.NewEmptyCheckpointDocument(bucketUUID)) 125 | } 126 | } 127 | 128 | return state, exist, nil 129 | } 130 | 131 | func (s *kafkaMetadata) Clear(_ []uint16) error { 132 | return nil 133 | } 134 | 135 | func NewKafkaMetadata( 136 | kafkaClient gKafka.Client, 137 | kafkaMetadataConfig map[string]string, 138 | ) metadata.Metadata { 139 | var topic string 140 | var partition int 141 | var replicationFactor int 142 | 143 | if _, ok := kafkaMetadataConfig["topic"]; ok { 144 | topic = kafkaMetadataConfig["topic"] 145 | } else { 146 | err := errors.New("topic is not defined") 147 | logger.Log.Error("error while kafka metadata, err: %v", err) 148 | panic(err) 149 | } 150 | 151 | if topic == "" { 152 | err := errors.New("topic is empty") 153 | logger.Log.Error("error while kafka metadata, err: %v", err) 154 | panic(err) 155 | } 156 | 157 | if _, ok := kafkaMetadataConfig["partition"]; ok { 158 | partition, _ = strconv.Atoi(kafkaMetadataConfig["partition"]) 159 | } else { 160 | partition = 25 161 | } 162 | 163 | if partition == 0 { 164 | err := errors.New("partition is 0") 165 | logger.Log.Error("error while kafka metadata, err: %v", err) 166 | panic(err) 167 | } 168 | 169 | if _, ok := kafkaMetadataConfig["replicationFactor"]; ok { 170 | replicationFactor, _ = strconv.Atoi(kafkaMetadataConfig["replicationFactor"]) 171 | } else { 172 | replicationFactor = 3 173 | } 174 | 175 | if replicationFactor == 0 { 176 | err := errors.New("replicationFactor is 0") 177 | logger.Log.Error("error while kafka metadata, err: %v", err) 178 | panic(err) 179 | } 180 | 181 | metadata := &kafkaMetadata{ 182 | kafkaClient: kafkaClient, 183 | writer: kafkaClient.Producer(nil), 184 | topic: topic, 185 | } 186 | 187 | err := kafkaClient.CreateCompactedTopic(topic, partition, replicationFactor) 188 | 189 | if err != nil && !errors.Is(err, kafka.TopicAlreadyExists) { 190 | logger.Log.Error("error while kafka metadata, err: %v", err) 191 | panic(err) 192 | } 193 | 194 | err = kafkaClient.CheckTopicIsCompacted(metadata.topic) 195 | if err != nil { 196 | logger.Log.Error("error while kafka metadata, err: %v", err) 197 | panic(err) 198 | } 199 | 200 | return metadata 201 | } 202 | -------------------------------------------------------------------------------- /kafka/producer/completion.go: -------------------------------------------------------------------------------- 1 | package producer 2 | 3 | import ( 4 | "github.com/Trendyol/go-dcp/logger" 5 | "github.com/segmentio/kafka-go" 6 | ) 7 | 8 | func DefaultCompletion(messages []kafka.Message, err error) { 9 | for i := range messages { 10 | if err != nil { 11 | logger.Log.Error("message produce error, error: %s message: %s, \n", err.Error(), messages[i].WriterData) 12 | } else { 13 | logger.Log.Debug("message produce success, message: %s\n", messages[i].WriterData) 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /kafka/producer/producer.go: -------------------------------------------------------------------------------- 1 | package producer 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/Trendyol/go-dcp/helpers" 7 | 8 | "github.com/Trendyol/go-dcp-kafka/config" 9 | gKafka "github.com/Trendyol/go-dcp-kafka/kafka" 10 | "github.com/Trendyol/go-dcp/models" 11 | "github.com/segmentio/kafka-go" 12 | ) 13 | 14 | type Metric struct { 15 | KafkaConnectorLatency int64 16 | BatchProduceLatency int64 17 | } 18 | 19 | type Producer struct { 20 | ProducerBatch *Batch 21 | } 22 | 23 | func NewProducer(kafkaClient gKafka.Client, 24 | config *config.Connector, 25 | dcpCheckpointCommit func(), 26 | sinkResponseHandler gKafka.SinkResponseHandler, 27 | completionHandler func(messages []kafka.Message, err error), 28 | ) (Producer, error) { 29 | writer := kafkaClient.Producer(completionHandler) 30 | 31 | if sinkResponseHandler != nil { 32 | sinkResponseHandler.OnInit(&gKafka.SinkResponseHandlerInitContext{ 33 | Config: config.Kafka, 34 | KafkaClient: kafkaClient, 35 | Writer: writer, 36 | }) 37 | } 38 | 39 | return Producer{ 40 | ProducerBatch: newBatch( 41 | config.Kafka.ProducerBatchTickerDuration, 42 | writer, 43 | config.Kafka.ProducerBatchSize, 44 | int64(helpers.ResolveUnionIntOrStringValue(config.Kafka.ProducerBatchBytes)), 45 | dcpCheckpointCommit, 46 | sinkResponseHandler, 47 | ), 48 | }, nil 49 | } 50 | 51 | func (p *Producer) StartBatch() { 52 | p.ProducerBatch.StartBatchTicker() 53 | } 54 | 55 | func (p *Producer) Produce( 56 | ctx *models.ListenerContext, 57 | eventTime time.Time, 58 | messages []kafka.Message, 59 | isLastChunk bool, 60 | ) { 61 | p.ProducerBatch.AddMessages(ctx, messages, eventTime, isLastChunk) 62 | } 63 | 64 | func (p *Producer) Close() error { 65 | p.ProducerBatch.Close() 66 | return p.ProducerBatch.Writer.Close() 67 | } 68 | 69 | func (p *Producer) GetMetric() *Metric { 70 | return p.ProducerBatch.metric 71 | } 72 | -------------------------------------------------------------------------------- /kafka/producer/producer_batch.go: -------------------------------------------------------------------------------- 1 | package producer 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | gKafka "github.com/Trendyol/go-dcp-kafka/kafka" 10 | "github.com/Trendyol/go-dcp-kafka/kafka/message" 11 | "github.com/Trendyol/go-dcp/logger" 12 | 13 | "github.com/Trendyol/go-dcp/models" 14 | "github.com/segmentio/kafka-go" 15 | ) 16 | 17 | type Batch struct { 18 | sinkResponseHandler gKafka.SinkResponseHandler 19 | batchTicker *time.Ticker 20 | Writer *kafka.Writer 21 | dcpCheckpointCommit func() 22 | metric *Metric 23 | messages []kafka.Message 24 | currentMessageBytes int64 25 | batchTickerDuration time.Duration 26 | batchLimit int 27 | batchBytes int64 28 | flushLock sync.Mutex 29 | isDcpRebalancing bool 30 | } 31 | 32 | func newBatch( 33 | batchTime time.Duration, 34 | writer *kafka.Writer, 35 | batchLimit int, 36 | batchBytes int64, 37 | dcpCheckpointCommit func(), 38 | sinkResponseHandler gKafka.SinkResponseHandler, 39 | ) *Batch { 40 | batch := &Batch{ 41 | batchTickerDuration: batchTime, 42 | batchTicker: time.NewTicker(batchTime), 43 | metric: &Metric{}, 44 | messages: make([]kafka.Message, 0, batchLimit), 45 | Writer: writer, 46 | batchLimit: batchLimit, 47 | dcpCheckpointCommit: dcpCheckpointCommit, 48 | batchBytes: batchBytes, 49 | sinkResponseHandler: sinkResponseHandler, 50 | } 51 | return batch 52 | } 53 | 54 | func (b *Batch) StartBatchTicker() { 55 | go func() { 56 | for { 57 | <-b.batchTicker.C 58 | b.FlushMessages() 59 | } 60 | }() 61 | } 62 | 63 | func (b *Batch) Close() { 64 | b.batchTicker.Stop() 65 | b.FlushMessages() 66 | } 67 | 68 | func (b *Batch) PrepareStartRebalancing() { 69 | b.flushLock.Lock() 70 | defer b.flushLock.Unlock() 71 | 72 | b.isDcpRebalancing = true 73 | b.messages = b.messages[:0] 74 | b.currentMessageBytes = 0 75 | } 76 | 77 | func (b *Batch) PrepareEndRebalancing() { 78 | b.flushLock.Lock() 79 | defer b.flushLock.Unlock() 80 | 81 | b.isDcpRebalancing = false 82 | } 83 | 84 | func (b *Batch) AddMessages(ctx *models.ListenerContext, messages []kafka.Message, eventTime time.Time, isLastChunk bool) { 85 | b.flushLock.Lock() 86 | if b.isDcpRebalancing { 87 | logger.Log.Error("could not add new message to batch while rebalancing") 88 | b.flushLock.Unlock() 89 | return 90 | } 91 | b.messages = append(b.messages, messages...) 92 | b.currentMessageBytes += totalSizeOfMessages(messages) 93 | if isLastChunk { 94 | ctx.Ack() 95 | } 96 | b.flushLock.Unlock() 97 | 98 | if isLastChunk { 99 | b.metric.KafkaConnectorLatency = time.Since(eventTime).Milliseconds() 100 | } 101 | 102 | if len(b.messages) >= b.batchLimit || b.currentMessageBytes >= b.batchBytes { 103 | b.FlushMessages() 104 | } 105 | } 106 | 107 | func (b *Batch) FlushMessages() { 108 | b.flushLock.Lock() 109 | defer b.flushLock.Unlock() 110 | if b.isDcpRebalancing { 111 | return 112 | } 113 | if len(b.messages) > 0 { 114 | startedTime := time.Now() 115 | err := b.Writer.WriteMessages(context.Background(), b.messages...) 116 | 117 | if err != nil && b.sinkResponseHandler == nil { 118 | err = fmt.Errorf("batch producer flush error %v", err) 119 | logger.Log.Error("error while flush, err: %v", err) 120 | panic(err) 121 | } 122 | 123 | b.metric.BatchProduceLatency = time.Since(startedTime).Milliseconds() 124 | 125 | if b.sinkResponseHandler != nil { 126 | switch e := err.(type) { 127 | case nil: 128 | b.handleResponseSuccess() 129 | case kafka.WriteErrors: 130 | b.handleWriteError(e) 131 | case kafka.MessageTooLargeError: 132 | b.handleMessageTooLargeError(e) 133 | default: 134 | b.handleResponseError(e) 135 | logger.Log.Error("batch producer flush default error %v", err) 136 | } 137 | } 138 | 139 | b.messages = b.messages[:0] 140 | b.currentMessageBytes = 0 141 | b.batchTicker.Reset(b.batchTickerDuration) 142 | } 143 | b.dcpCheckpointCommit() 144 | } 145 | 146 | func (b *Batch) handleWriteError(writeErrors kafka.WriteErrors) { 147 | for i := range writeErrors { 148 | if writeErrors[i] != nil { 149 | b.sinkResponseHandler.OnError(&gKafka.SinkResponseHandlerContext{ 150 | Message: convertKafkaMessage(b.messages[i]), 151 | Err: writeErrors[i], 152 | }) 153 | } else { 154 | b.sinkResponseHandler.OnSuccess(&gKafka.SinkResponseHandlerContext{ 155 | Message: convertKafkaMessage(b.messages[i]), 156 | Err: nil, 157 | }) 158 | } 159 | } 160 | } 161 | 162 | func (b *Batch) handleResponseError(err error) { 163 | for _, msg := range b.messages { 164 | b.sinkResponseHandler.OnError(&gKafka.SinkResponseHandlerContext{ 165 | Message: convertKafkaMessage(msg), 166 | Err: err, 167 | }) 168 | } 169 | } 170 | 171 | func (b *Batch) handleResponseSuccess() { 172 | for _, msg := range b.messages { 173 | b.sinkResponseHandler.OnSuccess(&gKafka.SinkResponseHandlerContext{ 174 | Message: convertKafkaMessage(msg), 175 | Err: nil, 176 | }) 177 | } 178 | } 179 | 180 | func (b *Batch) handleMessageTooLargeError(mTooLargeError kafka.MessageTooLargeError) { 181 | b.sinkResponseHandler.OnError(&gKafka.SinkResponseHandlerContext{ 182 | Message: convertKafkaMessage(mTooLargeError.Message), 183 | Err: mTooLargeError, 184 | }) 185 | } 186 | 187 | func convertKafkaMessage(src kafka.Message) *message.KafkaMessage { 188 | return &message.KafkaMessage{ 189 | Topic: src.Topic, 190 | Headers: src.Headers, 191 | Key: src.Key, 192 | Value: src.Value, 193 | } 194 | } 195 | 196 | func totalSizeOfMessages(messages []kafka.Message) int64 { 197 | var size int 198 | for _, m := range messages { 199 | headerSize := 0 200 | for _, header := range m.Headers { 201 | headerSize += 2 + len(header.Key) 202 | headerSize += len(header.Value) 203 | } 204 | size += 14 + (4 + len(m.Key)) + (4 + len(m.Value)) + headerSize 205 | } 206 | return int64(size) 207 | } 208 | -------------------------------------------------------------------------------- /kafka/rejection_log_sink_response_handler.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/Trendyol/go-dcp-kafka/config" 8 | "github.com/Trendyol/go-dcp/logger" 9 | jsoniter "github.com/json-iterator/go" 10 | "github.com/segmentio/kafka-go" 11 | ) 12 | 13 | type RejectionLogSinkResponseHandler struct { 14 | KafkaClient Client 15 | Writer *kafka.Writer 16 | Topic string 17 | Config config.Kafka 18 | } 19 | 20 | func (r *RejectionLogSinkResponseHandler) OnInit(ctx *SinkResponseHandlerInitContext) { 21 | r.Config = ctx.Config 22 | r.KafkaClient = ctx.KafkaClient 23 | r.Writer = ctx.Writer 24 | r.Topic = ctx.Config.RejectionLog.Topic 25 | 26 | err := r.KafkaClient.CheckTopics([]string{r.Topic}) 27 | if err != nil { 28 | logger.Log.Error("error while rejection topic exist request, err: %v", err) 29 | panic(err) 30 | } 31 | } 32 | 33 | func (r *RejectionLogSinkResponseHandler) OnSuccess(_ *SinkResponseHandlerContext) { 34 | } 35 | 36 | func (r *RejectionLogSinkResponseHandler) OnError(ctx *SinkResponseHandlerContext) { 37 | rejectionLog := r.buildRejectionLog(ctx) 38 | if err := r.publishToKafka(ctx, rejectionLog); err != nil { 39 | logger.Log.Error("failed to publish rejection log, err: %v", err) 40 | panic(err) 41 | } 42 | } 43 | 44 | func (r *RejectionLogSinkResponseHandler) buildRejectionLog(ctx *SinkResponseHandlerContext) RejectionLog { 45 | rejectionLog := RejectionLog{ 46 | Topic: ctx.Message.Topic, 47 | Key: ctx.Message.Key, 48 | Error: ctx.Err.Error(), 49 | } 50 | 51 | if r.Config.RejectionLog.IncludeValue { 52 | rejectionLog.Value = string(ctx.Message.Value) 53 | } 54 | 55 | return rejectionLog 56 | } 57 | 58 | func (r *RejectionLogSinkResponseHandler) publishToKafka(ctx *SinkResponseHandlerContext, rejectionLog RejectionLog) error { 59 | logBytes, err := jsoniter.Marshal(rejectionLog) 60 | if err != nil { 61 | return fmt.Errorf("failed to marshal rejection log: %w", err) 62 | } 63 | 64 | kafkaMessage := kafka.Message{ 65 | Topic: r.Topic, 66 | Key: rejectionLog.Key, 67 | Value: logBytes, 68 | Headers: ctx.Message.Headers, 69 | } 70 | 71 | if err := r.Writer.WriteMessages(context.Background(), kafkaMessage); err != nil { 72 | return fmt.Errorf("failed to write Kafka message: %w", err) 73 | } 74 | 75 | return nil 76 | } 77 | 78 | func NewRejectionLogSinkResponseHandler() SinkResponseHandler { 79 | return &RejectionLogSinkResponseHandler{} 80 | } 81 | 82 | type RejectionLog struct { 83 | Topic string 84 | Value string 85 | Error string 86 | Key []byte 87 | } 88 | -------------------------------------------------------------------------------- /kafka/sink_response_handler.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "github.com/Trendyol/go-dcp-kafka/config" 5 | "github.com/Trendyol/go-dcp-kafka/kafka/message" 6 | "github.com/segmentio/kafka-go" 7 | ) 8 | 9 | type SinkResponseHandlerContext struct { 10 | Message *message.KafkaMessage 11 | Err error 12 | } 13 | 14 | type SinkResponseHandlerInitContext struct { 15 | KafkaClient Client 16 | Writer *kafka.Writer 17 | Config config.Kafka 18 | } 19 | 20 | type SinkResponseHandler interface { 21 | OnSuccess(ctx *SinkResponseHandlerContext) 22 | OnError(ctx *SinkResponseHandlerContext) 23 | OnInit(ctx *SinkResponseHandlerInitContext) 24 | } 25 | -------------------------------------------------------------------------------- /mapper.go: -------------------------------------------------------------------------------- 1 | package dcpkafka 2 | 3 | import ( 4 | "github.com/Trendyol/go-dcp-kafka/couchbase" 5 | "github.com/Trendyol/go-dcp-kafka/kafka/message" 6 | ) 7 | 8 | type Mapper func(event couchbase.Event) []message.KafkaMessage 9 | 10 | func DefaultMapper(event couchbase.Event) []message.KafkaMessage { 11 | if event.IsExpired || event.IsDeleted { 12 | return nil 13 | } 14 | return []message.KafkaMessage{ 15 | { 16 | Key: event.Key, 17 | Value: event.Value, 18 | }, 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /metric/collector.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "github.com/Trendyol/go-dcp-kafka/kafka/producer" 5 | "github.com/Trendyol/go-dcp/helpers" 6 | "github.com/prometheus/client_golang/prometheus" 7 | ) 8 | 9 | type Collector struct { 10 | producer producer.Producer 11 | 12 | kafkaConnectorLatency *prometheus.Desc 13 | batchProduceLatency *prometheus.Desc 14 | } 15 | 16 | func (s *Collector) Describe(ch chan<- *prometheus.Desc) { 17 | prometheus.DescribeByCollect(s, ch) 18 | } 19 | 20 | func (s *Collector) Collect(ch chan<- prometheus.Metric) { 21 | producerMetric := s.producer.GetMetric() 22 | 23 | ch <- prometheus.MustNewConstMetric( 24 | s.kafkaConnectorLatency, 25 | prometheus.GaugeValue, 26 | float64(producerMetric.KafkaConnectorLatency), 27 | []string{}..., 28 | ) 29 | 30 | ch <- prometheus.MustNewConstMetric( 31 | s.batchProduceLatency, 32 | prometheus.GaugeValue, 33 | float64(producerMetric.BatchProduceLatency), 34 | []string{}..., 35 | ) 36 | } 37 | 38 | func NewMetricCollector(producer producer.Producer) *Collector { 39 | return &Collector{ 40 | producer: producer, 41 | 42 | kafkaConnectorLatency: prometheus.NewDesc( 43 | prometheus.BuildFQName(helpers.Name, "kafka_connector_latency_ms", "current"), 44 | "Kafka connector latency ms at 10sec windows", 45 | []string{}, 46 | nil, 47 | ), 48 | 49 | batchProduceLatency: prometheus.NewDesc( 50 | prometheus.BuildFQName(helpers.Name, "kafka_connector_batch_produce_latency_ms", "current"), 51 | "Kafka connector batch produce latency ms", 52 | []string{}, 53 | nil, 54 | ), 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /test/couchbase/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM couchbase:7.6.3 2 | 3 | ADD configure.sh /configure.sh 4 | RUN chmod +x /configure.sh 5 | RUN echo " image starting" 6 | 7 | EXPOSE 8091 8092 8093 8094 8095 8096 11207 11210 11211 8 | 9 | CMD ["/configure.sh"] 10 | -------------------------------------------------------------------------------- /test/couchbase/configure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Enables job control 4 | set -m 5 | 6 | # Enables error propagation 7 | set -e 8 | 9 | # Run the server and send it to the background 10 | /entrypoint.sh couchbase-server & 11 | 12 | # Check if couchbase server is up 13 | check_db() { 14 | curl --silent http://127.0.0.1:8091/pools > /dev/null 15 | echo $? 16 | } 17 | 18 | # Variable used in echo 19 | i=1 20 | # Echo with 21 | log() { 22 | echo "[$i] [$(date +"%T")] $@" 23 | i=`expr $i + 1` 24 | } 25 | 26 | # Wait until it's ready 27 | until [[ $(check_db) = 0 ]]; do 28 | >&2 log "Waiting for Couchbase Server to be available ..." 29 | sleep 1 30 | done 31 | 32 | couchbase-cli cluster-init -c localhost --cluster-name Cluster --cluster-username user \ 33 | --cluster-password password --services data --cluster-ramsize 1024 34 | 35 | couchbase-cli bucket-create -c couchbase --username user --password password --bucket dcp-test --bucket-type couchbase --bucket-ramsize 768 36 | couchbase-cli bucket-create -c couchbase --username user --password password --bucket checkpoint-bucket-name --bucket-type couchbase --bucket-ramsize 256 37 | 38 | cbimport json -c couchbase://127.0.0.1 -u user -p password --bucket-quota 768 -b dcp-test -d file://opt/couchbase/samples/travel-sample.zip -f sample 39 | 40 | echo "couchbase-dev started" 41 | 42 | fg 1 43 | -------------------------------------------------------------------------------- /test/integration/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20-alpine as builder 2 | 3 | WORKDIR /project 4 | 5 | RUN apk add build-base 6 | 7 | COPY .. . 8 | 9 | RUN go mod tidy 10 | RUN go test -v ./... -------------------------------------------------------------------------------- /test/integration/config.yml: -------------------------------------------------------------------------------- 1 | hosts: 2 | - localhost:8091 3 | username: user 4 | password: password 5 | bucketName: dcp-test 6 | rollbackMitigation: 7 | disabled: true 8 | checkpoint: 9 | type: manual 10 | dcp: 11 | group: 12 | name: groupName 13 | metadata: 14 | type: couchbase 15 | readOnly: true 16 | config: 17 | bucket: checkpoint-bucket-name 18 | scope: _default 19 | collection: _default 20 | kafka: 21 | collectionTopicMapping: 22 | _default: topicname 23 | brokers: 24 | - "localhost:9092" 25 | readTimeout: 30s 26 | writeTimeout: 30s 27 | producerBatchSize: 100 28 | producerBatchBytes: 1048576 29 | producerBatchTickerDuration: 5s 30 | metadataTTL: 2400s 31 | metadataTopics: 32 | - "topicname" -------------------------------------------------------------------------------- /test/integration/go.mod: -------------------------------------------------------------------------------- 1 | module integration-test-example 2 | 3 | go 1.20 4 | 5 | replace github.com/Trendyol/go-dcp-kafka => ../../. 6 | 7 | require ( 8 | github.com/Trendyol/go-dcp v1.2.6 9 | github.com/Trendyol/go-dcp-kafka v0.0.0 10 | github.com/segmentio/kafka-go v0.4.47 11 | ) 12 | 13 | require ( 14 | github.com/andybalholm/brotli v1.1.1 // indirect 15 | github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect 16 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect 17 | github.com/beorn7/perks v1.0.1 // indirect 18 | github.com/bytedance/sonic v1.12.8 // indirect 19 | github.com/bytedance/sonic/loader v0.2.2 // indirect 20 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 21 | github.com/cloudwego/base64x v0.1.5 // indirect 22 | github.com/couchbase/gocbcore/v10 v10.5.2 // indirect 23 | github.com/davecgh/go-spew v1.1.1 // indirect 24 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 25 | github.com/go-logr/logr v1.4.1 // indirect 26 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 27 | github.com/go-openapi/jsonreference v0.20.2 // indirect 28 | github.com/go-openapi/swag v0.22.3 // indirect 29 | github.com/gofiber/fiber/v2 v2.52.5 // indirect 30 | github.com/gogo/protobuf v1.3.2 // indirect 31 | github.com/golang/protobuf v1.5.4 // indirect 32 | github.com/golang/snappy v0.0.4 // indirect 33 | github.com/google/gnostic-models v0.6.8 // indirect 34 | github.com/google/gofuzz v1.2.0 // indirect 35 | github.com/google/uuid v1.6.0 // indirect 36 | github.com/josharian/intern v1.0.0 // indirect 37 | github.com/json-iterator/go v1.1.12 // indirect 38 | github.com/klauspost/compress v1.17.11 // indirect 39 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 40 | github.com/mailru/easyjson v0.7.7 // indirect 41 | github.com/mattn/go-colorable v0.1.13 // indirect 42 | github.com/mattn/go-isatty v0.0.20 // indirect 43 | github.com/mattn/go-runewidth v0.0.15 // indirect 44 | github.com/mhmtszr/concurrent-swiss-map v1.0.8 // indirect 45 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 46 | github.com/modern-go/reflect2 v1.0.2 // indirect 47 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 48 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 49 | github.com/prometheus/client_golang v1.20.5 // indirect 50 | github.com/prometheus/client_model v0.6.1 // indirect 51 | github.com/prometheus/common v0.58.0 // indirect 52 | github.com/prometheus/procfs v0.15.1 // indirect 53 | github.com/rivo/uniseg v0.4.7 // indirect 54 | github.com/sirupsen/logrus v1.9.3 // indirect 55 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 56 | github.com/valyala/bytebufferpool v1.0.0 // indirect 57 | github.com/valyala/fasthttp v1.57.0 // indirect 58 | github.com/valyala/tcplisten v1.0.0 // indirect 59 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 60 | github.com/xdg-go/scram v1.1.2 // indirect 61 | github.com/xdg-go/stringprep v1.0.4 // indirect 62 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect 63 | golang.org/x/net v0.33.0 // indirect 64 | golang.org/x/oauth2 v0.22.0 // indirect 65 | golang.org/x/sync v0.10.0 // indirect 66 | golang.org/x/sys v0.28.0 // indirect 67 | golang.org/x/term v0.27.0 // indirect 68 | golang.org/x/text v0.21.0 // indirect 69 | golang.org/x/time v0.3.0 // indirect 70 | google.golang.org/protobuf v1.36.4 // indirect 71 | gopkg.in/inf.v0 v0.9.1 // indirect 72 | gopkg.in/yaml.v2 v2.4.0 // indirect 73 | gopkg.in/yaml.v3 v3.0.1 // indirect 74 | k8s.io/api v0.29.4 // indirect 75 | k8s.io/apimachinery v0.29.4 // indirect 76 | k8s.io/client-go v0.29.4 // indirect 77 | k8s.io/klog/v2 v2.110.1 // indirect 78 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect 79 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 80 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 81 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 82 | sigs.k8s.io/yaml v1.3.0 // indirect 83 | ) 84 | -------------------------------------------------------------------------------- /test/integration/go.sum: -------------------------------------------------------------------------------- 1 | github.com/Trendyol/go-dcp v1.2.6 h1:4DlFXHYlN7b66jEFzRfP34s9smwBYjs4IsQCD+y/5pM= 2 | github.com/Trendyol/go-dcp v1.2.6/go.mod h1:t2vNISMXxbYPA6jNHBUNqdk0jzGuVMQKkZnCrSiQ1ik= 3 | github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= 4 | github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= 5 | github.com/ansrivas/fiberprometheus/v2 v2.7.0 h1:09XiSzG0J7aZp7RviklngdWdDbSybKjhuWAstp003Gg= 6 | github.com/ansrivas/fiberprometheus/v2 v2.7.0/go.mod h1:hSJdO65lfnWW70Qn9uGdXXsUUSkckbhuw5r/KesygpU= 7 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= 8 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= 9 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 10 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 11 | github.com/bytedance/sonic v1.12.8 h1:4xYRVRlXIgvSZ4e8iVTlMF5szgpXd4AfvuWgA8I8lgs= 12 | github.com/bytedance/sonic v1.12.8/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8= 13 | github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= 14 | github.com/bytedance/sonic/loader v0.2.2 h1:jxAJuN9fOot/cyz5Q6dUuMJF5OqQ6+5GfA8FjjQ0R4o= 15 | github.com/bytedance/sonic/loader v0.2.2/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= 16 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 17 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 18 | github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= 19 | github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= 20 | github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= 21 | github.com/couchbase/gocbcore/v10 v10.5.2 h1:DHK042E1RfhPBR3b14CITl5XHRsLjH3hpERuwUc5UIg= 22 | github.com/couchbase/gocbcore/v10 v10.5.2/go.mod h1:rulbgUK70EuyRUiLQ0LhQAfSI/Rl+jWws8tTbHzvB6M= 23 | github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4= 24 | github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY= 25 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 26 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 27 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 28 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 29 | github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= 30 | github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= 31 | github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= 32 | github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 33 | github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= 34 | github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 35 | github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= 36 | github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= 37 | github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= 38 | github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= 39 | github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= 40 | github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= 41 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= 42 | github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= 43 | github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= 44 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 45 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 46 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= 47 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 48 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 49 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 50 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 51 | github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= 52 | github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= 53 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 54 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 55 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 56 | github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= 57 | github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 58 | github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= 59 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 60 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 61 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 62 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 63 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 64 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 65 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 66 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 67 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 68 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 69 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 70 | github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= 71 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 72 | github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= 73 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 74 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 75 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 76 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 77 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 78 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 79 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 80 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 81 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 82 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 83 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 84 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 85 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 86 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 87 | github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= 88 | github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 89 | github.com/mhmtszr/concurrent-swiss-map v1.0.8 h1:GDSxgVrXsPFsraUJaPMm7ptYulj8qnWPgnwXcWbJNxo= 90 | github.com/mhmtszr/concurrent-swiss-map v1.0.8/go.mod h1:F6QETL48Qn7jEJ3ZPt7EqRZjAAZu7lRQeQGIzXuUIDc= 91 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 92 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 93 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 94 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 95 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 96 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 97 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 98 | github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= 99 | github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= 100 | github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= 101 | github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= 102 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 103 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 104 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 105 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 106 | github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= 107 | github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= 108 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 109 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 110 | github.com/prometheus/common v0.58.0 h1:N+N8vY4/23r6iYfD3UQZUoJPnUYAo7v6LG5XZxjZTXo= 111 | github.com/prometheus/common v0.58.0/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= 112 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 113 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 114 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 115 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 116 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 117 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 118 | github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0= 119 | github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= 120 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 121 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 122 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 123 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 124 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 125 | github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= 126 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 127 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 128 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 129 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 130 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 131 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 132 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 133 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 134 | github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU= 135 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= 136 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= 137 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 138 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 139 | github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg= 140 | github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE= 141 | github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= 142 | github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= 143 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 144 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 145 | github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= 146 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 147 | github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= 148 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 149 | github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= 150 | github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= 151 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 152 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 153 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 154 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= 155 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= 156 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 157 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 158 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 159 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 160 | golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= 161 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 162 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 163 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 164 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 165 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 166 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 167 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 168 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 169 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 170 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 171 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 172 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 173 | golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= 174 | golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= 175 | golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= 176 | golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= 177 | golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= 178 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 179 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 180 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 181 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 182 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 183 | golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= 184 | golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 185 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 186 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 187 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 188 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 189 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 190 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 191 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 192 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 193 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 194 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 195 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 196 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 197 | golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 198 | golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= 199 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 200 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 201 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 202 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 203 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 204 | golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= 205 | golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= 206 | golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= 207 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 208 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 209 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 210 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 211 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 212 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 213 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 214 | golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= 215 | golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= 216 | golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= 217 | golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 218 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 219 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 220 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 221 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 222 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 223 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 224 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= 225 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 226 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 227 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 228 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 229 | google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= 230 | google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 231 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 232 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 233 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 234 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 235 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 236 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 237 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 238 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 239 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 240 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 241 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 242 | k8s.io/api v0.29.4 h1:WEnF/XdxuCxdG3ayHNRR8yH3cI1B/llkWBma6bq4R3w= 243 | k8s.io/api v0.29.4/go.mod h1:DetSv0t4FBTcEpfA84NJV3g9a7+rSzlUHk5ADAYHUv0= 244 | k8s.io/apimachinery v0.29.4 h1:RaFdJiDmuKs/8cm1M6Dh1Kvyh59YQFDcFuFTSmXes6Q= 245 | k8s.io/apimachinery v0.29.4/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= 246 | k8s.io/client-go v0.29.4 h1:79ytIedxVfyXV8rpH3jCBW0u+un0fxHDwX5F9K8dPR8= 247 | k8s.io/client-go v0.29.4/go.mod h1:kC1thZQ4zQWYwldsfI088BbK6RkxK+aF5ebV8y9Q4tk= 248 | k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= 249 | k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= 250 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= 251 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= 252 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= 253 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 254 | nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= 255 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= 256 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= 257 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= 258 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= 259 | sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= 260 | sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= 261 | -------------------------------------------------------------------------------- /test/integration/integration_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "context" 5 | dcpkafka "github.com/Trendyol/go-dcp-kafka" 6 | "github.com/Trendyol/go-dcp-kafka/config" 7 | "github.com/Trendyol/go-dcp-kafka/couchbase" 8 | "github.com/Trendyol/go-dcp-kafka/kafka/message" 9 | dcpConfig "github.com/Trendyol/go-dcp/config" 10 | "github.com/segmentio/kafka-go" 11 | "sync" 12 | "testing" 13 | "time" 14 | ) 15 | 16 | func mapper(event couchbase.Event) []message.KafkaMessage { 17 | if event.IsExpired || event.IsDeleted { 18 | return nil 19 | } 20 | return []message.KafkaMessage{ 21 | { 22 | Headers: nil, 23 | Key: event.Key, 24 | Value: event.Value, 25 | }, 26 | } 27 | } 28 | 29 | func TestKafka(t *testing.T) { 30 | time.Sleep(time.Second * 30) 31 | 32 | connector, err := dcpkafka.NewConnectorBuilder(&config.Connector{ 33 | Dcp: dcpConfig.Dcp{ 34 | Hosts: []string{"localhost:8091"}, 35 | Username: "user", 36 | Password: "password", 37 | BucketName: "dcp-test", 38 | RollbackMitigation: dcpConfig.RollbackMitigation{ 39 | Disabled: true, 40 | }, 41 | Dcp: dcpConfig.ExternalDcp{ 42 | Group: dcpConfig.DCPGroup{ 43 | Name: "groupName", 44 | }, 45 | }, 46 | Metadata: dcpConfig.Metadata{ 47 | ReadOnly: true, 48 | Config: map[string]string{ 49 | "bucket": "checkpoint-bucket-name", 50 | "scope": "_default", 51 | "collection": "_default", 52 | }, 53 | Type: "couchbase", 54 | }, 55 | Debug: true}, 56 | Kafka: config.Kafka{ 57 | CollectionTopicMapping: map[string]string{"_default": "topicname"}, 58 | Brokers: []string{"localhost:9092"}, 59 | ProducerBatchBytes: 1048576, 60 | ProducerBatchSize: 100, 61 | ReadTimeout: 30 * time.Second, 62 | WriteTimeout: 30 * time.Second, 63 | MetadataTTL: 2400 * time.Second, 64 | MetadataTopics: []string{"topicname"}, 65 | }, 66 | }).SetMapper(mapper).Build() 67 | if err != nil { 68 | t.Fatal(err) 69 | return 70 | } 71 | 72 | var wg sync.WaitGroup 73 | wg.Add(1) 74 | go func() { 75 | defer wg.Done() 76 | connector.Start() 77 | }() 78 | 79 | go func() { 80 | r := kafka.NewReader(kafka.ReaderConfig{ 81 | Brokers: []string{"localhost:9092"}, 82 | Topic: "topicname", 83 | QueueCapacity: 200, 84 | MaxBytes: 10e6, // 10MB 85 | }) 86 | 87 | totalEvent := 0 88 | ctx, _ := context.WithTimeout(context.Background(), 5*time.Minute) 89 | 90 | for { 91 | select { 92 | case <-ctx.Done(): 93 | t.Fatalf("deadline exceed") 94 | default: 95 | m, err := r.ReadMessage(context.Background()) 96 | if err != nil { 97 | t.Fatalf("error while reading topic %s", err) 98 | } 99 | if m.Value != nil { 100 | totalEvent += 1 101 | } 102 | if totalEvent == 31591 { 103 | connector.Close() 104 | break 105 | } 106 | } 107 | } 108 | }() 109 | 110 | wg.Wait() 111 | } 112 | --------------------------------------------------------------------------------