├── .github ├── dependabot.yml ├── images │ ├── architecture.excalidraw │ ├── architecture.png │ ├── backoff_strategy_architecture.png │ └── cronsumer.png └── workflows │ ├── codeql.yml │ ├── integration-test.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── CODE-OF-CONDUCT.md ├── LICENCE ├── Makefile ├── README.md ├── SECURITY.md ├── backoff-strategy-structure.md ├── cronsumer.go ├── examples ├── docker-compose.yml ├── multiple-consumer │ ├── go.mod │ ├── go.sum │ └── main.go ├── single-consumer-with-backoff-strategy │ ├── go.mod │ ├── go.sum │ └── main.go ├── single-consumer-with-custom-logger │ ├── go.mod │ ├── go.sum │ ├── logger.go │ └── main.go ├── single-consumer-with-deadletter │ ├── go.mod │ ├── go.sum │ └── main.go ├── single-consumer-with-header-filter-function │ ├── go.mod │ ├── go.sum │ └── main.go ├── single-consumer-with-metric-collector │ ├── api.go │ ├── go.mod │ ├── go.sum │ ├── main.go │ └── metric.go ├── single-consumer-with-producer │ ├── go.mod │ ├── go.sum │ └── main.go └── single-consumer │ ├── go.mod │ ├── go.sum │ └── main.go ├── go.mod ├── go.sum ├── internal ├── collector.go ├── collector_test.go ├── consumer.go ├── cronsumer.go ├── cronsumer_client.go ├── cronsumer_client_test.go ├── cronsumer_test.go ├── message.go ├── message_header.go ├── message_header_test.go ├── message_test.go ├── metric.go ├── producer.go ├── secure.go ├── verify_topic.go └── verify_topic_test.go ├── pkg ├── kafka │ ├── backoff_strategy.go │ ├── config.go │ ├── config_test.go │ ├── cron_client.go │ ├── cronsumer_message.go │ └── cronsumer_message_test.go └── logger │ ├── logger.go │ └── zap.go ├── test └── integration │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ └── integration_test.go └── testdata └── message.json /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 10 -------------------------------------------------------------------------------- /.github/images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Trendyol/kafka-cronsumer/218a5163a16710cd556c739394ec93df6ba97a4c/.github/images/architecture.png -------------------------------------------------------------------------------- /.github/images/backoff_strategy_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Trendyol/kafka-cronsumer/218a5163a16710cd556c739394ec93df6ba97a4c/.github/images/backoff_strategy_architecture.png -------------------------------------------------------------------------------- /.github/images/cronsumer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Trendyol/kafka-cronsumer/218a5163a16710cd556c739394ec93df6ba97a4c/.github/images/cronsumer.png -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL Analysis" 2 | 3 | on: 4 | push: 5 | branches: [ "v2" ] 6 | pull_request: 7 | branches: [ "v2" ] 8 | schedule: 9 | - cron: '0 0 * * 6' # Runs only at 00.00 Saturdays. 10 | 11 | jobs: 12 | analyze: 13 | name: Analyze 14 | runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} 15 | timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} 16 | permissions: 17 | # required for all workflows 18 | security-events: write 19 | 20 | # only required for workflows in private repositories 21 | actions: read 22 | contents: read 23 | 24 | strategy: 25 | fail-fast: false 26 | matrix: 27 | language: [ 'go' ] 28 | 29 | steps: 30 | - name: Checkout repository 31 | uses: actions/checkout@v4 32 | 33 | # Initializes the CodeQL tools for scanning. 34 | - name: Initialize CodeQL 35 | uses: github/codeql-action/init@v3 36 | with: 37 | languages: ${{ matrix.language }} 38 | # If you wish to specify custom queries, you can do so here or in a config file. 39 | # By default, queries listed here will override any specified in a config file. 40 | # Prefix the list here with "+" to use these queries and those in the config file. 41 | 42 | # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 43 | # queries: security-extended,security-and-quality 44 | 45 | - name: Autobuild 46 | uses: github/codeql-action/autobuild@v3 47 | # If the Autobuild fails above, remove it and uncomment the following three lines. 48 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 49 | 50 | # - run: | 51 | # echo "Run, Build Application using script" 52 | # ./location_of_script_within_repo/buildscript.sh 53 | 54 | - name: Perform CodeQL Analysis 55 | uses: github/codeql-action/analyze@v3 56 | with: 57 | category: "/language:${{matrix.language}}" 58 | -------------------------------------------------------------------------------- /.github/workflows/integration-test.yml: -------------------------------------------------------------------------------- 1 | name: IntegrationTest 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v3 15 | with: 16 | fetch-depth: 0 17 | 18 | - name: Set up Go 19 | uses: actions/setup-go@v3 20 | with: 21 | go-version: 1.19 22 | 23 | - uses: actions/checkout@v3 24 | - name: Start containers 25 | run: make integration-compose 26 | 27 | - uses: actions/checkout@v3 28 | - name: Integration Test 29 | run: go test -timeout=15m -v test/integration/integration_test.go 30 | env: 31 | INPUT_PUBLISH: false 32 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: 🎉 Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | permissions: 9 | contents: write 10 | 11 | jobs: 12 | goreleaser: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout 16 | uses: actions/checkout@v3 17 | with: 18 | fetch-depth: 0 19 | 20 | - run: git tag ${{ github.event.inputs.tag }} 21 | 22 | - name: Set up Go 23 | uses: actions/setup-go@v3 24 | with: 25 | go-version: 1.19 26 | 27 | - name: Run GoReleaser 28 | uses: goreleaser/goreleaser-action@v4 29 | with: 30 | version: latest 31 | args: release --clean 32 | env: 33 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 34 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: 🔨Build And Test 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v3 15 | with: 16 | fetch-depth: 0 17 | 18 | - name: Set up Go 19 | uses: actions/setup-go@v3 20 | with: 21 | go-version: 1.19 22 | 23 | - name: Install dependencies 24 | run: go get . 25 | 26 | - name: golangci-lint 27 | uses: golangci/golangci-lint-action@v3 28 | with: 29 | version: v1.51 30 | args: -c .golangci.yml --timeout=5m -v 31 | 32 | - name: Build 33 | run: go build -v ./... 34 | 35 | - name: Test 36 | run: go test ./... -v -race -coverprofile=coverage.txt -covermode=atomic 37 | 38 | - name: Upload coverage 39 | run: bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | unit_coverage.html 3 | unit_coverage.out 4 | resource/ 5 | dist/ -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | lll: 3 | line-length: 140 4 | funlen: 5 | lines: 70 6 | 7 | linters: 8 | disable-all: true 9 | enable: 10 | - bodyclose 11 | - depguard 12 | - errcheck 13 | - dupl 14 | - exhaustive 15 | - funlen 16 | - goconst 17 | - gocritic 18 | - gocyclo 19 | - revive 20 | - gosimple 21 | - govet 22 | - gosec 23 | - ineffassign 24 | - lll 25 | - misspell 26 | - nakedret 27 | - gofumpt 28 | - nolintlint 29 | - staticcheck 30 | - stylecheck 31 | - typecheck 32 | - unconvert 33 | - unparam 34 | - unused 35 | - whitespace 36 | 37 | issues: 38 | # Excluding configuration per-path, per-linter, per-text and per-source 39 | exclude-rules: 40 | - path: internal/kafka/ 41 | text: "dot-imports" 42 | linters: 43 | - revive 44 | - linters: 45 | - stylecheck 46 | text: "ST1001:" 47 | - path: _test\.go 48 | linters: 49 | - errcheck 50 | - funlen 51 | 52 | service: 53 | golangci-lint-version: 1.51.x # use the fixed version to not introduce new linters unexpectedly 54 | prepare: 55 | - echo "here I can run custom commands, but no preparation needed for this repo" -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: kafka-cronsumer 2 | release: 3 | github: 4 | name: kafka-cronsumer 5 | owner: Trendyol 6 | before: 7 | hooks: 8 | - go mod tidy 9 | builds: 10 | - skip: true 11 | changelog: 12 | sort: asc 13 | filters: 14 | exclude: 15 | - '^docs:' 16 | - '^test:' 17 | - '^chore:' -------------------------------------------------------------------------------- /CODE-OF-CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | sametileri07@gmail.com. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Trendyol 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ## help: print this help message 2 | help: 3 | @echo "Usage:" 4 | @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ":" | sed -e 's/^/ /' 5 | 6 | ## lint: runs golangci lint based on .golangci.yml configuration 7 | .PHONY: lint 8 | lint: 9 | @if ! test -f `go env GOPATH`/bin/golangci-lint; then go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.0; fi 10 | golangci-lint run -c .golangci.yml --fix -v 11 | 12 | ## test: runs tests 13 | .PHONY: test 14 | test: 15 | go test -v ./... -coverprofile=unit_coverage.out -short 16 | 17 | ## unit-coverage-html: extract unit tests coverage to html format 18 | .PHONY: unit-coverage-html 19 | unit-coverage-html: 20 | make test 21 | go tool cover -html=unit_coverage.out -o unit_coverage.html 22 | 23 | ## godoc: generate documentation 24 | .PHONY: godoc 25 | godoc: 26 | @if ! test -f `go env GOPATH`/bin/godoc; then go install golang.org/x/tools/cmd/godoc; fi 27 | godoc -http=127.0.0.1:6060 28 | 29 | ## produce: produce test message (requires jq and kafka-console-producer) 30 | ## : make produce topic=exception 31 | .PHONY: produce 32 | produce: 33 | jq -rc . ./testdata/message.json | kafka-console-producer --bootstrap-server 127.0.0.1:9092 --topic ${topic} 34 | 35 | # default value 36 | export topic=exception 37 | ## produce-with-header: produce test message with retry header (requires jq and kcat) 38 | ## : make produce-with-header topic=exception 39 | .PHONY: produce-with-header 40 | produce-with-header: 41 | jq -rc . ./testdata/message.json | kcat -b 127.0.0.1:9092 -t ${topic} -P -H x-retry-count=1 42 | 43 | 44 | .PHONY: integration-compose 45 | integration-compose: 46 | docker compose -f test/integration/docker-compose.yml up --wait --build --force-recreate --remove-orphans 47 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Security updates are applied only to the most recent releases. 6 | 7 | ## Reporting a Vulnerability 8 | 9 | To securely report a vulnerability, please [submit a new issue on GitHub](https://github.com/Trendyol/kafka-cronsumer/issues/new). 10 | 11 | ## Vulnerability Process 12 | 13 | 1. Your report will be acknowledged as soon as possible. 14 | 2. The team will investigate and update the issue with relevant information. 15 | 3. If the team does not confirm the report, no further action will be taken and the issue will be closed. 16 | 4. If the team confirms the report, the team will take action to fix it immediately: 17 | 1. Commits will be handled in a private repository for review and testing. 18 | 2. Release a new patch version from the private repository. 19 | 3. Write a report disclosing the vulnerability. 20 | -------------------------------------------------------------------------------- /backoff-strategy-structure.md: -------------------------------------------------------------------------------- 1 | ## How BackOff Strategy Works 2 | 3 | ![How BackOff Strategy Works](.github/images/backoff_strategy_architecture.png) 4 | 5 | **The users can configure consumer easily like this;** 6 | 7 | `... 8 | 9 | cron: "*/1 * * * *" 10 | backOffStrategy: "exponential" 11 | duration: 1m 12 | concurrency: 1 13 | ` 14 | 15 | # Backoff Strategies Descriptions 16 | 17 | 1. **Linear Backoff:** It involves linearly increasing the time between each retry attempt. (1, 2, 3, 4..) 18 | 2. **Exponential Backoff:** It involves exponentially increasing the time between each retry attempt. (1, 2, 4, 8..) 19 | 3. **Fixed Backoff:** In this strategy, the delay between retry attempts remains constant. This strategy is simple to implement but may not be suitable for all scenarios. 20 | 21 | 22 | -------------------------------------------------------------------------------- /cronsumer.go: -------------------------------------------------------------------------------- 1 | // Package cronsumer This package implements a topic management strategy which consumes messages with cron based manner. 2 | // It mainly created for exception/retry management. 3 | package cronsumer 4 | 5 | import ( 6 | "github.com/Trendyol/kafka-cronsumer/internal" 7 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 8 | "github.com/Trendyol/kafka-cronsumer/pkg/logger" 9 | ) 10 | 11 | // New returns the newly created kafka consumer instance. 12 | // config.Config specifies cron, duration and so many parameters. 13 | // ConsumeFn describes how to consume messages from specified topic. 14 | func New(cfg *kafka.Config, c kafka.ConsumeFn) kafka.Cronsumer { 15 | cfg.Logger = logger.New(cfg.LogLevel) 16 | verifyTopicOnStartup(cfg) 17 | cfg.Logger.Infof("Topic [%s] verified successfully!", cfg.Consumer.Topic) 18 | return internal.NewCronsumer(cfg, c) 19 | } 20 | 21 | func verifyTopicOnStartup(cfg *kafka.Config) { 22 | kclient, err := internal.NewKafkaClient(cfg) 23 | if err != nil { 24 | panic("panic when initializing kafka client for verify topic error: " + err.Error()) 25 | } 26 | exist, err := internal.VerifyTopics(kclient, cfg.Consumer.Topic) 27 | if err != nil { 28 | panic("panic " + err.Error()) 29 | } 30 | if !exist { 31 | panic("topic: " + cfg.Consumer.Topic + " does not exist, please check cluster authority etc.") 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | services: 3 | zookeeper: 4 | image: debezium/zookeeper:2.6 5 | ports: 6 | - "2181:2181" 7 | 8 | kafka: 9 | image: confluentinc/cp-kafka:6.1.1 10 | depends_on: 11 | - zookeeper 12 | ports: 13 | - "9092:9092" 14 | - "29092:29092" 15 | environment: 16 | KAFKA_BROKER_ID: 1 17 | KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" 18 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 19 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 20 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 21 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 22 | KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 23 | KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 24 | 25 | kafka-ui: 26 | image: quay.io/cloudhut/kowl:master 27 | restart: on-failure 28 | depends_on: 29 | - zookeeper 30 | - kafka 31 | ports: 32 | - 8080:8080 33 | environment: 34 | KAFKA_BROKERS: kafka:9092 35 | 36 | kafka-create-topics: 37 | image: confluentinc/cp-kafka:6.1.1 38 | restart: on-failure 39 | depends_on: 40 | - zookeeper 41 | - kafka 42 | command: "bash -c 'echo Waiting for Kafka to be ready... && \ 43 | cub kafka-ready -b kafka:9092 1 20 && \ 44 | kafka-topics --create --topic exception --if-not-exists --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 && \ 45 | kafka-topics --create --topic exception-1 --if-not-exists --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 && \ 46 | kafka-topics --create --topic exception-2 --if-not-exists --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 && \ 47 | kafka-topics --create --topic dead-letter --if-not-exists --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 && \ 48 | sleep infinity'" 49 | environment: 50 | KAFKA_BROKER_ID: ignored 51 | KAFKA_ZOOKEEPER_CONNECT: ignored -------------------------------------------------------------------------------- /examples/multiple-consumer/go.mod: -------------------------------------------------------------------------------- 1 | module multiple-consumer 2 | 3 | go 1.19 4 | 5 | replace github.com/Trendyol/kafka-cronsumer => ../.. 6 | 7 | require github.com/Trendyol/kafka-cronsumer v0.0.0-00010101000000-000000000000 8 | 9 | require ( 10 | github.com/klauspost/compress v1.16.4 // indirect 11 | github.com/pierrec/lz4/v4 v4.1.17 // indirect 12 | github.com/robfig/cron/v3 v3.0.1 // indirect 13 | github.com/segmentio/kafka-go v0.4.42 // indirect 14 | github.com/xdg/scram v1.0.5 // indirect 15 | github.com/xdg/stringprep v1.0.3 // indirect 16 | go.uber.org/atomic v1.7.0 // indirect 17 | go.uber.org/multierr v1.6.0 // indirect 18 | go.uber.org/zap v1.24.0 // indirect 19 | golang.org/x/crypto v0.1.0 // indirect 20 | golang.org/x/text v0.7.0 // indirect 21 | ) 22 | -------------------------------------------------------------------------------- /examples/multiple-consumer/go.sum: -------------------------------------------------------------------------------- 1 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 6 | github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= 7 | github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 8 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 9 | github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= 10 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 11 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 12 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 13 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 14 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 15 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 16 | github.com/segmentio/kafka-go v0.4.39 h1:75smaomhvkYRwtuOwqLsdhgCG30B82NsbdkdDfFbvrw= 17 | github.com/segmentio/kafka-go v0.4.39/go.mod h1:T0MLgygYvmqmBvC+s8aCcbVNfJN4znVne5j0Pzowp/Q= 18 | github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= 19 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 20 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 21 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 22 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 23 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 24 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 25 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 26 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 27 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 28 | github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= 29 | github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= 30 | github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= 31 | github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= 32 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 33 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 34 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 35 | go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= 36 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 37 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 38 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 39 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 40 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 41 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 42 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 43 | golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= 44 | golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= 45 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 46 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 47 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 48 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 49 | golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 50 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 51 | golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= 52 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 53 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 54 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 55 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 56 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 57 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 58 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 59 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 60 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 61 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 62 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 63 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 64 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 65 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 66 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 67 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 68 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 69 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 70 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 71 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 72 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 73 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 74 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 75 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 76 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 77 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 78 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 79 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 80 | -------------------------------------------------------------------------------- /examples/multiple-consumer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | cronsumer "github.com/Trendyol/kafka-cronsumer" 6 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 7 | "time" 8 | ) 9 | 10 | func main() { 11 | firstCfg := &kafka.Config{ 12 | Brokers: []string{"localhost:29092"}, 13 | Consumer: kafka.ConsumerConfig{ 14 | GroupID: "sample-consumer-1", 15 | Topic: "exception-1", 16 | Cron: "*/1 * * * *", 17 | Duration: 20 * time.Second, 18 | }, 19 | LogLevel: "info", 20 | } 21 | var firstConsumerFn kafka.ConsumeFn = func(message kafka.Message) error { 22 | fmt.Printf("First consumer > Message received: %s\n", string(message.Value)) 23 | return nil 24 | } 25 | first := cronsumer.New(firstCfg, firstConsumerFn) 26 | first.Start() 27 | 28 | secondCfg := &kafka.Config{ 29 | Brokers: []string{"localhost:29092"}, 30 | Consumer: kafka.ConsumerConfig{ 31 | GroupID: "sample-consumer-2", 32 | Topic: "exception-2", 33 | Cron: "*/1 * * * *", 34 | Duration: 20 * time.Second, 35 | }, 36 | LogLevel: "info", 37 | } 38 | 39 | var secondConsumerFn kafka.ConsumeFn = func(message kafka.Message) error { 40 | fmt.Printf("Second consumer > Message received: %s\n", string(message.Value)) 41 | return nil 42 | } 43 | second := cronsumer.New(secondCfg, secondConsumerFn) 44 | second.Start() 45 | 46 | select {} // block main goroutine (we did to show it by on purpose) 47 | } 48 | -------------------------------------------------------------------------------- /examples/single-consumer-with-backoff-strategy/go.mod: -------------------------------------------------------------------------------- 1 | module single-consumer-with-producer 2 | 3 | go 1.19 4 | 5 | replace github.com/Trendyol/kafka-cronsumer => ../.. 6 | 7 | require github.com/Trendyol/kafka-cronsumer v0.0.0-00010101000000-000000000000 8 | 9 | require ( 10 | github.com/beorn7/perks v1.0.1 // indirect 11 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 12 | github.com/golang/protobuf v1.5.3 // indirect 13 | github.com/klauspost/compress v1.16.4 // indirect 14 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 15 | github.com/pierrec/lz4/v4 v4.1.17 // indirect 16 | github.com/prometheus/client_golang v1.16.0 // indirect 17 | github.com/prometheus/client_model v0.3.0 // indirect 18 | github.com/prometheus/common v0.42.0 // indirect 19 | github.com/prometheus/procfs v0.10.1 // indirect 20 | github.com/robfig/cron/v3 v3.0.1 // indirect 21 | github.com/segmentio/kafka-go v0.4.42 // indirect 22 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 23 | github.com/xdg-go/scram v1.1.2 // indirect 24 | github.com/xdg-go/stringprep v1.0.4 // indirect 25 | go.uber.org/atomic v1.7.0 // indirect 26 | go.uber.org/multierr v1.6.0 // indirect 27 | go.uber.org/zap v1.24.0 // indirect 28 | golang.org/x/sys v0.8.0 // indirect 29 | golang.org/x/text v0.7.0 // indirect 30 | google.golang.org/protobuf v1.30.0 // indirect 31 | ) 32 | -------------------------------------------------------------------------------- /examples/single-consumer-with-backoff-strategy/go.sum: -------------------------------------------------------------------------------- 1 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= 2 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 3 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 4 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 5 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 8 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 10 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 11 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 12 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 13 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 14 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 15 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 16 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 17 | github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= 18 | github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 19 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 20 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 21 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 22 | github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= 23 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 24 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 25 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 26 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 27 | github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= 28 | github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= 29 | github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= 30 | github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= 31 | github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= 32 | github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= 33 | github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= 34 | github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= 35 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 36 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 37 | github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= 38 | github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= 39 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 40 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 41 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 42 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 43 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 44 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 45 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 46 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 47 | github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= 48 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 49 | github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= 50 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 51 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 52 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 53 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 54 | go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= 55 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 56 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 57 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 58 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 59 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 60 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 61 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 62 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 63 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 64 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 65 | golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= 66 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 67 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 68 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 69 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 70 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 71 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 72 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 73 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 74 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 75 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 76 | golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= 77 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 78 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 79 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 80 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 81 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 82 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 83 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 84 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 85 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 86 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 87 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 88 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 89 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 90 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 91 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 92 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 93 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 94 | google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= 95 | google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 96 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 97 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 98 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 99 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 100 | -------------------------------------------------------------------------------- /examples/single-consumer-with-backoff-strategy/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | cronsumer "github.com/Trendyol/kafka-cronsumer" 6 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 7 | "strconv" 8 | "time" 9 | ) 10 | 11 | func main() { 12 | config := &kafka.Config{ 13 | Brokers: []string{"localhost:29092"}, 14 | Consumer: kafka.ConsumerConfig{ 15 | GroupID: "sample-consumer-with-producer", 16 | Topic: "exception", 17 | MaxRetry: 3, 18 | Cron: "*/1 * * * *", 19 | Duration: 20 * time.Second, 20 | BackOffStrategy: kafka.GetBackoffStrategy(kafka.ExponentialBackOffStrategy), 21 | }, 22 | LogLevel: "info", 23 | } 24 | 25 | var consumeFn kafka.ConsumeFn = func(message kafka.Message) error { 26 | fmt.Printf("consumer > Message received: %s\n", string(message.Value)) 27 | return nil 28 | } 29 | 30 | c := cronsumer.New(config, consumeFn) 31 | c.Start() 32 | 33 | produceTime := time.Now().UnixNano() 34 | produceTimeStr := strconv.FormatInt(produceTime, 10) 35 | 36 | firstMessageWithRetryAttempt := kafka.NewMessageBuilder(). 37 | WithHeaders([]kafka.Header{ 38 | {Key: "x-retry-count", Value: []byte("3")}, 39 | {Key: "x-retry-attempt-count", Value: []byte("6")}, 40 | {Key: "x-produce-time", Value: []byte(produceTimeStr)}, 41 | }). 42 | WithTopic(config.Consumer.Topic). 43 | WithKey(nil). 44 | WithValue([]byte(`{ "foo": "bar" }`)). 45 | Build() 46 | 47 | secondMessageWithRetryAttempt := kafka.NewMessageBuilder(). 48 | WithHeaders([]kafka.Header{ 49 | {Key: "x-retry-count", Value: []byte("3")}, 50 | {Key: "x-retry-attempt-count", Value: []byte("7")}, 51 | {Key: "x-produce-time", Value: []byte(produceTimeStr)}, 52 | }). 53 | WithTopic(config.Consumer.Topic). 54 | WithKey(nil). 55 | WithValue([]byte(`{ "foo2": "bar2" }`)). 56 | Build() 57 | 58 | c.ProduceBatch([]kafka.Message{firstMessageWithRetryAttempt, secondMessageWithRetryAttempt}) 59 | 60 | select {} // showing purpose 61 | } 62 | -------------------------------------------------------------------------------- /examples/single-consumer-with-custom-logger/go.mod: -------------------------------------------------------------------------------- 1 | module single-consumer-with-custom-logger 2 | 3 | go 1.19 4 | 5 | replace github.com/Trendyol/kafka-cronsumer => ../.. 6 | 7 | require github.com/Trendyol/kafka-cronsumer v0.0.0-00010101000000-000000000000 8 | 9 | require ( 10 | github.com/klauspost/compress v1.16.4 // indirect 11 | github.com/pierrec/lz4/v4 v4.1.17 // indirect 12 | github.com/robfig/cron/v3 v3.0.1 // indirect 13 | github.com/segmentio/kafka-go v0.4.42 // indirect 14 | github.com/xdg/scram v1.0.5 // indirect 15 | github.com/xdg/stringprep v1.0.3 // indirect 16 | go.uber.org/atomic v1.7.0 // indirect 17 | go.uber.org/multierr v1.6.0 // indirect 18 | go.uber.org/zap v1.24.0 // indirect 19 | golang.org/x/crypto v0.1.0 // indirect 20 | golang.org/x/text v0.7.0 // indirect 21 | ) 22 | -------------------------------------------------------------------------------- /examples/single-consumer-with-custom-logger/go.sum: -------------------------------------------------------------------------------- 1 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 6 | github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= 7 | github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 8 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 9 | github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= 10 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 11 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 12 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 13 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 14 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 15 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 16 | github.com/segmentio/kafka-go v0.4.39 h1:75smaomhvkYRwtuOwqLsdhgCG30B82NsbdkdDfFbvrw= 17 | github.com/segmentio/kafka-go v0.4.39/go.mod h1:T0MLgygYvmqmBvC+s8aCcbVNfJN4znVne5j0Pzowp/Q= 18 | github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= 19 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 20 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 21 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 22 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 23 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 24 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 25 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 26 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 27 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 28 | github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= 29 | github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= 30 | github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= 31 | github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= 32 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 33 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 34 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 35 | go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= 36 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 37 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 38 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 39 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 40 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 41 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 42 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 43 | golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= 44 | golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= 45 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 46 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 47 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 48 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 49 | golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 50 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 51 | golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= 52 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 53 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 54 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 55 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 56 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 57 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 58 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 59 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 60 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 61 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 62 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 63 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 64 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 65 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 66 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 67 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 68 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 69 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 70 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 71 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 72 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 73 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 74 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 75 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 76 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 77 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 78 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 79 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 80 | -------------------------------------------------------------------------------- /examples/single-consumer-with-custom-logger/logger.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/Trendyol/kafka-cronsumer/pkg/logger" 6 | ) 7 | 8 | type myLogger struct{} 9 | 10 | var _ logger.Interface = (*myLogger)(nil) 11 | 12 | func (m myLogger) With(args ...interface{}) logger.Interface { 13 | return m 14 | } 15 | 16 | func (m myLogger) Debug(args ...interface{}) { 17 | fmt.Println(args...) 18 | } 19 | 20 | func (m myLogger) Info(args ...interface{}) { 21 | fmt.Println(args...) 22 | } 23 | 24 | func (m myLogger) Warn(args ...interface{}) { 25 | fmt.Println(args...) 26 | } 27 | 28 | func (m myLogger) Error(args ...interface{}) { 29 | fmt.Println(args...) 30 | } 31 | 32 | func (m myLogger) Debugf(format string, args ...interface{}) { 33 | fmt.Println(args...) 34 | } 35 | 36 | func (m myLogger) Infof(format string, args ...interface{}) { 37 | fmt.Println(args...) 38 | } 39 | 40 | func (m myLogger) Warnf(format string, args ...interface{}) { 41 | fmt.Println(args...) 42 | } 43 | 44 | func (m myLogger) Errorf(format string, args ...interface{}) { 45 | fmt.Println(args...) 46 | } 47 | 48 | func (m myLogger) Infow(msg string, keysAndValues ...interface{}) { 49 | fmt.Println(msg) 50 | } 51 | 52 | func (m myLogger) Errorw(msg string, keysAndValues ...interface{}) { 53 | fmt.Println(msg) 54 | } 55 | 56 | func (m myLogger) Warnw(msg string, keysAndValues ...interface{}) { 57 | fmt.Println(msg) 58 | } 59 | -------------------------------------------------------------------------------- /examples/single-consumer-with-custom-logger/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | cronsumer "github.com/Trendyol/kafka-cronsumer" 6 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 7 | "time" 8 | ) 9 | 10 | func main() { 11 | config := &kafka.Config{ 12 | Brokers: []string{"localhost:29092"}, 13 | Consumer: kafka.ConsumerConfig{ 14 | GroupID: "sample-consumer-with-custom-logger", 15 | StartOffset: kafka.OffsetLatest, 16 | Topic: "exception", 17 | MaxRetry: 3, 18 | Cron: "*/1 * * * *", 19 | Duration: 20 * time.Second, 20 | }, 21 | LogLevel: "debug", 22 | } 23 | 24 | var consumeFn kafka.ConsumeFn = func(message kafka.Message) error { 25 | fmt.Printf("consumer > Message received: %s\n", string(message.Value)) 26 | return nil 27 | } 28 | 29 | c := cronsumer.New(config, consumeFn) 30 | c.WithLogger(&myLogger{}) 31 | c.Run() 32 | } 33 | -------------------------------------------------------------------------------- /examples/single-consumer-with-deadletter/go.mod: -------------------------------------------------------------------------------- 1 | module single-consumer-with-deadletter 2 | 3 | go 1.19 4 | 5 | replace github.com/Trendyol/kafka-cronsumer => ../.. 6 | 7 | require github.com/Trendyol/kafka-cronsumer v0.0.0-00010101000000-000000000000 8 | 9 | require ( 10 | github.com/klauspost/compress v1.16.4 // indirect 11 | github.com/pierrec/lz4/v4 v4.1.17 // indirect 12 | github.com/robfig/cron/v3 v3.0.1 // indirect 13 | github.com/segmentio/kafka-go v0.4.42 // indirect 14 | github.com/xdg/scram v1.0.5 // indirect 15 | github.com/xdg/stringprep v1.0.3 // indirect 16 | go.uber.org/atomic v1.7.0 // indirect 17 | go.uber.org/multierr v1.6.0 // indirect 18 | go.uber.org/zap v1.24.0 // indirect 19 | golang.org/x/crypto v0.1.0 // indirect 20 | golang.org/x/text v0.7.0 // indirect 21 | ) 22 | -------------------------------------------------------------------------------- /examples/single-consumer-with-deadletter/go.sum: -------------------------------------------------------------------------------- 1 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 6 | github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= 7 | github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 8 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 9 | github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= 10 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 11 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 12 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 13 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 14 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 15 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 16 | github.com/segmentio/kafka-go v0.4.39 h1:75smaomhvkYRwtuOwqLsdhgCG30B82NsbdkdDfFbvrw= 17 | github.com/segmentio/kafka-go v0.4.39/go.mod h1:T0MLgygYvmqmBvC+s8aCcbVNfJN4znVne5j0Pzowp/Q= 18 | github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= 19 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 20 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 21 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 22 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 23 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 24 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 25 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 26 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 27 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 28 | github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= 29 | github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= 30 | github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= 31 | github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= 32 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 33 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 34 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 35 | go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= 36 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 37 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 38 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 39 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 40 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 41 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 42 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 43 | golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= 44 | golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= 45 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 46 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 47 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 48 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 49 | golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 50 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 51 | golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= 52 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 53 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 54 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 55 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 56 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 57 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 58 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 59 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 60 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 61 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 62 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 63 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 64 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 65 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 66 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 67 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 68 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 69 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 70 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 71 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 72 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 73 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 74 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 75 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 76 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 77 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 78 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 79 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 80 | -------------------------------------------------------------------------------- /examples/single-consumer-with-deadletter/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | cronsumer "github.com/Trendyol/kafka-cronsumer" 7 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 8 | "time" 9 | ) 10 | 11 | func main() { 12 | config := &kafka.Config{ 13 | Brokers: []string{"localhost:29092"}, 14 | Consumer: kafka.ConsumerConfig{ 15 | GroupID: "sample-consumer-with-dead-letter", 16 | Topic: "exception", 17 | DeadLetterTopic: "dead-letter", 18 | MaxRetry: 1, 19 | Cron: "*/1 * * * *", 20 | Duration: 20 * time.Second, 21 | }, 22 | LogLevel: "info", 23 | } 24 | 25 | var consumeFn kafka.ConsumeFn = func(message kafka.Message) error { 26 | fmt.Printf("consumer > Message received: %s\n", string(message.Value)) 27 | return errors.New("error to show dead letter future") 28 | } 29 | 30 | c := cronsumer.New(config, consumeFn) 31 | c.Run() 32 | } 33 | -------------------------------------------------------------------------------- /examples/single-consumer-with-header-filter-function/go.mod: -------------------------------------------------------------------------------- 1 | module single-consumer 2 | 3 | go 1.19 4 | 5 | replace github.com/Trendyol/kafka-cronsumer => ../.. 6 | 7 | require github.com/Trendyol/kafka-cronsumer v0.0.0-00010101000000-000000000000 8 | 9 | require ( 10 | github.com/beorn7/perks v1.0.1 // indirect 11 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 12 | github.com/golang/protobuf v1.5.3 // indirect 13 | github.com/klauspost/compress v1.16.4 // indirect 14 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 15 | github.com/pierrec/lz4/v4 v4.1.17 // indirect 16 | github.com/prometheus/client_golang v1.16.0 // indirect 17 | github.com/prometheus/client_model v0.3.0 // indirect 18 | github.com/prometheus/common v0.42.0 // indirect 19 | github.com/prometheus/procfs v0.10.1 // indirect 20 | github.com/robfig/cron/v3 v3.0.1 // indirect 21 | github.com/segmentio/kafka-go v0.4.42 // indirect 22 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 23 | github.com/xdg-go/scram v1.1.2 // indirect 24 | github.com/xdg-go/stringprep v1.0.4 // indirect 25 | go.uber.org/atomic v1.7.0 // indirect 26 | go.uber.org/multierr v1.6.0 // indirect 27 | go.uber.org/zap v1.24.0 // indirect 28 | golang.org/x/sys v0.8.0 // indirect 29 | golang.org/x/text v0.7.0 // indirect 30 | google.golang.org/protobuf v1.30.0 // indirect 31 | ) 32 | -------------------------------------------------------------------------------- /examples/single-consumer-with-header-filter-function/go.sum: -------------------------------------------------------------------------------- 1 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 2 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 6 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 7 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 8 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 9 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 10 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 11 | github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 12 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 13 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 14 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 15 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 16 | github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= 17 | github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= 18 | github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= 19 | github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= 20 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 21 | github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= 22 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 23 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 24 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 25 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 26 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 27 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 28 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 29 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 30 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 31 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 32 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 33 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 34 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 35 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 36 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 37 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 38 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 39 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 40 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 41 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 42 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 43 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 44 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 45 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 46 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 47 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 48 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 49 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 50 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 51 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 52 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 53 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 54 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 55 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 56 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 57 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 58 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 59 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 60 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 61 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 62 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 63 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 64 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 65 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 66 | google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 67 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 68 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 69 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 70 | -------------------------------------------------------------------------------- /examples/single-consumer-with-header-filter-function/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | cronsumer "github.com/Trendyol/kafka-cronsumer" 6 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 7 | "time" 8 | ) 9 | 10 | func main() { 11 | config := &kafka.Config{ 12 | Brokers: []string{"localhost:29092"}, 13 | Consumer: kafka.ConsumerConfig{ 14 | GroupID: "sample-consumer", 15 | Topic: "exception", 16 | Cron: "*/1 * * * *", 17 | Duration: 20 * time.Second, 18 | SkipMessageByHeaderFn: SkipMessageByHeaderFn, 19 | }, 20 | LogLevel: "info", 21 | } 22 | 23 | var consumeFn kafka.ConsumeFn = func(message kafka.Message) error { 24 | fmt.Printf("consumer > Message received: %s\n", string(message.Value)) 25 | return nil 26 | } 27 | 28 | c := cronsumer.New(config, consumeFn) 29 | c.Run() 30 | } 31 | 32 | func SkipMessageByHeaderFn(headers []kafka.Header) bool { 33 | for _, header := range headers { 34 | if header.Key == "skipMessage" { 35 | // If a kafka message comes with `skipMessage` header key, it will be skipped! 36 | return true 37 | } 38 | } 39 | return false 40 | } 41 | -------------------------------------------------------------------------------- /examples/single-consumer-with-metric-collector/api.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 6 | "github.com/gofiber/fiber/v2" 7 | "github.com/prometheus/client_golang/prometheus" 8 | ) 9 | 10 | const port = 8090 11 | 12 | func StartAPI(cfg kafka.Config, metricCollectors ...prometheus.Collector) { 13 | f := fiber.New( 14 | fiber.Config{ 15 | DisableStartupMessage: true, 16 | DisableDefaultDate: true, 17 | DisableHeaderNormalizing: true, 18 | }, 19 | ) 20 | 21 | metricMiddleware, err := NewMetricMiddleware(cfg, f, metricCollectors...) 22 | 23 | if err == nil { 24 | f.Use(metricMiddleware) 25 | } else { 26 | fmt.Printf("metric middleware cannot be initialized: %v", err) 27 | } 28 | 29 | fmt.Printf("server starting on port %d", port) 30 | 31 | go listen(f) 32 | } 33 | 34 | func listen(f *fiber.App) { 35 | if err := f.Listen(fmt.Sprintf(":%d", port)); err != nil { 36 | fmt.Printf("server cannot start on port %d, err: %v", port, err) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /examples/single-consumer-with-metric-collector/go.mod: -------------------------------------------------------------------------------- 1 | module single-consumer-with-custom-logger 2 | 3 | go 1.19 4 | 5 | replace github.com/Trendyol/kafka-cronsumer => ../.. 6 | 7 | require ( 8 | github.com/Trendyol/kafka-cronsumer v0.0.0-00010101000000-000000000000 9 | github.com/ansrivas/fiberprometheus/v2 v2.6.1 10 | github.com/gofiber/fiber/v2 v2.48.0 11 | github.com/prometheus/client_golang v1.16.0 12 | ) 13 | 14 | require ( 15 | github.com/andybalholm/brotli v1.0.5 // indirect 16 | github.com/beorn7/perks v1.0.1 // indirect 17 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 18 | github.com/gofiber/adaptor/v2 v2.2.1 // indirect 19 | github.com/golang/protobuf v1.5.3 // indirect 20 | github.com/google/uuid v1.3.0 // indirect 21 | github.com/klauspost/compress v1.16.6 // indirect 22 | github.com/mattn/go-colorable v0.1.13 // indirect 23 | github.com/mattn/go-isatty v0.0.19 // indirect 24 | github.com/mattn/go-runewidth v0.0.14 // indirect 25 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 26 | github.com/pierrec/lz4/v4 v4.1.17 // indirect 27 | github.com/prometheus/client_model v0.4.0 // indirect 28 | github.com/prometheus/common v0.44.0 // indirect 29 | github.com/prometheus/procfs v0.11.0 // indirect 30 | github.com/rivo/uniseg v0.4.4 // indirect 31 | github.com/robfig/cron/v3 v3.0.1 // indirect 32 | github.com/segmentio/kafka-go v0.4.42 // indirect 33 | github.com/valyala/bytebufferpool v1.0.0 // indirect 34 | github.com/valyala/fasthttp v1.48.0 // indirect 35 | github.com/valyala/tcplisten v1.0.0 // indirect 36 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 37 | github.com/xdg-go/scram v1.1.2 // indirect 38 | github.com/xdg-go/stringprep v1.0.4 // indirect 39 | go.uber.org/atomic v1.7.0 // indirect 40 | go.uber.org/multierr v1.6.0 // indirect 41 | go.uber.org/zap v1.24.0 // indirect 42 | golang.org/x/sys v0.10.0 // indirect 43 | golang.org/x/text v0.9.0 // indirect 44 | google.golang.org/protobuf v1.30.0 // indirect 45 | ) 46 | -------------------------------------------------------------------------------- /examples/single-consumer-with-metric-collector/go.sum: -------------------------------------------------------------------------------- 1 | github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= 2 | github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= 3 | github.com/ansrivas/fiberprometheus/v2 v2.6.1 h1:wac3pXaE6BYYTF04AC6K0ktk6vCD+MnDOJZ3SK66kXM= 4 | github.com/ansrivas/fiberprometheus/v2 v2.6.1/go.mod h1:MloIKvy4yN6hVqlRpJ/jDiR244YnWJaQC0FIqS8A+MY= 5 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= 6 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 7 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 8 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 9 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 10 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 12 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/gofiber/adaptor/v2 v2.2.1 h1:givE7iViQWlsTR4Jh7tB4iXzrlKBgiraB/yTdHs9Lv4= 14 | github.com/gofiber/adaptor/v2 v2.2.1/go.mod h1:AhR16dEqs25W2FY/l8gSj1b51Azg5dtPDmm+pruNOrc= 15 | github.com/gofiber/fiber/v2 v2.48.0 h1:cRVMCb9aUJDsyHxGFLwz/sGzDggdailZZyptU9F9cU0= 16 | github.com/gofiber/fiber/v2 v2.48.0/go.mod h1:xqJgfqrc23FJuqGOW6DVgi3HyZEm2Mn9pRqUb2kHSX8= 17 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 18 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 19 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 20 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 21 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 22 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 23 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 24 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 25 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 26 | github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= 27 | github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 28 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 29 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 30 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 31 | github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= 32 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 33 | github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= 34 | github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 35 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 36 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 37 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 38 | github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= 39 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 40 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 41 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 42 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 43 | github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= 44 | github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= 45 | github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= 46 | github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= 47 | github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= 48 | github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= 49 | github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= 50 | github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= 51 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 52 | github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= 53 | github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 54 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 55 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 56 | github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= 57 | github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= 58 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 59 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 60 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 61 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 62 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 63 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 64 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 65 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 66 | github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= 67 | github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= 68 | github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= 69 | github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= 70 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 71 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 72 | github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= 73 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 74 | github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= 75 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 76 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 77 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 78 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 79 | go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= 80 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 81 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 82 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 83 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 84 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 85 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 86 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 87 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 88 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 89 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 90 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 91 | golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= 92 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 93 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 94 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 95 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 96 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 97 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 98 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 99 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 100 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 101 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 102 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 103 | golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= 104 | golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 105 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 106 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 107 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 108 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 109 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 110 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 111 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 112 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 113 | golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= 114 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 115 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 116 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 117 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 118 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 119 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 120 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 121 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 122 | google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= 123 | google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 124 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 125 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 126 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 127 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 128 | -------------------------------------------------------------------------------- /examples/single-consumer-with-metric-collector/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | cronsumer "github.com/Trendyol/kafka-cronsumer" 7 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 8 | "time" 9 | ) 10 | 11 | func main() { 12 | config := &kafka.Config{ 13 | Brokers: []string{"localhost:29092"}, 14 | Consumer: kafka.ConsumerConfig{ 15 | GroupID: "sample-consumer-with-metric-collector", 16 | StartOffset: kafka.OffsetLatest, 17 | Topic: "exception", 18 | MaxRetry: 3, 19 | Cron: "*/1 * * * *", 20 | Duration: 20 * time.Second, 21 | }, 22 | LogLevel: "debug", 23 | } 24 | 25 | var consumeFn kafka.ConsumeFn = func(message kafka.Message) error { 26 | fmt.Printf("consumer > Message received: %s\n", string(message.Value)) 27 | return errors.New("err occurred") 28 | } 29 | 30 | c := cronsumer.New(config, consumeFn) 31 | StartAPI(*config, c.GetMetricCollectors()...) 32 | c.Run() 33 | } 34 | -------------------------------------------------------------------------------- /examples/single-consumer-with-metric-collector/metric.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 5 | "github.com/ansrivas/fiberprometheus/v2" 6 | "github.com/gofiber/fiber/v2" 7 | "github.com/prometheus/client_golang/prometheus" 8 | ) 9 | 10 | func NewMetricMiddleware(cfg kafka.Config, app *fiber.App, metricCollectors ...prometheus.Collector) (func(ctx *fiber.Ctx) error, error) { 11 | prometheus.DefaultRegisterer.MustRegister(metricCollectors...) 12 | 13 | fiberPrometheus := fiberprometheus.New(cfg.Consumer.GroupID) 14 | fiberPrometheus.RegisterAt(app, "/metrics") 15 | 16 | return fiberPrometheus.Middleware, nil 17 | } 18 | -------------------------------------------------------------------------------- /examples/single-consumer-with-producer/go.mod: -------------------------------------------------------------------------------- 1 | module single-consumer-with-producer 2 | 3 | go 1.19 4 | 5 | replace github.com/Trendyol/kafka-cronsumer => ../.. 6 | 7 | require github.com/Trendyol/kafka-cronsumer v0.0.0-00010101000000-000000000000 8 | 9 | require ( 10 | github.com/klauspost/compress v1.16.4 // indirect 11 | github.com/pierrec/lz4/v4 v4.1.17 // indirect 12 | github.com/robfig/cron/v3 v3.0.1 // indirect 13 | github.com/segmentio/kafka-go v0.4.42 // indirect 14 | github.com/xdg/scram v1.0.5 // indirect 15 | github.com/xdg/stringprep v1.0.3 // indirect 16 | go.uber.org/atomic v1.7.0 // indirect 17 | go.uber.org/multierr v1.6.0 // indirect 18 | go.uber.org/zap v1.24.0 // indirect 19 | golang.org/x/crypto v0.1.0 // indirect 20 | golang.org/x/text v0.7.0 // indirect 21 | ) 22 | -------------------------------------------------------------------------------- /examples/single-consumer-with-producer/go.sum: -------------------------------------------------------------------------------- 1 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 6 | github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= 7 | github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 8 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 9 | github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= 10 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 11 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 12 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 13 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 14 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 15 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 16 | github.com/segmentio/kafka-go v0.4.39 h1:75smaomhvkYRwtuOwqLsdhgCG30B82NsbdkdDfFbvrw= 17 | github.com/segmentio/kafka-go v0.4.39/go.mod h1:T0MLgygYvmqmBvC+s8aCcbVNfJN4znVne5j0Pzowp/Q= 18 | github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= 19 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 20 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 21 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 22 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 23 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 24 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 25 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 26 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 27 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 28 | github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= 29 | github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= 30 | github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= 31 | github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= 32 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 33 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 34 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 35 | go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= 36 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 37 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 38 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 39 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 40 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 41 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 42 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 43 | golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= 44 | golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= 45 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 46 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 47 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 48 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 49 | golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 50 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 51 | golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= 52 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 53 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 54 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 55 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 56 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 57 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 58 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 59 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 60 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 61 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 62 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 63 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 64 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 65 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 66 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 67 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 68 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 69 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 70 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 71 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 72 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 73 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 74 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 75 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 76 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 77 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 78 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 79 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 80 | -------------------------------------------------------------------------------- /examples/single-consumer-with-producer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | cronsumer "github.com/Trendyol/kafka-cronsumer" 6 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 7 | "time" 8 | ) 9 | 10 | func main() { 11 | config := &kafka.Config{ 12 | Brokers: []string{"localhost:29092"}, 13 | Consumer: kafka.ConsumerConfig{ 14 | GroupID: "sample-consumer-with-producer", 15 | Topic: "exception", 16 | MaxRetry: 3, 17 | Cron: "*/1 * * * *", 18 | Duration: 20 * time.Second, 19 | }, 20 | LogLevel: "info", 21 | } 22 | 23 | var consumeFn kafka.ConsumeFn = func(message kafka.Message) error { 24 | fmt.Printf("consumer > Message received: %s\n", string(message.Value)) 25 | return nil 26 | } 27 | 28 | c := cronsumer.New(config, consumeFn) 29 | c.Start() 30 | 31 | // If we want to produce a message to exception topic 32 | message := kafka.NewMessageBuilder(). 33 | WithTopic(config.Consumer.Topic). 34 | WithKey(nil). 35 | WithValue([]byte(`{ "foo": "bar" }`)). 36 | Build() 37 | c.Produce(message) 38 | 39 | // If we want to produce list of messages as batch 40 | c.ProduceBatch([]kafka.Message{ 41 | {Topic: config.Consumer.Topic, Value: []byte(`{ "foo": "bar1" }`)}, 42 | {Topic: config.Consumer.Topic, Value: []byte(`{ "foo": "bar2" }`)}, 43 | {Topic: config.Consumer.Topic, Value: []byte(`{ "foo": "bar3" }`)}, 44 | {Topic: config.Consumer.Topic, Value: []byte(`{ "foo": "bar4" }`)}, 45 | {Topic: config.Consumer.Topic, Value: []byte(`{ "foo": "bar5" }`)}, 46 | }) 47 | 48 | select {} // showing purpose 49 | } 50 | -------------------------------------------------------------------------------- /examples/single-consumer/go.mod: -------------------------------------------------------------------------------- 1 | module single-consumer 2 | 3 | go 1.19 4 | 5 | replace github.com/Trendyol/kafka-cronsumer => ../.. 6 | 7 | require github.com/Trendyol/kafka-cronsumer v0.0.0-00010101000000-000000000000 8 | 9 | require ( 10 | github.com/beorn7/perks v1.0.1 // indirect 11 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 12 | github.com/golang/protobuf v1.5.3 // indirect 13 | github.com/klauspost/compress v1.16.4 // indirect 14 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 15 | github.com/pierrec/lz4/v4 v4.1.17 // indirect 16 | github.com/prometheus/client_golang v1.16.0 // indirect 17 | github.com/prometheus/client_model v0.3.0 // indirect 18 | github.com/prometheus/common v0.42.0 // indirect 19 | github.com/prometheus/procfs v0.10.1 // indirect 20 | github.com/robfig/cron/v3 v3.0.1 // indirect 21 | github.com/segmentio/kafka-go v0.4.42 // indirect 22 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 23 | github.com/xdg-go/scram v1.1.2 // indirect 24 | github.com/xdg-go/stringprep v1.0.4 // indirect 25 | go.uber.org/atomic v1.7.0 // indirect 26 | go.uber.org/multierr v1.6.0 // indirect 27 | go.uber.org/zap v1.24.0 // indirect 28 | golang.org/x/sys v0.8.0 // indirect 29 | golang.org/x/text v0.7.0 // indirect 30 | google.golang.org/protobuf v1.30.0 // indirect 31 | ) 32 | -------------------------------------------------------------------------------- /examples/single-consumer/go.sum: -------------------------------------------------------------------------------- 1 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= 2 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 3 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 4 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 5 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 8 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 10 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 11 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 12 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 13 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 14 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 15 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 16 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 17 | github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= 18 | github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 19 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 20 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 21 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 22 | github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= 23 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 24 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 25 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 26 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 27 | github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= 28 | github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= 29 | github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= 30 | github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= 31 | github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= 32 | github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= 33 | github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= 34 | github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= 35 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 36 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 37 | github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= 38 | github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= 39 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 40 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 41 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 42 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 43 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 44 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 45 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 46 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 47 | github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= 48 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 49 | github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= 50 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 51 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 52 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 53 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 54 | go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= 55 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 56 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 57 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 58 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 59 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 60 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 61 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 62 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 63 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 64 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 65 | golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= 66 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 67 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 68 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 69 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 70 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 71 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 72 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 73 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 74 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 75 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 76 | golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= 77 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 78 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 79 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 80 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 81 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 82 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 83 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 84 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 85 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 86 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 87 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 88 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 89 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 90 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 91 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 92 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 93 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 94 | google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= 95 | google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 96 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 97 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 98 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 99 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 100 | -------------------------------------------------------------------------------- /examples/single-consumer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | cronsumer "github.com/Trendyol/kafka-cronsumer" 6 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 7 | "time" 8 | ) 9 | 10 | func main() { 11 | config := &kafka.Config{ 12 | Brokers: []string{"localhost:29092"}, 13 | Consumer: kafka.ConsumerConfig{ 14 | GroupID: "sample-consumer", 15 | Topic: "exception-not-exist", 16 | Cron: "*/1 * * * *", 17 | Duration: 20 * time.Second, 18 | }, 19 | LogLevel: "info", 20 | } 21 | 22 | var consumeFn kafka.ConsumeFn = func(message kafka.Message) error { 23 | fmt.Printf("consumer > Message received: %s\n", string(message.Value)) 24 | return nil 25 | } 26 | 27 | c := cronsumer.New(config, consumeFn) 28 | c.Run() 29 | } 30 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Trendyol/kafka-cronsumer 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/prometheus/client_golang v1.16.0 7 | github.com/robfig/cron/v3 v3.0.1 8 | github.com/segmentio/kafka-go v0.4.42 9 | go.uber.org/zap v1.24.0 10 | ) 11 | 12 | require ( 13 | github.com/beorn7/perks v1.0.1 // indirect 14 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 15 | github.com/golang/protobuf v1.5.3 // indirect 16 | github.com/klauspost/compress v1.16.4 // indirect 17 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 18 | github.com/pierrec/lz4/v4 v4.1.17 // indirect 19 | github.com/pkg/errors v0.9.1 // indirect 20 | github.com/prometheus/client_model v0.3.0 // indirect 21 | github.com/prometheus/common v0.42.0 // indirect 22 | github.com/prometheus/procfs v0.10.1 // indirect 23 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 24 | github.com/xdg-go/scram v1.1.2 // indirect 25 | github.com/xdg-go/stringprep v1.0.4 // indirect 26 | go.uber.org/atomic v1.7.0 // indirect 27 | go.uber.org/goleak v1.1.12 // indirect 28 | go.uber.org/multierr v1.6.0 // indirect 29 | golang.org/x/sys v0.8.0 // indirect 30 | golang.org/x/text v0.7.0 // indirect 31 | google.golang.org/protobuf v1.30.0 // indirect 32 | ) 33 | 34 | replace golang.org/x/crypto => golang.org/x/crypto v0.6.0 35 | 36 | replace golang.org/x/net => golang.org/x/net v0.7.0 37 | 38 | replace github.com/stretchr/testify => github.com/stretchr/testify v1.8.0 39 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= 2 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 3 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 4 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 5 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 9 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 10 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 11 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 12 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 13 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 14 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 15 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 16 | github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= 17 | github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 18 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 19 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 20 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 21 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 22 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 23 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 24 | github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= 25 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 26 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 27 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 28 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 29 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 30 | github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= 31 | github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= 32 | github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= 33 | github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= 34 | github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= 35 | github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= 36 | github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= 37 | github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= 38 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 39 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 40 | github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= 41 | github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= 42 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 43 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 44 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 45 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 46 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 47 | github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= 48 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 49 | github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= 50 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 51 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 52 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 53 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 54 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 55 | go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= 56 | go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= 57 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 58 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 59 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 60 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 61 | golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= 62 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 63 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 64 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 65 | golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= 66 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 67 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 68 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 69 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 70 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 71 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 72 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 73 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 74 | golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= 75 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 76 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 77 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 78 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 79 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 80 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 81 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 82 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 83 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 84 | golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 85 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 86 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 87 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 88 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 89 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 90 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 91 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 92 | google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= 93 | google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 94 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 95 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 96 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 97 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 98 | -------------------------------------------------------------------------------- /internal/collector.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import "github.com/prometheus/client_golang/prometheus" 4 | 5 | type Collector struct { 6 | cronsumerMetric *CronsumerMetric 7 | 8 | totalRetriedMessagesCounter *prometheus.Desc 9 | totalDiscardedMessagesCounter *prometheus.Desc 10 | } 11 | 12 | func NewCollector(metricPrefix string, cronsumerMetric *CronsumerMetric) *Collector { 13 | if metricPrefix == "" { 14 | metricPrefix = Name 15 | } 16 | 17 | return &Collector{ 18 | cronsumerMetric: cronsumerMetric, 19 | 20 | totalRetriedMessagesCounter: prometheus.NewDesc( 21 | prometheus.BuildFQName(metricPrefix, "retried_messages_total", "current"), 22 | "Total number of retried messages.", 23 | []string{}, 24 | nil, 25 | ), 26 | totalDiscardedMessagesCounter: prometheus.NewDesc( 27 | prometheus.BuildFQName(metricPrefix, "discarded_messages_total", "current"), 28 | "Total number of discarded messages.", 29 | []string{}, 30 | nil, 31 | ), 32 | } 33 | } 34 | 35 | func (s *Collector) Describe(ch chan<- *prometheus.Desc) { 36 | prometheus.DescribeByCollect(s, ch) 37 | } 38 | 39 | func (s *Collector) Collect(ch chan<- prometheus.Metric) { 40 | ch <- prometheus.MustNewConstMetric( 41 | s.totalRetriedMessagesCounter, 42 | prometheus.CounterValue, 43 | float64(s.cronsumerMetric.TotalRetriedMessagesCounter), 44 | []string{}..., 45 | ) 46 | 47 | ch <- prometheus.MustNewConstMetric( 48 | s.totalDiscardedMessagesCounter, 49 | prometheus.CounterValue, 50 | float64(s.cronsumerMetric.TotalDiscardedMessagesCounter), 51 | []string{}..., 52 | ) 53 | } 54 | -------------------------------------------------------------------------------- /internal/collector_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/prometheus/client_golang/prometheus" 8 | ) 9 | 10 | func Test_NewCollector(t *testing.T) { 11 | t.Run("Should_Register_Metrics_Have_Default_Name", func(t *testing.T) { 12 | cronsumerMetric := &CronsumerMetric{ 13 | TotalRetriedMessagesCounter: 0, 14 | TotalDiscardedMessagesCounter: 0, 15 | } 16 | expectedTotalRetriedMessagesCounter := prometheus.NewDesc( 17 | prometheus.BuildFQName(Name, "retried_messages_total", "current"), 18 | "Total number of retried messages.", 19 | []string{}, 20 | nil, 21 | ) 22 | expectedTotalDiscardedMessagesCounter := prometheus.NewDesc( 23 | prometheus.BuildFQName(Name, "discarded_messages_total", "current"), 24 | "Total number of discarded messages.", 25 | []string{}, 26 | nil, 27 | ) 28 | 29 | collector := NewCollector("", cronsumerMetric) 30 | 31 | if !reflect.DeepEqual(collector.totalDiscardedMessagesCounter, expectedTotalDiscardedMessagesCounter) { 32 | t.Errorf("Expected: %+v, Actual: %+v", collector.totalDiscardedMessagesCounter, expectedTotalDiscardedMessagesCounter) 33 | } 34 | if !reflect.DeepEqual(collector.totalRetriedMessagesCounter, expectedTotalRetriedMessagesCounter) { 35 | t.Errorf("Expected: %+v, Actual: %+v", collector.totalRetriedMessagesCounter, expectedTotalRetriedMessagesCounter) 36 | } 37 | }) 38 | t.Run("Should_Register_Metrics_Have_Specified_Name", func(t *testing.T) { 39 | cronsumerMetric := &CronsumerMetric{ 40 | TotalRetriedMessagesCounter: 0, 41 | TotalDiscardedMessagesCounter: 0, 42 | } 43 | expectedTotalRetriedMessagesCounter := prometheus.NewDesc( 44 | prometheus.BuildFQName("custom_prefix", "retried_messages_total", "current"), 45 | "Total number of retried messages.", 46 | []string{}, 47 | nil, 48 | ) 49 | expectedTotalDiscardedMessagesCounter := prometheus.NewDesc( 50 | prometheus.BuildFQName("custom_prefix", "discarded_messages_total", "current"), 51 | "Total number of discarded messages.", 52 | []string{}, 53 | nil, 54 | ) 55 | 56 | collector := NewCollector("custom_prefix", cronsumerMetric) 57 | 58 | if !reflect.DeepEqual(collector.totalDiscardedMessagesCounter, expectedTotalDiscardedMessagesCounter) { 59 | t.Errorf("Expected: %+v, Actual: %+v", collector.totalDiscardedMessagesCounter, expectedTotalDiscardedMessagesCounter) 60 | } 61 | if !reflect.DeepEqual(collector.totalRetriedMessagesCounter, expectedTotalRetriedMessagesCounter) { 62 | t.Errorf("Expected: %+v, Actual: %+v", collector.totalRetriedMessagesCounter, expectedTotalRetriedMessagesCounter) 63 | } 64 | }) 65 | } 66 | -------------------------------------------------------------------------------- /internal/consumer.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 8 | segmentio "github.com/segmentio/kafka-go" 9 | ) 10 | 11 | type Consumer interface { 12 | ReadMessage(ctx context.Context) (*segmentio.Message, error) 13 | Stop() 14 | } 15 | 16 | type kafkaConsumer struct { 17 | consumer *segmentio.Reader 18 | cfg *kafka.Config 19 | } 20 | 21 | func newConsumer(kafkaConfig *kafka.Config) *kafkaConsumer { 22 | readerConfig := segmentio.ReaderConfig{ 23 | Brokers: kafkaConfig.Brokers, 24 | GroupID: kafkaConfig.Consumer.GroupID, 25 | GroupTopics: []string{kafkaConfig.Consumer.Topic}, 26 | MinBytes: kafkaConfig.Consumer.MinBytes, 27 | MaxBytes: kafkaConfig.Consumer.MaxBytes, 28 | MaxWait: kafkaConfig.Consumer.MaxWait, 29 | CommitInterval: kafkaConfig.Consumer.CommitInterval, 30 | HeartbeatInterval: kafkaConfig.Consumer.HeartbeatInterval, 31 | SessionTimeout: kafkaConfig.Consumer.SessionTimeout, 32 | RebalanceTimeout: kafkaConfig.Consumer.RebalanceTimeout, 33 | StartOffset: kafkaConfig.Consumer.StartOffset.Value(), 34 | RetentionTime: kafkaConfig.Consumer.RetentionTime, 35 | QueueCapacity: kafkaConfig.Consumer.QueueCapacity, 36 | } 37 | 38 | readerConfig.Dialer = &segmentio.Dialer{ 39 | ClientID: kafkaConfig.Consumer.ClientID, 40 | } 41 | 42 | if kafkaConfig.SASL.Enabled { 43 | readerConfig.Dialer.TLS = NewTLSConfig(kafkaConfig) 44 | readerConfig.Dialer.SASLMechanism = Mechanism(kafkaConfig.SASL) 45 | 46 | if kafkaConfig.SASL.Rack != "" { 47 | readerConfig.GroupBalancers = []segmentio.GroupBalancer{segmentio.RackAffinityGroupBalancer{Rack: kafkaConfig.SASL.Rack}} 48 | } 49 | } 50 | 51 | return &kafkaConsumer{ 52 | consumer: segmentio.NewReader(readerConfig), 53 | cfg: kafkaConfig, 54 | } 55 | } 56 | 57 | func (k kafkaConsumer) ReadMessage(ctx context.Context) (*segmentio.Message, error) { 58 | msg, err := k.consumer.ReadMessage(ctx) 59 | if err != nil { 60 | if isContextCancelled(err) { 61 | k.cfg.Logger.Debug("kafka-go context is cancelled") 62 | return nil, nil 63 | } 64 | return nil, err 65 | } 66 | 67 | return &msg, nil 68 | } 69 | 70 | func isContextCancelled(err error) bool { 71 | return errors.Is(err, context.Canceled) 72 | } 73 | 74 | func (k kafkaConsumer) Stop() { 75 | if err := k.consumer.Close(); err != nil { 76 | k.cfg.Logger.Errorf("Error while closing kafka consumer %v", err) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /internal/cronsumer.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 8 | ) 9 | 10 | type cronsumer struct { 11 | messageChannel chan MessageWrapper 12 | 13 | kafkaConsumer Consumer 14 | kafkaProducer Producer 15 | 16 | consumeFn func(message kafka.Message) error 17 | 18 | metric *CronsumerMetric 19 | maxRetry int 20 | deadLetterTopic string 21 | skipMessageByHeaderFn kafka.SkipMessageByHeaderFn 22 | 23 | cfg *kafka.Config 24 | } 25 | 26 | func newCronsumer(cfg *kafka.Config, c func(message kafka.Message) error) *cronsumer { 27 | cfg.SetDefaults() 28 | cfg.Validate() 29 | 30 | return &cronsumer{ 31 | cfg: cfg, 32 | messageChannel: make(chan MessageWrapper), 33 | kafkaConsumer: newConsumer(cfg), 34 | kafkaProducer: newProducer(cfg), 35 | consumeFn: c, 36 | skipMessageByHeaderFn: cfg.Consumer.SkipMessageByHeaderFn, 37 | metric: &CronsumerMetric{}, 38 | maxRetry: cfg.Consumer.MaxRetry, 39 | deadLetterTopic: cfg.Consumer.DeadLetterTopic, 40 | } 41 | } 42 | 43 | func (k *cronsumer) SetupConcurrentWorkers(concurrency int) { 44 | for i := 0; i < concurrency; i++ { 45 | go k.processMessage() 46 | } 47 | } 48 | 49 | func (k *cronsumer) Listen(ctx context.Context, strategyName string, cancelFuncWrapper *func()) { 50 | startTime := time.Now() 51 | startTimeUnixNano := startTime.UnixNano() 52 | 53 | retryStrategy := kafka.GetBackoffStrategy(strategyName) 54 | 55 | for { 56 | m, err := k.kafkaConsumer.ReadMessage(ctx) 57 | if err != nil { 58 | //nolint:lll 59 | k.cfg.Logger.Warnf("Message from %s could not read with consumer group %s, error %s", k.cfg.Consumer.Topic, k.cfg.Consumer.GroupID, err.Error()) 60 | return 61 | } 62 | if m == nil { 63 | return 64 | } 65 | 66 | msg := NewMessageWrapper(*m, strategyName) 67 | 68 | if k.skipMessageByHeaderFn != nil && k.skipMessageByHeaderFn(msg.Headers) { 69 | k.cfg.Logger.Debugf("Message from %s is not processed. Header filter applied. Headers: %v", k.cfg.Consumer.Topic, msg.Headers.Pretty()) 70 | continue 71 | } 72 | 73 | if msg.ProduceTime >= startTimeUnixNano { 74 | (*cancelFuncWrapper)() 75 | 76 | k.cfg.Logger.Infof("Next iteration message from topic %s has been detected, resending the message to exception", msg.Topic) 77 | 78 | if err = k.kafkaProducer.ProduceWithRetryOption(*msg, false, false); err != nil { 79 | k.cfg.Logger.Errorf("Error %s sending next iteration KafkaMessage: %#v", err.Error(), *msg) 80 | } 81 | 82 | return 83 | } 84 | 85 | if retryStrategy.String() == kafka.FixedBackOffStrategy { 86 | k.sendToMessageChannel(*msg) 87 | continue 88 | } 89 | 90 | if retryStrategy != nil && retryStrategy.ShouldIncreaseRetryAttemptCount(msg.RetryCount, msg.RetryAttemptCount) { 91 | k.cfg.Logger.Infof( 92 | "Message not processed cause of %s backoff strategy retryCount: %d retryAttempt %d", 93 | strategyName, msg.RetryCount, msg.RetryAttemptCount, 94 | ) 95 | 96 | if err = k.kafkaProducer.ProduceWithRetryOption(*msg, false, true); err != nil { 97 | k.cfg.Logger.Errorf("Error %s sending next iteration KafkaMessage: %#v", err.Error(), *msg) 98 | } 99 | } else { 100 | k.sendToMessageChannel(*msg) 101 | } 102 | } 103 | } 104 | 105 | func (k *cronsumer) Stop() { 106 | close(k.messageChannel) 107 | k.kafkaConsumer.Stop() 108 | k.kafkaProducer.Close() 109 | } 110 | 111 | func (k *cronsumer) GetMetric() *CronsumerMetric { 112 | return k.metric 113 | } 114 | 115 | func (k *cronsumer) processMessage() { 116 | for msg := range k.messageChannel { 117 | if err := k.consumeFn(msg.Message); err != nil { 118 | msg.AddHeader(createErrHeader(err)) 119 | k.produce(msg) 120 | } 121 | } 122 | } 123 | 124 | func (k *cronsumer) sendToMessageChannel(msg MessageWrapper) { 125 | defer k.recoverMessage(msg) 126 | k.messageChannel <- msg 127 | } 128 | 129 | func (k *cronsumer) recoverMessage(msg MessageWrapper) { 130 | // sending MessageWrapper to closed channel panic could be occurred cause of concurrency for exception topic listeners 131 | if r := recover(); r != nil { 132 | k.cfg.Logger.Warnf("Recovered MessageWrapper: %s", string(msg.Value)) 133 | k.produce(msg) 134 | } 135 | } 136 | 137 | func (k *cronsumer) produce(msg MessageWrapper) { 138 | if msg.IsGteMaxRetryCount(k.maxRetry) { 139 | //nolint:lll 140 | k.cfg.Logger.Infof("Message from %s exceeds to retry limit %d. KafkaMessage: %s, Headers=%v", k.cfg.Consumer.Topic, k.maxRetry, msg.Value, msg.Headers.Pretty()) 141 | 142 | if k.isDeadLetterTopicFeatureEnabled() { 143 | msg.RouteMessageToTopic(k.deadLetterTopic) 144 | if err := k.kafkaProducer.ProduceWithRetryOption(msg, true, false); err != nil { 145 | k.cfg.Logger.Errorf("Error %s sending KafkaMessage to dead letter topic. KafkaMessage: %s", err.Error(), string(msg.Value)) 146 | } 147 | } 148 | 149 | k.metric.TotalDiscardedMessagesCounter++ 150 | 151 | return 152 | } 153 | 154 | if err := k.kafkaProducer.ProduceWithRetryOption(msg, true, false); err != nil { 155 | k.cfg.Logger.Errorf("Error %s sending KafkaMessage %s to the topic %s", err.Error(), string(msg.Value), k.cfg.Consumer.Topic) 156 | } else { 157 | k.metric.TotalRetriedMessagesCounter++ 158 | } 159 | } 160 | 161 | func (k *cronsumer) isDeadLetterTopicFeatureEnabled() bool { 162 | return k.deadLetterTopic != "" 163 | } 164 | -------------------------------------------------------------------------------- /internal/cronsumer_client.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/prometheus/client_golang/prometheus" 8 | 9 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 10 | 11 | "github.com/Trendyol/kafka-cronsumer/pkg/logger" 12 | 13 | gocron "github.com/robfig/cron/v3" 14 | ) 15 | 16 | type cronsumerClient struct { 17 | cfg *kafka.Config 18 | cron *gocron.Cron 19 | consumer *cronsumer 20 | metricCollectors []prometheus.Collector 21 | } 22 | 23 | func NewCronsumer(cfg *kafka.Config, fn kafka.ConsumeFn) kafka.Cronsumer { 24 | c := newCronsumer(cfg, fn) 25 | 26 | return &cronsumerClient{ 27 | cron: gocron.New(), 28 | consumer: c, 29 | cfg: cfg, 30 | metricCollectors: []prometheus.Collector{NewCollector(cfg.MetricPrefix, c.metric)}, 31 | } 32 | } 33 | 34 | func (s *cronsumerClient) WithLogger(logger logger.Interface) { 35 | s.cfg.Logger = logger 36 | } 37 | 38 | func (s *cronsumerClient) Start() { 39 | s.setup() 40 | s.cron.Start() 41 | } 42 | 43 | func (s *cronsumerClient) Run() { 44 | s.setup() 45 | s.cron.Run() 46 | } 47 | 48 | func (s *cronsumerClient) Stop() { 49 | s.cron.Stop() 50 | s.consumer.Stop() 51 | } 52 | 53 | func (s *cronsumerClient) Produce(message kafka.Message) error { 54 | return s.consumer.kafkaProducer.Produce(message) 55 | } 56 | 57 | func (s *cronsumerClient) ProduceBatch(messages []kafka.Message) error { 58 | return s.consumer.kafkaProducer.ProduceBatch(messages) 59 | } 60 | 61 | func (s *cronsumerClient) GetMetricCollectors() []prometheus.Collector { 62 | return s.metricCollectors 63 | } 64 | 65 | func (s *cronsumerClient) setup() { 66 | cfg := s.cfg.Consumer 67 | 68 | s.consumer.SetupConcurrentWorkers(cfg.Concurrency) 69 | schedule, err := gocron.ParseStandard(cfg.Cron) 70 | if err != nil { 71 | panic("Cron parse error: " + err.Error()) 72 | } 73 | 74 | _, _ = s.cron.AddFunc(cfg.Cron, func() { 75 | cancelFuncWrapper := s.startListen(cfg) 76 | if cfg.Duration == kafka.NonStopWork { 77 | now := time.Now() 78 | nextRun := schedule.Next(now) 79 | duration := nextRun.Sub(now) 80 | time.AfterFunc(duration, cancelFuncWrapper) 81 | } else { 82 | time.AfterFunc(cfg.Duration, cancelFuncWrapper) 83 | } 84 | }) 85 | } 86 | 87 | func (s *cronsumerClient) startListen(cfg kafka.ConsumerConfig) func() { 88 | s.cfg.Logger.Debug("Consuming " + cfg.Topic + " started at time: " + time.Now().String()) 89 | 90 | ctx, cancel := context.WithCancel(context.Background()) 91 | cancelFuncWrapper := func() { 92 | s.cfg.Logger.Debug("Consuming " + cfg.Topic + " paused at " + time.Now().String()) 93 | cancel() 94 | } 95 | 96 | go s.consumer.Listen(ctx, cfg.BackOffStrategy.String(), &cancelFuncWrapper) 97 | return cancelFuncWrapper 98 | } 99 | -------------------------------------------------------------------------------- /internal/cronsumer_client_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 8 | ) 9 | 10 | func Test_GetMetricsCollector(t *testing.T) { 11 | t.Parallel() 12 | 13 | t.Run("with FixedBackOffStrategy", func(t *testing.T) { 14 | kafkaConfig := &kafka.Config{ 15 | Brokers: []string{"localhost:29092"}, 16 | Consumer: kafka.ConsumerConfig{ 17 | GroupID: "sample-consumer", 18 | Topic: "exception", 19 | Cron: "@every 1s", 20 | Duration: 20 * time.Second, 21 | BackOffStrategy: kafka.GetBackoffStrategy(kafka.FixedBackOffStrategy), 22 | }, 23 | LogLevel: "info", 24 | } 25 | 26 | var firstConsumerFn kafka.ConsumeFn = func(message kafka.Message) error { 27 | return nil 28 | } 29 | 30 | // When 31 | c := NewCronsumer(kafkaConfig, firstConsumerFn) 32 | 33 | c.Start() 34 | 35 | collector := c.GetMetricCollectors() 36 | // Then 37 | if collector == nil { 38 | t.Errorf("Expected not nil: %+v", collector) 39 | } 40 | }) 41 | 42 | t.Run("with ExponentialBackOffStrategy", func(t *testing.T) { 43 | kafkaConfig := &kafka.Config{ 44 | Brokers: []string{"localhost:29092"}, 45 | Consumer: kafka.ConsumerConfig{ 46 | GroupID: "sample-consumer", 47 | Topic: "exception", 48 | Cron: "@every 1s", 49 | Duration: 20 * time.Second, 50 | BackOffStrategy: kafka.GetBackoffStrategy(kafka.ExponentialBackOffStrategy), 51 | }, 52 | LogLevel: "info", 53 | } 54 | 55 | var firstConsumerFn kafka.ConsumeFn = func(message kafka.Message) error { 56 | return nil 57 | } 58 | 59 | // When 60 | c := NewCronsumer(kafkaConfig, firstConsumerFn) 61 | 62 | c.Start() 63 | 64 | collector := c.GetMetricCollectors() 65 | // Then 66 | if collector == nil { 67 | t.Errorf("Expected not nil: %+v", collector) 68 | } 69 | }) 70 | 71 | t.Run("with LinearBackOffStrategy", func(t *testing.T) { 72 | kafkaConfig := &kafka.Config{ 73 | Brokers: []string{"localhost:29092"}, 74 | Consumer: kafka.ConsumerConfig{ 75 | GroupID: "sample-consumer", 76 | Topic: "exception", 77 | Cron: "@every 1s", 78 | Duration: 20 * time.Second, 79 | BackOffStrategy: kafka.GetBackoffStrategy(kafka.LinearBackOffStrategy), 80 | }, 81 | LogLevel: "info", 82 | } 83 | 84 | var firstConsumerFn kafka.ConsumeFn = func(message kafka.Message) error { 85 | return nil 86 | } 87 | 88 | // When 89 | c := NewCronsumer(kafkaConfig, firstConsumerFn) 90 | 91 | c.Start() 92 | 93 | collector := c.GetMetricCollectors() 94 | // Then 95 | if collector == nil { 96 | t.Errorf("Expected not nil: %+v", collector) 97 | } 98 | }) 99 | } 100 | -------------------------------------------------------------------------------- /internal/cronsumer_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 10 | "github.com/Trendyol/kafka-cronsumer/pkg/logger" 11 | segmentio "github.com/segmentio/kafka-go" 12 | ) 13 | 14 | func Test_Produce_Max_Retry_Count_Reach(t *testing.T) { 15 | // Given 16 | kafkaConfig := &kafka.Config{ 17 | Brokers: []string{"localhost:29092"}, 18 | Consumer: kafka.ConsumerConfig{ 19 | GroupID: "sample-consumer", 20 | Topic: "exception", 21 | Cron: "@every 1s", 22 | Duration: 20 * time.Second, 23 | }, 24 | LogLevel: "info", 25 | Logger: logger.New("info"), 26 | } 27 | 28 | var firstConsumerFn kafka.ConsumeFn = func(message kafka.Message) error { 29 | return nil 30 | } 31 | c := &cronsumer{ 32 | cfg: kafkaConfig, 33 | messageChannel: make(chan MessageWrapper), 34 | kafkaConsumer: mockConsumer{}, 35 | kafkaProducer: newProducer(kafkaConfig), 36 | consumeFn: firstConsumerFn, 37 | metric: &CronsumerMetric{}, 38 | maxRetry: 1, 39 | deadLetterTopic: kafkaConfig.Consumer.DeadLetterTopic, 40 | } 41 | m := MessageWrapper{ 42 | Message: kafka.Message{ 43 | Headers: []kafka.Header{ 44 | {Key: RetryHeaderKey, Value: []byte("1")}, 45 | }, 46 | Topic: "exception", 47 | }, 48 | RetryCount: 1, 49 | } 50 | 51 | // When 52 | c.produce(m) 53 | 54 | // Then 55 | if !reflect.DeepEqual(c.metric.TotalDiscardedMessagesCounter, int64(1)) { 56 | t.Errorf("Expected: %+v, Actual: %+v", c.metric.TotalDiscardedMessagesCounter, int64(1)) 57 | } 58 | } 59 | 60 | func Test_Produce_Max_Retry_Count_Reach_Dead_Letter_Topic_Feature_Enabled(t *testing.T) { 61 | // Given 62 | 63 | var firstConsumerFn kafka.ConsumeFn = func(message kafka.Message) error { 64 | return nil 65 | } 66 | c := &cronsumer{ 67 | cfg: &kafka.Config{ 68 | Logger: logger.New("info"), 69 | }, 70 | messageChannel: make(chan MessageWrapper), 71 | kafkaConsumer: mockConsumer{}, 72 | kafkaProducer: &mockProducer{}, 73 | consumeFn: firstConsumerFn, 74 | metric: &CronsumerMetric{}, 75 | maxRetry: 1, 76 | deadLetterTopic: "abc", 77 | } 78 | m := MessageWrapper{ 79 | Message: kafka.Message{ 80 | Headers: []kafka.Header{ 81 | {Key: RetryHeaderKey, Value: []byte("1")}, 82 | }, 83 | Topic: "exception", 84 | }, 85 | RetryCount: 1, 86 | } 87 | 88 | // When 89 | c.produce(m) 90 | 91 | // Then 92 | if !reflect.DeepEqual(c.metric.TotalDiscardedMessagesCounter, int64(1)) { 93 | t.Errorf("Expected: %+v, Actual: %+v", c.metric.TotalDiscardedMessagesCounter, int64(1)) 94 | } 95 | } 96 | 97 | func Test_Produce_With_Retry(t *testing.T) { 98 | // Given 99 | kafkaConfig := &kafka.Config{ 100 | Brokers: []string{"localhost:29092"}, 101 | Consumer: kafka.ConsumerConfig{ 102 | GroupID: "sample-consumer", 103 | Topic: "exception", 104 | Cron: "@every 1s", 105 | Duration: 20 * time.Second, 106 | }, 107 | LogLevel: "info", 108 | Logger: logger.New("info"), 109 | } 110 | 111 | var firstConsumerFn kafka.ConsumeFn = func(message kafka.Message) error { 112 | return nil 113 | } 114 | producer := newMockProducer() 115 | c := &cronsumer{ 116 | cfg: kafkaConfig, 117 | messageChannel: make(chan MessageWrapper), 118 | kafkaConsumer: mockConsumer{}, 119 | kafkaProducer: &producer, 120 | consumeFn: firstConsumerFn, 121 | metric: &CronsumerMetric{}, 122 | maxRetry: 3, 123 | deadLetterTopic: kafkaConfig.Consumer.DeadLetterTopic, 124 | } 125 | m := MessageWrapper{ 126 | Message: kafka.Message{ 127 | Headers: []kafka.Header{ 128 | {Key: RetryHeaderKey, Value: []byte("1")}, 129 | }, 130 | Topic: "exception", 131 | }, 132 | RetryCount: 1, 133 | } 134 | 135 | // When 136 | c.produce(m) 137 | 138 | // Then 139 | if !reflect.DeepEqual(c.metric.TotalRetriedMessagesCounter, int64(1)) { 140 | t.Errorf("Expected: %+v, Actual: %+v", c.metric.TotalRetriedMessagesCounter, int64(1)) 141 | } 142 | } 143 | 144 | func Test_Recover_Message(t *testing.T) { 145 | // Given 146 | kafkaConfig := &kafka.Config{ 147 | Brokers: []string{"localhost:29092"}, 148 | Consumer: kafka.ConsumerConfig{ 149 | GroupID: "sample-consumer", 150 | Topic: "exception", 151 | Cron: "@every 1s", 152 | Duration: 20 * time.Second, 153 | }, 154 | LogLevel: "info", 155 | Logger: logger.New("info"), 156 | } 157 | 158 | var firstConsumerFn kafka.ConsumeFn = func(message kafka.Message) error { 159 | return nil 160 | } 161 | producer := newMockProducer() 162 | c := &cronsumer{ 163 | cfg: kafkaConfig, 164 | messageChannel: make(chan MessageWrapper), 165 | kafkaConsumer: mockConsumer{}, 166 | kafkaProducer: &producer, 167 | consumeFn: firstConsumerFn, 168 | metric: &CronsumerMetric{}, 169 | maxRetry: 3, 170 | deadLetterTopic: kafkaConfig.Consumer.DeadLetterTopic, 171 | } 172 | m := MessageWrapper{ 173 | Message: kafka.Message{ 174 | Headers: []kafka.Header{ 175 | {Key: RetryHeaderKey, Value: []byte("1")}, 176 | }, 177 | Topic: "exception", 178 | }, 179 | RetryCount: 1, 180 | } 181 | 182 | // When 183 | c.recoverMessage(m) 184 | 185 | // Then 186 | if !reflect.DeepEqual(c.metric.TotalDiscardedMessagesCounter, int64(0)) { 187 | t.Errorf("Expected: %+v, Actual: %+v", c.metric.TotalDiscardedMessagesCounter, int64(0)) 188 | } 189 | if !reflect.DeepEqual(c.metric.TotalRetriedMessagesCounter, int64(0)) { 190 | t.Errorf("Expected: %+v, Actual: %+v", c.metric.TotalRetriedMessagesCounter, int64(0)) 191 | } 192 | } 193 | 194 | type mockConsumer struct{} 195 | 196 | func (c mockConsumer) Stop() { 197 | } 198 | 199 | func (c mockConsumer) ReadMessage(ctx context.Context) (*segmentio.Message, error) { 200 | return &segmentio.Message{}, nil 201 | } 202 | 203 | type mockProducer struct { 204 | w *segmentio.Writer 205 | cfg *kafka.Config 206 | } 207 | 208 | func newMockProducer() mockProducer { 209 | producer := &segmentio.Writer{ 210 | Addr: segmentio.TCP("abc"), 211 | Balancer: &segmentio.LeastBytes{}, 212 | BatchTimeout: 1, 213 | BatchSize: 1, 214 | AllowAutoTopicCreation: true, 215 | } 216 | 217 | return mockProducer{ 218 | w: producer, 219 | cfg: &kafka.Config{}, 220 | } 221 | } 222 | 223 | func (k *mockProducer) ProduceWithRetryOption(message MessageWrapper, increaseRetry bool, increaseRetryAttempt bool) error { 224 | return nil 225 | } 226 | 227 | func (k *mockProducer) Produce(m kafka.Message) error { 228 | return nil 229 | } 230 | 231 | func (k *mockProducer) ProduceBatch(messages []kafka.Message) error { 232 | return nil 233 | } 234 | 235 | func (k *mockProducer) Close() { 236 | } 237 | -------------------------------------------------------------------------------- /internal/message.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "time" 7 | "unsafe" 8 | 9 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 10 | 11 | segmentio "github.com/segmentio/kafka-go" 12 | ) 13 | 14 | const ( 15 | RetryHeaderKey = "x-retry-count" 16 | RetryAttemptHeaderKey = "x-retry-attempt-count" 17 | MessageProduceTimeHeaderKey = "x-produce-time" 18 | MessageErrHeaderKey = "x-error-message" 19 | ) 20 | 21 | type MessageWrapper struct { 22 | kafka.Message 23 | RetryCount int 24 | ProduceTime int64 // Nano time 25 | RetryAttemptCount int 26 | } 27 | 28 | func NewMessageWrapper(msg segmentio.Message, strategyName string) *MessageWrapper { 29 | mw := &MessageWrapper{ 30 | RetryCount: getRetryCount(&msg), 31 | ProduceTime: getMessageProduceTime(&msg), 32 | } 33 | 34 | // Don't add x-retry-attempt-count for fixed strategy. 35 | if strategyName != kafka.FixedBackOffStrategy { 36 | mw.RetryAttemptCount = getRetryAttemptCount(&msg) 37 | } 38 | 39 | mw.Message = kafka.Message{ 40 | Topic: msg.Topic, 41 | Partition: msg.Partition, 42 | Offset: msg.Offset, 43 | HighWaterMark: msg.HighWaterMark, 44 | Key: msg.Key, 45 | Value: msg.Value, 46 | Headers: FromHeaders(msg.Headers), 47 | Time: msg.Time, 48 | } 49 | 50 | return mw 51 | } 52 | 53 | func (m *MessageWrapper) To(increaseRetry bool, increaseRetryAttempt bool) segmentio.Message { 54 | if increaseRetry { 55 | m.IncreaseRetryCount() 56 | m.NewProduceTime() 57 | m.ResetRetryAttempt() 58 | } 59 | 60 | if increaseRetryAttempt { 61 | m.IncreaseRetryAttemptCount() 62 | m.NewProduceTime() 63 | } 64 | 65 | return segmentio.Message{ 66 | Topic: m.Topic, 67 | Key: m.Key, 68 | Value: m.Value, 69 | Headers: ToHeaders(m.Headers), 70 | } 71 | } 72 | 73 | func (m *MessageWrapper) ResetRetryAttempt() { 74 | for i := range m.Headers { 75 | if m.Headers[i].Key == RetryAttemptHeaderKey { 76 | m.Headers[i].Value = []byte("1") 77 | } 78 | } 79 | } 80 | 81 | func (m *MessageWrapper) IncreaseRetryAttemptCount() { 82 | for i := range m.Headers { 83 | if m.Headers[i].Key == RetryAttemptHeaderKey { 84 | byteToStr := *((*string)(unsafe.Pointer(&m.Headers[i].Value))) 85 | retryAttempt, _ := strconv.Atoi(byteToStr) 86 | x := strconv.Itoa(retryAttempt + 1) 87 | m.Headers[i].Value = []byte(x) 88 | } 89 | } 90 | } 91 | 92 | func (m *MessageWrapper) IncreaseRetryCount() { 93 | for i := range m.Headers { 94 | if m.Headers[i].Key == RetryHeaderKey { 95 | byteToStr := *((*string)(unsafe.Pointer(&m.Headers[i].Value))) 96 | retry, _ := strconv.Atoi(byteToStr) 97 | x := strconv.Itoa(retry + 1) 98 | m.Headers[i].Value = []byte(x) 99 | } 100 | } 101 | } 102 | 103 | func (m *MessageWrapper) NewProduceTime() { 104 | for i := range m.Headers { 105 | if m.Headers[i].Key == MessageProduceTimeHeaderKey { 106 | m.Headers[i].Value = []byte(fmt.Sprint(time.Now().UnixNano())) 107 | } 108 | } 109 | } 110 | 111 | func (m *MessageWrapper) GetHeaders() map[string][]byte { 112 | mp := map[string][]byte{} 113 | for i := range m.Headers { 114 | mp[m.Headers[i].Key] = m.Headers[i].Value 115 | } 116 | return mp 117 | } 118 | 119 | func (m *MessageWrapper) IsGteMaxRetryCount(maxRetry int) bool { 120 | return m.RetryCount >= maxRetry 121 | } 122 | 123 | func (m *MessageWrapper) RouteMessageToTopic(topic string) { 124 | m.Topic = topic 125 | } 126 | -------------------------------------------------------------------------------- /internal/message_header.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 7 | segmentio "github.com/segmentio/kafka-go" 8 | ) 9 | 10 | func ToHeaders(h []kafka.Header) []segmentio.Header { 11 | r := make([]segmentio.Header, 0, len(h)) 12 | 13 | for i := range h { 14 | r = append(r, segmentio.Header{ 15 | Key: h[i].Key, 16 | Value: h[i].Value, 17 | }) 18 | } 19 | 20 | return r 21 | } 22 | 23 | func FromHeaders(sh []segmentio.Header) []kafka.Header { 24 | r := make([]kafka.Header, 0, len(sh)) 25 | 26 | for i := range sh { 27 | r = append(r, kafka.Header{ 28 | Key: sh[i].Key, 29 | Value: sh[i].Value, 30 | }) 31 | } 32 | 33 | return r 34 | } 35 | 36 | func createErrHeader(consumeErr error) kafka.Header { 37 | return kafka.Header{ 38 | Key: MessageErrHeaderKey, 39 | Value: []byte(consumeErr.Error()), 40 | } 41 | } 42 | 43 | func getRetryCount(message *segmentio.Message) int { 44 | for i := range message.Headers { 45 | if message.Headers[i].Key != RetryHeaderKey { 46 | continue 47 | } 48 | 49 | retryCount, _ := strconv.Atoi(string(message.Headers[i].Value)) 50 | return retryCount 51 | } 52 | 53 | message.Headers = append(message.Headers, segmentio.Header{ 54 | Key: RetryHeaderKey, 55 | Value: []byte("0"), 56 | }) 57 | 58 | return 0 59 | } 60 | 61 | func getRetryAttemptCount(message *segmentio.Message) int { 62 | for i := range message.Headers { 63 | if message.Headers[i].Key != RetryAttemptHeaderKey { 64 | continue 65 | } 66 | 67 | retryCount, _ := strconv.Atoi(string(message.Headers[i].Value)) 68 | return retryCount 69 | } 70 | 71 | message.Headers = append(message.Headers, segmentio.Header{ 72 | Key: RetryAttemptHeaderKey, 73 | Value: []byte("1"), 74 | }) 75 | 76 | return 1 77 | } 78 | 79 | func getMessageProduceTime(message *segmentio.Message) int64 { 80 | for i := range message.Headers { 81 | if message.Headers[i].Key != MessageProduceTimeHeaderKey { 82 | continue 83 | } 84 | 85 | ts, _ := strconv.Atoi(string(message.Headers[i].Value)) 86 | return int64(ts) 87 | } 88 | 89 | message.Headers = append(message.Headers, segmentio.Header{ 90 | Key: MessageProduceTimeHeaderKey, 91 | Value: []byte("0"), 92 | }) 93 | 94 | return 0 95 | } 96 | -------------------------------------------------------------------------------- /internal/message_header_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "strconv" 7 | "testing" 8 | 9 | pkg "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 10 | "github.com/segmentio/kafka-go" 11 | "github.com/segmentio/kafka-go/protocol" 12 | ) 13 | 14 | func Test_getMessageProduceTime(t *testing.T) { 15 | t.Run("Should Return Value When Produce Time Does Exist", func(t *testing.T) { 16 | // Given 17 | var expected int64 = 1691092689380152000 18 | expectedStr := "1691092689380152000" 19 | km := &kafka.Message{ 20 | Headers: []protocol.Header{ 21 | {Key: RetryHeaderKey, Value: []byte("1")}, 22 | {Key: MessageProduceTimeHeaderKey, Value: []byte(expectedStr)}, 23 | }, 24 | } 25 | 26 | // When 27 | actual := getMessageProduceTime(km) 28 | 29 | // Then 30 | if actual != expected { 31 | t.Errorf("Expected: %d, Actual: %d", expected, actual) 32 | } 33 | }) 34 | t.Run("Should Return Default Value When Produce Time Does Not Exist", func(t *testing.T) { 35 | // Given 36 | km := &kafka.Message{ 37 | Headers: []protocol.Header{}, 38 | } 39 | 40 | // When 41 | actual := getMessageProduceTime(km) 42 | 43 | // Then 44 | if actual != 0 { 45 | t.Errorf("Expected: %d, Actual: %d", 0, actual) 46 | } 47 | }) 48 | } 49 | 50 | func Test_getRetryCount(t *testing.T) { 51 | t.Parallel() 52 | 53 | t.Run("When X-Retry-Count not found with existent headers", func(t *testing.T) { 54 | // Given 55 | km := &kafka.Message{ 56 | Headers: []protocol.Header{ 57 | {Key: "Some Header", Value: []byte("Some Value")}, 58 | }, 59 | } 60 | 61 | // When 62 | rc := getRetryCount(km) 63 | 64 | // Then 65 | if rc != 0 { 66 | t.Errorf("Expected: %d, Actual: %d", 0, rc) 67 | } 68 | }) 69 | t.Run("When X-Retry-Count not found", func(t *testing.T) { 70 | // Given 71 | km := &kafka.Message{ 72 | Headers: nil, 73 | } 74 | 75 | // When 76 | rc := getRetryCount(km) 77 | 78 | // Then 79 | if rc != 0 { 80 | t.Errorf("Expected: %d, Actual: %d", 0, rc) 81 | } 82 | }) 83 | t.Run("When X-Retry-Count exists", func(t *testing.T) { 84 | // Given 85 | km := &kafka.Message{ 86 | Headers: []protocol.Header{ 87 | {Key: RetryHeaderKey, Value: []byte("2")}, 88 | }, 89 | } 90 | 91 | // When 92 | rc := getRetryCount(km) 93 | 94 | // Then 95 | actual := strconv.Itoa(rc) 96 | expected := string(km.Headers[0].Value) 97 | 98 | if expected != actual { 99 | t.Errorf("Expected: %s, Actual: %s", expected, actual) 100 | } 101 | }) 102 | } 103 | 104 | func Test_FromHeaders(t *testing.T) { 105 | // Given 106 | expected := []pkg.Header{ 107 | {Key: "x-retry-count", Value: []byte("1")}, 108 | } 109 | // When 110 | actual := ToHeaders(expected) 111 | actualHeader := actual[0] 112 | expectedHeader := expected[0] 113 | // Then 114 | if actualHeader.Key != expectedHeader.Key { 115 | t.Errorf("Expected: %s, Actual: %s", actualHeader.Key, expectedHeader.Key) 116 | } 117 | if !bytes.Equal(actualHeader.Value, expectedHeader.Value) { 118 | t.Errorf("Expected: %s, Actual: %s", expectedHeader.Value, expectedHeader.Value) 119 | } 120 | } 121 | 122 | func Test_ToHeaders(t *testing.T) { 123 | // Given 124 | expected := []kafka.Header{ 125 | {Key: "x-retry-count", Value: []byte("1")}, 126 | } 127 | // When 128 | actual := FromHeaders(expected) 129 | actualHeader := actual[0] 130 | expectedHeader := expected[0] 131 | // Then 132 | if actualHeader.Key != expectedHeader.Key { 133 | t.Errorf("Expected: %s, Actual: %s", actualHeader.Key, expectedHeader.Key) 134 | } 135 | if !bytes.Equal(actualHeader.Value, expectedHeader.Value) { 136 | t.Errorf("Expected: %s, Actual: %s", expectedHeader.Value, expectedHeader.Value) 137 | } 138 | } 139 | 140 | func Test_getRetryAttempt(t *testing.T) { 141 | t.Parallel() 142 | 143 | t.Run("When X-Retry-Attempt-Count not found with existent headers", func(t *testing.T) { 144 | // Given 145 | km := &kafka.Message{ 146 | Headers: []protocol.Header{ 147 | {Key: "Some Header", Value: []byte("Some Value")}, 148 | }, 149 | } 150 | 151 | // When 152 | rc := getRetryAttemptCount(km) 153 | 154 | // Then 155 | if rc != 1 { 156 | t.Errorf("Expected: %d, Actual: %d", 1, rc) 157 | } 158 | }) 159 | t.Run("When X-Retry-Attempt-Count exists", func(t *testing.T) { 160 | // Given 161 | km := &kafka.Message{ 162 | Headers: []protocol.Header{ 163 | {Key: RetryAttemptHeaderKey, Value: []byte("2")}, 164 | }, 165 | } 166 | 167 | // When 168 | rc := getRetryAttemptCount(km) 169 | 170 | // Then 171 | actual := strconv.Itoa(rc) 172 | expected := string(km.Headers[0].Value) 173 | 174 | if expected != actual { 175 | t.Errorf("Expected: %s, Actual: %s", expected, actual) 176 | } 177 | }) 178 | 179 | t.Run("When X-Retry-Attempt-Count not found", func(t *testing.T) { 180 | // Given 181 | km := &kafka.Message{ 182 | Headers: nil, 183 | } 184 | 185 | // When 186 | rc := getRetryAttemptCount(km) 187 | 188 | // Then 189 | if rc != 1 { 190 | t.Errorf("Expected: %d, Actual: %d", 1, rc) 191 | } 192 | }) 193 | } 194 | 195 | func TestMessage_CreateErrHeader(t *testing.T) { 196 | // Given 197 | e := errors.New("err") 198 | 199 | // When 200 | h := createErrHeader(e) 201 | 202 | // Then 203 | if h.Key != MessageErrHeaderKey { 204 | t.Fatalf("Header key must be equal to X-ErrMessage") 205 | } 206 | 207 | if h.Value == nil { 208 | t.Fatalf("Header value must be present") 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /internal/message_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "bytes" 5 | _ "embed" 6 | "testing" 7 | "time" 8 | 9 | segmentio "github.com/segmentio/kafka-go" 10 | 11 | . "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 12 | ) 13 | 14 | func Test_NewMessageWrapper(t *testing.T) { 15 | // Given 16 | expected := segmentio.Message{ 17 | Topic: "topic", 18 | Partition: 1, 19 | Offset: 1, 20 | HighWaterMark: 1, 21 | Key: []byte("1"), 22 | Value: []byte("1"), 23 | Headers: []segmentio.Header{ 24 | {Key: RetryHeaderKey, Value: []byte("1")}, 25 | }, 26 | WriterData: "1", 27 | Time: time.Now(), 28 | } 29 | 30 | // When 31 | actual := NewMessageWrapper(expected, FixedBackOffStrategy) 32 | actualHeader := actual.Headers[0] 33 | expectedHeader := expected.Headers[0] 34 | 35 | // Then 36 | if actual.Topic != expected.Topic { 37 | t.Errorf("Expected: %s, Actual: %s", expected.Topic, actual.Topic) 38 | } 39 | if actual.Partition != expected.Partition { 40 | t.Errorf("Expected: %d, Actual: %d", expected.Partition, actual.Partition) 41 | } 42 | if actual.Offset != expected.Offset { 43 | t.Errorf("Expected: %d, Actual: %d", expected.Offset, actual.Offset) 44 | } 45 | if actual.HighWaterMark != expected.HighWaterMark { 46 | t.Errorf("Expected: %d, Actual: %d", expected.HighWaterMark, actual.HighWaterMark) 47 | } 48 | if !bytes.Equal(actual.Key, expected.Key) { 49 | t.Errorf("Expected: %s, Actual: %s", expected.Value, actual.Value) 50 | } 51 | if !bytes.Equal(actual.Value, expected.Value) { 52 | t.Errorf("Expected: %s, Actual: %s", expected.Value, actual.Value) 53 | } 54 | if actualHeader.Key != expectedHeader.Key { 55 | t.Errorf("Expected: %s, Actual: %s", actualHeader.Key, expectedHeader.Key) 56 | } 57 | if !bytes.Equal(actualHeader.Value, expectedHeader.Value) { 58 | t.Errorf("Expected: %s, Actual: %s", expectedHeader.Value, expectedHeader.Value) 59 | } 60 | if actual.Time != expected.Time { 61 | t.Errorf("Expected: %s, Actual: %s", expected.Value, actual.Value) 62 | } 63 | } 64 | 65 | func Test_increaseRetryCount(t *testing.T) { 66 | // Given 67 | m := MessageWrapper{ 68 | Message: Message{ 69 | Headers: []Header{ 70 | {Key: RetryHeaderKey, Value: []byte("1")}, 71 | }, 72 | Topic: "exception", 73 | }, 74 | RetryCount: 1, 75 | } 76 | 77 | // When 78 | m.IncreaseRetryCount() 79 | 80 | // Then 81 | actual := m.GetHeaders()[RetryHeaderKey] 82 | expected := []byte("2") 83 | if !bytes.Equal(expected, actual) { 84 | t.Errorf("Expected: %s, Actual: %s", expected, actual) 85 | } 86 | } 87 | 88 | func Test_increaseRetryAttemptCount(t *testing.T) { 89 | // Given 90 | m := MessageWrapper{ 91 | Message: Message{ 92 | Headers: []Header{ 93 | {Key: RetryHeaderKey, Value: []byte("2")}, 94 | {Key: RetryAttemptHeaderKey, Value: []byte("1")}, 95 | }, 96 | Topic: "exception", 97 | }, 98 | RetryCount: 1, 99 | } 100 | 101 | // When 102 | m.IncreaseRetryAttemptCount() 103 | 104 | // Then 105 | actualRetryCount := m.GetHeaders()[RetryHeaderKey] 106 | expectedRetryCount := []byte("2") 107 | if !bytes.Equal(expectedRetryCount, actualRetryCount) { 108 | t.Errorf("Expected: %s, Actual: %s", expectedRetryCount, actualRetryCount) 109 | } 110 | 111 | actualRetryAttempt := m.GetHeaders()[RetryAttemptHeaderKey] 112 | expectedRetryAttempt := []byte("2") 113 | if !bytes.Equal(expectedRetryAttempt, actualRetryAttempt) { 114 | t.Errorf("Expected: %s, Actual: %s", expectedRetryAttempt, actualRetryAttempt) 115 | } 116 | } 117 | 118 | func TestMessageWrapper_NewProduceTime(t *testing.T) { 119 | // Given 120 | mw := MessageWrapper{ 121 | Message: Message{Headers: []Header{ 122 | {Key: MessageProduceTimeHeaderKey, Value: []byte("some value")}, 123 | }}, 124 | } 125 | 126 | // When 127 | mw.NewProduceTime() 128 | 129 | // Then 130 | actual := mw.GetHeaders()[MessageProduceTimeHeaderKey] 131 | notExpected := []byte("some value") 132 | 133 | if bytes.Equal(actual, notExpected) { 134 | t.Errorf("Not Expected: %s, Actual: %s", notExpected, actual) 135 | } 136 | } 137 | 138 | func TestMessageWrapper_IsExceedMaxRetryCount(t *testing.T) { 139 | // Given 140 | maxRetry := 2 141 | m1 := MessageWrapper{RetryCount: 3} 142 | m2 := MessageWrapper{RetryCount: 1} 143 | 144 | // When 145 | actual1 := m1.IsGteMaxRetryCount(maxRetry) 146 | actual2 := m2.IsGteMaxRetryCount(maxRetry) 147 | 148 | // Then 149 | if actual1 != true { 150 | t.Fatal() 151 | } 152 | if actual2 != false { 153 | t.Fatal() 154 | } 155 | } 156 | 157 | func TestMessageWrapper_To_With_Increase_Retry(t *testing.T) { 158 | // Given 159 | expected := MessageWrapper{ 160 | Message: Message{ 161 | Topic: "topic", 162 | Key: []byte("key"), 163 | Value: []byte("1"), 164 | Headers: []Header{ 165 | {Key: "x-retry-count", Value: []byte("1")}, 166 | {Key: "x-retry-attempt-count", Value: []byte("1")}, 167 | }, 168 | }, 169 | RetryCount: 1, 170 | RetryAttemptCount: 0, 171 | } 172 | // When 173 | actual := expected.To(true, false) 174 | actualHeader := actual.Headers[0] 175 | expectedHeader := expected.Headers[0] 176 | 177 | retryAttemptHeader := actual.Headers[1] 178 | expectedRetryAttemptHeader := expected.Headers[1] 179 | 180 | // Then 181 | if actual.Topic != expected.Topic { 182 | t.Errorf("Expected: %s, Actual: %s", expected.Topic, actual.Topic) 183 | } 184 | if !bytes.Equal(actual.Key, expected.Key) { 185 | t.Errorf("Expected: %s, Actual: %s", expected.Key, actual.Key) 186 | } 187 | if !bytes.Equal(actual.Value, expected.Value) { 188 | t.Errorf("Expected: %s, Actual: %s", expected.Value, actual.Value) 189 | } 190 | if actualHeader.Key != expectedHeader.Key { 191 | t.Errorf("Expected: %s, Actual: %s", actualHeader.Key, expectedHeader.Key) 192 | } 193 | if !bytes.Equal(actualHeader.Value, expectedHeader.Value) { 194 | t.Errorf("Expected: %s, Actual: %s", expectedHeader.Value, expectedHeader.Value) 195 | } 196 | if retryAttemptHeader.Key != expectedRetryAttemptHeader.Key { 197 | t.Errorf("Expected: %s, Actual: %s", actualHeader.Key, expectedHeader.Key) 198 | } 199 | if !bytes.Equal(retryAttemptHeader.Value, expectedRetryAttemptHeader.Value) { 200 | t.Errorf("Expected: %s, Actual: %s", expectedHeader.Value, expectedHeader.Value) 201 | } 202 | } 203 | 204 | func TestMessageWrapper_To_With_Increase_Retry_Attempt(t *testing.T) { 205 | // Given 206 | expected := MessageWrapper{ 207 | Message: Message{ 208 | Topic: "topic", 209 | Value: []byte("1"), 210 | Headers: []Header{ 211 | {Key: "x-retry-count", Value: []byte("1")}, 212 | {Key: "x-retry-attempt-count", Value: []byte("1")}, 213 | }, 214 | }, 215 | RetryCount: 1, 216 | RetryAttemptCount: 2, 217 | } 218 | // When 219 | actual := expected.To(false, true) 220 | actualHeader := actual.Headers[0] 221 | expectedHeader := expected.Headers[0] 222 | 223 | retryAttemptHeader := actual.Headers[1] 224 | expectedRetryAttemptHeader := expected.Headers[1] 225 | 226 | // Then 227 | if actual.Topic != expected.Topic { 228 | t.Errorf("Expected: %s, Actual: %s", expected.Topic, actual.Topic) 229 | } 230 | if !bytes.Equal(actual.Value, expected.Value) { 231 | t.Errorf("Expected: %s, Actual: %s", expected.Value, actual.Value) 232 | } 233 | if actualHeader.Key != expectedHeader.Key { 234 | t.Errorf("Expected: %s, Actual: %s", actualHeader.Key, expectedHeader.Key) 235 | } 236 | if !bytes.Equal(actualHeader.Value, []byte("1")) { 237 | t.Errorf("Expected: %s, Actual: %s", expectedHeader.Value, expectedHeader.Value) 238 | } 239 | if retryAttemptHeader.Key != expectedRetryAttemptHeader.Key { 240 | t.Errorf("Expected: %s, Actual: %s", actualHeader.Key, expectedHeader.Key) 241 | } 242 | if !bytes.Equal(retryAttemptHeader.Value, []byte("2")) { 243 | t.Errorf("Expected: %s, Actual: %s", expectedHeader.Value, expectedHeader.Value) 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /internal/metric.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | const Name = "kafka_cronsumer" 4 | 5 | type CronsumerMetric struct { 6 | TotalRetriedMessagesCounter int64 7 | TotalDiscardedMessagesCounter int64 8 | } 9 | -------------------------------------------------------------------------------- /internal/producer.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 7 | segmentio "github.com/segmentio/kafka-go" 8 | ) 9 | 10 | type Producer interface { 11 | ProduceWithRetryOption(message MessageWrapper, increaseRetry bool, increaseRetryAttempt bool) error 12 | Produce(message kafka.Message) error 13 | ProduceBatch(messages []kafka.Message) error 14 | Close() 15 | } 16 | 17 | type kafkaProducer struct { 18 | w *segmentio.Writer 19 | cfg *kafka.Config 20 | } 21 | 22 | func newProducer(kafkaConfig *kafka.Config) Producer { 23 | if kafkaConfig.Producer.Balancer == nil { 24 | kafkaConfig.Producer.Balancer = &segmentio.LeastBytes{} 25 | } 26 | 27 | producer := &segmentio.Writer{ 28 | Addr: kafkaConfig.GetBrokerAddr(), 29 | Balancer: kafkaConfig.Producer.Balancer, 30 | BatchTimeout: kafkaConfig.Producer.BatchTimeout, 31 | BatchSize: kafkaConfig.Producer.BatchSize, 32 | AllowAutoTopicCreation: true, 33 | } 34 | 35 | transport := &segmentio.Transport{ 36 | ClientID: kafkaConfig.ClientID, 37 | } 38 | 39 | if kafkaConfig.SASL.Enabled { 40 | transport.TLS = NewTLSConfig(kafkaConfig) 41 | transport.SASL = Mechanism(kafkaConfig.SASL) 42 | } 43 | 44 | producer.Transport = transport 45 | 46 | return &kafkaProducer{ 47 | w: producer, 48 | cfg: kafkaConfig, 49 | } 50 | } 51 | 52 | func (k *kafkaProducer) ProduceWithRetryOption(message MessageWrapper, increaseRetry bool, increaseRetryAttempt bool) error { 53 | return k.w.WriteMessages(context.Background(), message.To(increaseRetry, increaseRetryAttempt)) 54 | } 55 | 56 | func (k *kafkaProducer) Produce(m kafka.Message) error { 57 | return k.w.WriteMessages(context.Background(), segmentio.Message{ 58 | Topic: m.Topic, 59 | Partition: m.Partition, 60 | HighWaterMark: m.HighWaterMark, 61 | Key: m.Key, 62 | Value: m.Value, 63 | Headers: ToHeaders(m.Headers), 64 | }) 65 | } 66 | 67 | func (k *kafkaProducer) ProduceBatch(messages []kafka.Message) error { 68 | segmentioMessages := make([]segmentio.Message, 0, len(messages)) 69 | for i := range messages { 70 | segmentioMessages = append(segmentioMessages, segmentio.Message{ 71 | Topic: messages[i].Topic, 72 | Partition: messages[i].Partition, 73 | HighWaterMark: messages[i].HighWaterMark, 74 | Key: messages[i].Key, 75 | Value: messages[i].Value, 76 | Headers: ToHeaders(messages[i].Headers), 77 | }) 78 | } 79 | return k.w.WriteMessages(context.Background(), segmentioMessages...) 80 | } 81 | 82 | func (k *kafkaProducer) Close() { 83 | err := k.w.Close() 84 | if err != nil { 85 | k.cfg.Logger.Errorf("Error while closing kafka producer %v", err) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /internal/secure.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "os" 7 | 8 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 9 | 10 | "github.com/segmentio/kafka-go/sasl" 11 | "github.com/segmentio/kafka-go/sasl/scram" 12 | ) 13 | 14 | func NewTLSConfig(cfg *kafka.Config) *tls.Config { 15 | rootCA, err := os.ReadFile(cfg.SASL.RootCAPath) 16 | if err != nil { 17 | panic("Error while reading Root CA file: " + cfg.SASL.RootCAPath + " error: " + err.Error()) 18 | } 19 | 20 | caCertPool := x509.NewCertPool() 21 | if ok := caCertPool.AppendCertsFromPEM(rootCA); !ok { 22 | panic("failed to append Root CA certificates from file: " + cfg.SASL.RootCAPath) 23 | } 24 | 25 | interCA, err := os.ReadFile(cfg.SASL.IntermediateCAPath) 26 | if err != nil { 27 | cfg.Logger.Warnf("Unable to read Intermediate CA file: %s, error: %v", cfg.SASL.IntermediateCAPath, err) 28 | cfg.Logger.Info("Intermediate CA will be skipped.") 29 | } else if ok := caCertPool.AppendCertsFromPEM(interCA); !ok { 30 | cfg.Logger.Warnf("Failed to append Intermediate CA certificates from file: %s", cfg.SASL.IntermediateCAPath) 31 | } 32 | 33 | return &tls.Config{ 34 | RootCAs: caCertPool, 35 | MinVersion: tls.VersionTLS12, 36 | } 37 | } 38 | 39 | // TODO: we can support `plain` authentication type 40 | // link: https://github.com/segmentio/kafka-go#plain 41 | func Mechanism(sasl kafka.SASLConfig) sasl.Mechanism { 42 | mechanism, err := scram.Mechanism(scram.SHA512, sasl.Username, sasl.Password) 43 | if err != nil { 44 | panic("Error while creating SCRAM configuration, error: " + err.Error()) 45 | } 46 | 47 | return mechanism 48 | } 49 | -------------------------------------------------------------------------------- /internal/verify_topic.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 8 | segmentio "github.com/segmentio/kafka-go" 9 | ) 10 | 11 | type kafkaClient interface { 12 | Metadata(ctx context.Context, req *segmentio.MetadataRequest) (*segmentio.MetadataResponse, error) 13 | GetClient() *segmentio.Client 14 | } 15 | 16 | type client struct { 17 | *segmentio.Client 18 | } 19 | 20 | func NewKafkaClient(cfg *kafka.Config) (kafkaClient, error) { 21 | kc := client{ 22 | Client: &segmentio.Client{ 23 | Addr: segmentio.TCP(cfg.Brokers...), 24 | }, 25 | } 26 | 27 | transport := &segmentio.Transport{ 28 | MetadataTopics: []string{cfg.Consumer.Topic}, 29 | } 30 | 31 | if cfg.SASL.Enabled { 32 | transport.TLS = NewTLSConfig(cfg) 33 | transport.SASL = Mechanism(cfg.SASL) 34 | } 35 | 36 | kc.Transport = transport 37 | return &kc, nil 38 | } 39 | 40 | func (c *client) GetClient() *segmentio.Client { 41 | return c.Client 42 | } 43 | 44 | func VerifyTopics(client kafkaClient, topics ...string) (bool, error) { 45 | metadata, err := client.Metadata(context.Background(), &segmentio.MetadataRequest{ 46 | Topics: topics, 47 | }) 48 | if err != nil { 49 | return false, fmt.Errorf("error when during verifyTopics metadata request %w", err) 50 | } 51 | return checkTopicsWithinMetadata(metadata, topics) 52 | } 53 | 54 | func checkTopicsWithinMetadata(metadata *segmentio.MetadataResponse, topics []string) (bool, error) { 55 | metadataTopics := make(map[string]struct{}, len(metadata.Topics)) 56 | for _, topic := range metadata.Topics { 57 | if topic.Error != nil { 58 | continue 59 | } 60 | metadataTopics[topic.Name] = struct{}{} 61 | } 62 | 63 | for _, topic := range topics { 64 | if _, exist := metadataTopics[topic]; !exist { 65 | return false, nil 66 | } 67 | } 68 | return true, nil 69 | } 70 | -------------------------------------------------------------------------------- /internal/verify_topic_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | 8 | "github.com/Trendyol/kafka-cronsumer/pkg/kafka" 9 | segmentio "github.com/segmentio/kafka-go" 10 | ) 11 | 12 | type mockKafkaClientWrapper struct { 13 | wantErr bool 14 | wantExistTopic bool 15 | } 16 | 17 | func (m mockKafkaClientWrapper) GetClient() *segmentio.Client { 18 | return &segmentio.Client{} 19 | } 20 | 21 | func (m mockKafkaClientWrapper) Metadata(_ context.Context, _ *segmentio.MetadataRequest) (*segmentio.MetadataResponse, error) { 22 | if m.wantErr { 23 | return nil, errors.New("metadataReqErr") 24 | } 25 | 26 | if !m.wantExistTopic { 27 | return &segmentio.MetadataResponse{ 28 | Topics: []segmentio.Topic{ 29 | {Name: "topic1", Error: segmentio.UnknownTopicOrPartition}, 30 | {Name: "topic2", Error: nil}, 31 | }, 32 | }, nil 33 | } 34 | 35 | return &segmentio.MetadataResponse{ 36 | Topics: []segmentio.Topic{ 37 | {Name: "topic1", Error: nil}, 38 | {Name: "topic2", Error: nil}, 39 | }, 40 | }, nil 41 | } 42 | 43 | func Test_kafkaClientWrapper_VerifyTopics(t *testing.T) { 44 | t.Run("Should_Return_Error_When_Metadata_Request_Has_Failed", func(t *testing.T) { 45 | // Given 46 | mockClient := mockKafkaClientWrapper{wantErr: true} 47 | 48 | // When 49 | _, err := VerifyTopics(mockClient, "topic1") 50 | 51 | // Then 52 | if err == nil { 53 | t.Error("metadata request must be failed!") 54 | } 55 | }) 56 | t.Run("Should_Return_False_When_Given_Topic_Does_Not_Exist", func(t *testing.T) { 57 | // Given 58 | mockClient := mockKafkaClientWrapper{wantExistTopic: false} 59 | 60 | // When 61 | exist, err := VerifyTopics(mockClient, "topic1") 62 | 63 | // Then 64 | if exist { 65 | t.Errorf("topic %s must not exist", "topic1") 66 | } 67 | if err != nil { 68 | t.Error("err must be nil") 69 | } 70 | }) 71 | t.Run("Should_Return_True_When_Given_Topic_Exist", func(t *testing.T) { 72 | // Given 73 | mockClient := mockKafkaClientWrapper{wantExistTopic: true} 74 | 75 | // When 76 | exist, err := VerifyTopics(mockClient, "topic1") 77 | 78 | // Then 79 | if !exist { 80 | t.Errorf("topic %s must exist", "topic1") 81 | } 82 | if err != nil { 83 | t.Error("err must be nil") 84 | } 85 | }) 86 | } 87 | 88 | func Test_newKafkaClient(t *testing.T) { 89 | // Given 90 | cfg := &kafka.Config{Brokers: []string{"127.0.0.1:9092"}, Consumer: kafka.ConsumerConfig{Topic: "topic"}} 91 | 92 | // When 93 | sut, err := NewKafkaClient(cfg) 94 | 95 | // Then 96 | if sut.GetClient().Addr.String() != "127.0.0.1:9092" { 97 | t.Errorf("broker address must be 127.0.0.1:9092") 98 | } 99 | if err != nil { 100 | t.Errorf("err must be nil") 101 | } 102 | } 103 | 104 | func Test_kClient_GetClient(t *testing.T) { 105 | // Given 106 | mockClient := mockKafkaClientWrapper{} 107 | 108 | // When 109 | sut := mockClient.GetClient() 110 | 111 | // Then 112 | if sut == nil { 113 | t.Error("client must not be nil") 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /pkg/kafka/backoff_strategy.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "math" 5 | ) 6 | 7 | type BackoffStrategyInterface interface { 8 | ShouldIncreaseRetryAttemptCount(retryCount int, retryAttemptCount int) bool 9 | String() string 10 | } 11 | 12 | type LinearBackoffStrategy struct{} 13 | 14 | func (s *LinearBackoffStrategy) ShouldIncreaseRetryAttemptCount(retryCount int, retryAttemptCount int) bool { 15 | return retryCount >= retryAttemptCount 16 | } 17 | 18 | func (s *LinearBackoffStrategy) String() string { 19 | return LinearBackOffStrategy 20 | } 21 | 22 | type ExponentialBackoffStrategy struct{} 23 | 24 | func (s *ExponentialBackoffStrategy) ShouldIncreaseRetryAttemptCount(retryCount int, retryAttemptCount int) bool { 25 | return int(math.Pow(2, float64(retryCount))) > retryAttemptCount 26 | } 27 | 28 | func (s *ExponentialBackoffStrategy) String() string { 29 | return ExponentialBackOffStrategy 30 | } 31 | 32 | type FixedBackoffStrategy struct{} 33 | 34 | func (s *FixedBackoffStrategy) ShouldIncreaseRetryAttemptCount(_ int, _ int) bool { 35 | return false 36 | } 37 | 38 | func (s *FixedBackoffStrategy) String() string { 39 | return FixedBackOffStrategy 40 | } 41 | 42 | func GetBackoffStrategy(strategyName string) BackoffStrategyInterface { 43 | switch strategyName { 44 | case FixedBackOffStrategy: 45 | return &FixedBackoffStrategy{} 46 | case LinearBackOffStrategy: 47 | return &LinearBackoffStrategy{} 48 | case ExponentialBackOffStrategy: 49 | return &ExponentialBackoffStrategy{} 50 | default: 51 | return nil 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pkg/kafka/config.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "net" 5 | "strconv" 6 | "time" 7 | 8 | segmentio "github.com/segmentio/kafka-go" 9 | 10 | "github.com/Trendyol/kafka-cronsumer/pkg/logger" 11 | ) 12 | 13 | type Offset string 14 | 15 | const ( 16 | OffsetEarliest = "earliest" 17 | OffsetLatest = "latest" 18 | ExponentialBackOffStrategy = "exponential" 19 | LinearBackOffStrategy = "linear" 20 | FixedBackOffStrategy = "fixed" 21 | ) 22 | 23 | //nolint:all 24 | var NonStopWork time.Duration = 0 25 | 26 | type Config struct { 27 | Brokers []string `yaml:"brokers"` 28 | Consumer ConsumerConfig `yaml:"consumer"` 29 | Producer ProducerConfig `yaml:"producer"` 30 | SASL SASLConfig `yaml:"sasl"` 31 | LogLevel logger.Level `yaml:"logLevel"` 32 | Logger logger.Interface `yaml:"-"` 33 | ClientID string `yaml:"clientId"` 34 | 35 | // MetricPrefix is used for prometheus fq name prefix. 36 | // If not provided, default metric prefix value is `kafka_cronsumer`. 37 | // Currently, there are two exposed prometheus metrics. `retried_messages_total_current` and `discarded_messages_total_current`. 38 | // So, if default metric prefix used, metrics names are `kafka_cronsumer_retried_messages_total_current` and 39 | // `kafka_cronsumer_discarded_messages_total_current`. 40 | MetricPrefix string `yaml:"metricPrefix"` 41 | } 42 | 43 | func (c *Config) GetBrokerAddr() net.Addr { 44 | if len(c.Producer.Brokers) == 0 { 45 | c.Producer.Brokers = c.Brokers 46 | } 47 | 48 | return segmentio.TCP(c.Producer.Brokers...) 49 | } 50 | 51 | type SASLConfig struct { 52 | Enabled bool `yaml:"enabled"` 53 | AuthType string `yaml:"authType"` // plain or scram 54 | Username string `yaml:"username"` 55 | Password string `yaml:"password"` 56 | RootCAPath string `yaml:"rootCAPath"` 57 | IntermediateCAPath string `yaml:"intermediateCAPath"` 58 | Rack string `yaml:"rack"` 59 | } 60 | 61 | type ConsumerConfig struct { 62 | ClientID string `yaml:"clientId"` 63 | GroupID string `yaml:"groupId"` 64 | Topic string `yaml:"topic"` 65 | DeadLetterTopic string `yaml:"deadLetterTopic"` 66 | MinBytes int `yaml:"minBytes"` 67 | MaxBytes int `yaml:"maxBytes"` 68 | MaxRetry int `yaml:"maxRetry"` 69 | MaxWait time.Duration `yaml:"maxWait"` 70 | CommitInterval time.Duration `yaml:"commitInterval"` 71 | HeartbeatInterval time.Duration `yaml:"heartbeatInterval"` 72 | SessionTimeout time.Duration `yaml:"sessionTimeout"` 73 | RebalanceTimeout time.Duration `yaml:"rebalanceTimeout"` 74 | StartOffset Offset `yaml:"startOffset"` 75 | RetentionTime time.Duration `yaml:"retentionTime"` 76 | Concurrency int `yaml:"concurrency"` 77 | Duration time.Duration `yaml:"duration"` 78 | Cron string `yaml:"cron"` 79 | BackOffStrategy BackoffStrategyInterface `yaml:"backOffStrategy"` 80 | SkipMessageByHeaderFn SkipMessageByHeaderFn `yaml:"skipMessageByHeaderFn"` 81 | QueueCapacity int `yaml:"queueCapacity"` 82 | } 83 | 84 | type ProducerConfig struct { 85 | Brokers []string `yaml:"brokers"` 86 | BatchSize int `yaml:"batchSize"` 87 | BatchTimeout time.Duration `yaml:"batchTimeout"` 88 | Balancer segmentio.Balancer `yaml:"balancer"` 89 | } 90 | 91 | type SkipMessageByHeaderFn func(headers []Header) bool 92 | 93 | func (c *Config) SetDefaults() { 94 | if c.Consumer.MaxRetry == 0 { 95 | c.Consumer.MaxRetry = 3 96 | } 97 | if c.Consumer.Concurrency == 0 { 98 | c.Consumer.Concurrency = 1 99 | } 100 | if c.Consumer.MinBytes == 0 { 101 | c.Consumer.MinBytes = 1 102 | } 103 | if c.Consumer.MaxBytes == 0 { 104 | c.Consumer.MaxBytes = 1e6 // 1MB 105 | } 106 | if c.Consumer.MaxWait == 0 { 107 | c.Consumer.MaxWait = 10 * time.Second 108 | } 109 | if c.Consumer.CommitInterval == 0 { 110 | c.Consumer.CommitInterval = time.Second 111 | } 112 | if c.Consumer.HeartbeatInterval == 0 { 113 | c.Consumer.HeartbeatInterval = 3 * time.Second 114 | } 115 | if c.Consumer.SessionTimeout == 0 { 116 | c.Consumer.SessionTimeout = 30 * time.Second 117 | } 118 | if c.Consumer.RebalanceTimeout == 0 { 119 | c.Consumer.RebalanceTimeout = 30 * time.Second 120 | } 121 | if c.Consumer.RetentionTime == 0 { 122 | c.Consumer.RetentionTime = 24 * time.Hour 123 | } 124 | if c.Consumer.BackOffStrategy == nil { 125 | c.Consumer.BackOffStrategy = GetBackoffStrategy(FixedBackOffStrategy) 126 | } 127 | if c.Consumer.QueueCapacity == 0 { 128 | c.Consumer.QueueCapacity = 100 129 | } 130 | if c.Producer.BatchSize == 0 { 131 | c.Producer.BatchSize = 100 132 | } 133 | if c.Producer.BatchTimeout == 0 { 134 | c.Producer.BatchTimeout = time.Second 135 | } 136 | } 137 | 138 | func (c *Config) Validate() { 139 | if c.Consumer.GroupID == "" { 140 | panic("you have to set consumer group id") 141 | } 142 | if c.Consumer.Topic == "" { 143 | panic("you have to set topic") 144 | } 145 | if c.Consumer.Cron == "" { 146 | panic("you have to set cron expression") 147 | } 148 | if !isValidBackOffStrategy(c.Consumer.BackOffStrategy) { 149 | panic("you have to set valid backoff strategy") 150 | } 151 | } 152 | 153 | func (o Offset) Value() int64 { 154 | switch o { 155 | case OffsetEarliest: 156 | return segmentio.FirstOffset 157 | case OffsetLatest: 158 | return segmentio.LastOffset 159 | case "": 160 | return segmentio.FirstOffset 161 | default: 162 | offsetValue, err := strconv.ParseInt(string(o), 10, 64) 163 | if err == nil { 164 | return offsetValue 165 | } 166 | return segmentio.FirstOffset 167 | } 168 | } 169 | 170 | func ToStringOffset(offset int64) Offset { 171 | switch offset { 172 | case segmentio.FirstOffset: 173 | return OffsetEarliest 174 | case segmentio.LastOffset: 175 | return OffsetLatest 176 | default: 177 | return OffsetEarliest 178 | } 179 | } 180 | 181 | func isValidBackOffStrategy(strategy BackoffStrategyInterface) bool { 182 | switch strategy.String() { 183 | case ExponentialBackOffStrategy, LinearBackOffStrategy, FixedBackOffStrategy: 184 | return true 185 | default: 186 | return false 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /pkg/kafka/config_test.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | "time" 7 | 8 | "github.com/Trendyol/kafka-cronsumer/pkg/logger" 9 | segmentio "github.com/segmentio/kafka-go" 10 | ) 11 | 12 | func TestConfig_SetDefaults(t *testing.T) { 13 | type fields struct { 14 | Brokers []string 15 | Consumer ConsumerConfig 16 | Producer ProducerConfig 17 | SASL SASLConfig 18 | LogLevel logger.Level 19 | Logger logger.Interface 20 | } 21 | tests := []struct { 22 | name string 23 | fields fields 24 | expected fields 25 | }{ 26 | { 27 | name: "should be set to default values when any value is empty", 28 | expected: fields{ 29 | Consumer: ConsumerConfig{ 30 | MaxRetry: 3, 31 | Concurrency: 1, 32 | MinBytes: 1, 33 | MaxBytes: 1e6, 34 | MaxWait: 10 * time.Second, 35 | CommitInterval: time.Second, 36 | HeartbeatInterval: 3 * time.Second, 37 | SessionTimeout: 30 * time.Second, 38 | RebalanceTimeout: 30 * time.Second, 39 | RetentionTime: 24 * time.Hour, 40 | BackOffStrategy: GetBackoffStrategy(FixedBackOffStrategy), 41 | QueueCapacity: 100, 42 | }, 43 | Producer: ProducerConfig{ 44 | BatchSize: 100, 45 | BatchTimeout: time.Second, 46 | }, 47 | }, 48 | }, 49 | } 50 | for _, tt := range tests { 51 | t.Run(tt.name, func(t *testing.T) { 52 | k := &Config{ 53 | Brokers: tt.fields.Brokers, 54 | Consumer: tt.fields.Consumer, 55 | Producer: tt.fields.Producer, 56 | SASL: tt.fields.SASL, 57 | LogLevel: tt.fields.LogLevel, 58 | Logger: tt.fields.Logger, 59 | } 60 | k.SetDefaults() 61 | 62 | if !reflect.DeepEqual(tt.expected.Consumer, k.Consumer) { 63 | t.Errorf("Expected: %+v, Actual: %+v", tt.expected.Consumer, k.Consumer) 64 | } 65 | if !reflect.DeepEqual(tt.expected.SASL, k.SASL) { 66 | t.Errorf("Expected: %+v, Actual: %+v", tt.expected.SASL, k.SASL) 67 | } 68 | if !reflect.DeepEqual(tt.expected.Producer, k.Producer) { 69 | t.Errorf("Expected: %+v, Actual: %+v", tt.expected.Producer, k.Producer) 70 | } 71 | if !reflect.DeepEqual(tt.expected.Logger, k.Logger) { 72 | t.Errorf("Expected: %+v, Actual: %+v", tt.expected.Logger, k.Logger) 73 | } 74 | if !reflect.DeepEqual(tt.expected.LogLevel, k.LogLevel) { 75 | t.Errorf("Expected: %+v, Actual: %+v", tt.expected.LogLevel, k.LogLevel) 76 | } 77 | if !reflect.DeepEqual(tt.expected.Brokers, k.Brokers) { 78 | t.Errorf("Expected: %+v, Actual: %+v", tt.expected.Brokers, k.Brokers) 79 | } 80 | }) 81 | } 82 | } 83 | 84 | func TestConfig_Validate(t *testing.T) { 85 | type fields struct { 86 | Brokers []string 87 | Consumer ConsumerConfig 88 | Producer ProducerConfig 89 | SASL SASLConfig 90 | LogLevel logger.Level 91 | Logger logger.Interface 92 | } 93 | tests := []struct { 94 | name string 95 | panic string 96 | fields fields 97 | }{ 98 | { 99 | name: "should be throw panic when consumer groupId value is empty", 100 | panic: "you have to set consumer group id", 101 | }, 102 | { 103 | name: "should be throw panic when consumer topic value is empty", 104 | panic: "you have to set topic", 105 | fields: fields{ 106 | Consumer: ConsumerConfig{ 107 | GroupID: "groupId", 108 | BackOffStrategy: GetBackoffStrategy(FixedBackOffStrategy), 109 | }, 110 | }, 111 | }, 112 | { 113 | name: "should be throw panic when consumer cron value is empty", 114 | panic: "you have to set cron expression", 115 | fields: fields{ 116 | Consumer: ConsumerConfig{ 117 | GroupID: "groupId", 118 | Topic: "topic", 119 | BackOffStrategy: GetBackoffStrategy(FixedBackOffStrategy), 120 | }, 121 | }, 122 | }, 123 | { 124 | name: "should be throw panic when consumer duration value is empty", 125 | panic: "you have to set panic duration", 126 | fields: fields{ 127 | Consumer: ConsumerConfig{ 128 | GroupID: "groupId", 129 | Topic: "topic", 130 | Cron: "cron", 131 | BackOffStrategy: GetBackoffStrategy(FixedBackOffStrategy), 132 | }, 133 | }, 134 | }, 135 | { 136 | name: "should be success when consumer topic and groupId value is not empty", 137 | fields: fields{ 138 | Consumer: ConsumerConfig{ 139 | GroupID: "groupId", 140 | Topic: "topic", 141 | Cron: "cron", 142 | Duration: time.Second, 143 | BackOffStrategy: GetBackoffStrategy(FixedBackOffStrategy), 144 | }, 145 | }, 146 | }, 147 | } 148 | for _, tt := range tests { 149 | t.Run(tt.name, func(t *testing.T) { 150 | k := &Config{ 151 | Brokers: tt.fields.Brokers, 152 | Consumer: tt.fields.Consumer, 153 | Producer: tt.fields.Producer, 154 | SASL: tt.fields.SASL, 155 | LogLevel: tt.fields.LogLevel, 156 | Logger: tt.fields.Logger, 157 | } 158 | 159 | defer func() { 160 | if r := recover(); r != nil { 161 | if tt.panic != r || len(tt.panic) == 0 { 162 | t.Errorf("Expected: %+v, Actual: %+v", tt.panic, r) 163 | } 164 | } 165 | }() 166 | 167 | k.Validate() 168 | }) 169 | } 170 | } 171 | 172 | func TestOffset_Value(t *testing.T) { 173 | tests := []struct { 174 | name string 175 | o Offset 176 | want int64 177 | }{ 178 | { 179 | name: "should be returned first offset when the value is equal to earliest", 180 | o: OffsetEarliest, 181 | want: segmentio.FirstOffset, 182 | }, 183 | { 184 | name: "should be returned last offset when the value is equal to latest", 185 | o: OffsetLatest, 186 | want: segmentio.LastOffset, 187 | }, 188 | { 189 | name: "should be returned first offset when the value is empty", 190 | o: "", 191 | want: segmentio.FirstOffset, 192 | }, 193 | { 194 | name: "should be returned input value when the value is valid and other than predefined values", 195 | o: "10", 196 | want: 10, 197 | }, 198 | { 199 | name: "should be returned first offset when the value is not valid", 200 | o: "test", 201 | want: segmentio.FirstOffset, 202 | }, 203 | } 204 | for _, tt := range tests { 205 | t.Run(tt.name, func(t *testing.T) { 206 | actual := tt.o.Value() 207 | if !reflect.DeepEqual(tt.want, actual) { 208 | t.Errorf("Expected: %+v, Actual: %+v", tt.want, actual) 209 | } 210 | }) 211 | } 212 | } 213 | 214 | func TestToStringOffset(t *testing.T) { 215 | type args struct { 216 | offset int64 217 | } 218 | tests := []struct { 219 | name string 220 | args args 221 | want Offset 222 | }{ 223 | { 224 | name: "should return `earliest` when the value is -2", 225 | args: args{offset: -2}, 226 | want: OffsetEarliest, 227 | }, 228 | { 229 | name: "should return `latest` when the value is -1", 230 | args: args{offset: -1}, 231 | want: OffsetLatest, 232 | }, 233 | { 234 | name: "should return `latest` when the value is not equal -2 or -1", 235 | args: args{offset: 0}, 236 | want: OffsetEarliest, 237 | }, 238 | } 239 | for _, tt := range tests { 240 | t.Run(tt.name, func(t *testing.T) { 241 | if got := ToStringOffset(tt.args.offset); got != tt.want { 242 | t.Errorf("ToStringOffset() = %v, want %v", got, tt.want) 243 | } 244 | }) 245 | } 246 | } 247 | 248 | func TestConfig_GetBrokerAddr(t *testing.T) { 249 | t.Run("Should_Return_Default_Broker_Addr_When_Producer_Broker_Not_Given", func(t *testing.T) { 250 | // Given 251 | kafkaConfig := &Config{ 252 | Brokers: []string{"127.0.0.1:9092"}, 253 | Producer: ProducerConfig{}, 254 | } 255 | 256 | // When 257 | result := kafkaConfig.GetBrokerAddr() 258 | 259 | // Then 260 | if result.String() != "127.0.0.1:9092" { 261 | t.Errorf("Expected: 127.0.0.1:9092, Actual: %+v", result.String()) 262 | } 263 | }) 264 | t.Run("Should_Return_Producer_Broker_Addr_When_Its_Given", func(t *testing.T) { 265 | // Given 266 | kafkaConfig := &Config{ 267 | Brokers: []string{"127.0.0.1:9092"}, 268 | Producer: ProducerConfig{ 269 | Brokers: []string{"127.0.0.2:9092"}, 270 | }, 271 | } 272 | 273 | // When 274 | result := kafkaConfig.GetBrokerAddr() 275 | 276 | // Then 277 | if result.String() != "127.0.0.2:9092" { 278 | t.Errorf("Expected: 127.0.0.2:9092, Actual: %+v", result.String()) 279 | } 280 | }) 281 | } 282 | -------------------------------------------------------------------------------- /pkg/kafka/cron_client.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "github.com/Trendyol/kafka-cronsumer/pkg/logger" 5 | "github.com/prometheus/client_golang/prometheus" 6 | ) 7 | 8 | // ConsumeFn function describes how to consume messages from specified topic 9 | type ConsumeFn func(message Message) error 10 | 11 | type Cronsumer interface { 12 | // Start starts the kafka consumer KafkaCronsumer with a new goroutine so its asynchronous operation (non-blocking) 13 | Start() 14 | 15 | // Run runs the kafka consumer KafkaCronsumer with the caller goroutine so its synchronous operation (blocking) 16 | Run() 17 | 18 | // Stop stops the cron and kafka KafkaCronsumer consumer 19 | Stop() 20 | 21 | // WithLogger for injecting custom log implementation 22 | WithLogger(logger logger.Interface) 23 | 24 | // Produce produces the message to kafka KafkaCronsumer producer. Offset and Time fields will be ignored in the message. 25 | Produce(message Message) error 26 | 27 | // ProduceBatch produces the list of messages to kafka KafkaCronsumer producer. 28 | ProduceBatch(messages []Message) error 29 | 30 | // GetMetricCollectors for the purpose of making metric collectors available to other libraries 31 | GetMetricCollectors() []prometheus.Collector 32 | } 33 | -------------------------------------------------------------------------------- /pkg/kafka/cronsumer_message.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | ) 8 | 9 | type Header struct { 10 | Key string 11 | Value []byte 12 | } 13 | 14 | type Headers []Header 15 | 16 | // Pretty Writes every header key and value, it is useful for debugging purpose 17 | func (hs Headers) Pretty() string { 18 | headerStrings := make([]string, len(hs)) 19 | for i := range hs { 20 | headerStrings[i] = fmt.Sprintf("%s: %s", hs[i].Key, string(hs[i].Value)) 21 | } 22 | return strings.Join(headerStrings, ", ") 23 | } 24 | 25 | type Message struct { 26 | Topic string 27 | Partition int 28 | Offset int64 29 | HighWaterMark int64 30 | Key []byte 31 | Value []byte 32 | Headers Headers 33 | Time time.Time 34 | } 35 | 36 | type MessageBuilder struct { 37 | topic *string 38 | key []byte 39 | value []byte 40 | headers Headers 41 | partition *int 42 | highWaterMark *int64 43 | } 44 | 45 | func NewMessageBuilder() *MessageBuilder { 46 | return &MessageBuilder{} 47 | } 48 | 49 | func (mb *MessageBuilder) WithTopic(topic string) *MessageBuilder { 50 | mb.topic = &topic 51 | return mb 52 | } 53 | 54 | func (mb *MessageBuilder) WithKey(key []byte) *MessageBuilder { 55 | mb.key = key 56 | return mb 57 | } 58 | 59 | func (mb *MessageBuilder) WithValue(value []byte) *MessageBuilder { 60 | mb.value = value 61 | return mb 62 | } 63 | 64 | func (mb *MessageBuilder) WithPartition(partition int) *MessageBuilder { 65 | mb.partition = &partition 66 | return mb 67 | } 68 | 69 | func (mb *MessageBuilder) WithHeaders(headers []Header) *MessageBuilder { 70 | mb.headers = headers 71 | return mb 72 | } 73 | 74 | func (mb *MessageBuilder) WithHighWatermark(highWaterMark int64) *MessageBuilder { 75 | mb.highWaterMark = &highWaterMark 76 | return mb 77 | } 78 | 79 | // AddHeader works as a idempotent function 80 | func (m *Message) AddHeader(header Header) { 81 | for i := range m.Headers { 82 | if m.Headers[i].Key == header.Key { 83 | m.Headers[i].Value = header.Value 84 | return 85 | } 86 | } 87 | 88 | m.Headers = append(m.Headers, header) 89 | } 90 | 91 | func (mb *MessageBuilder) Build() Message { 92 | m := Message{} 93 | 94 | if mb.topic != nil { 95 | m.Topic = *mb.topic 96 | } 97 | if mb.key != nil { 98 | m.Key = mb.key 99 | } 100 | if mb.value != nil { 101 | m.Value = mb.value 102 | } 103 | if mb.partition != nil { 104 | m.Partition = *mb.partition 105 | } 106 | if mb.headers != nil { 107 | m.Headers = mb.headers 108 | } 109 | if mb.highWaterMark != nil { 110 | m.HighWaterMark = *mb.highWaterMark 111 | } 112 | 113 | return m 114 | } 115 | -------------------------------------------------------------------------------- /pkg/kafka/cronsumer_message_test.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func Test_Should_Build_Message_With_All_Fields(t *testing.T) { 10 | t.Run("Builds message with all fields", func(t *testing.T) { 11 | // Given 12 | topic := "test-topic" 13 | key := []byte("test-key") 14 | value := []byte("test-value") 15 | partition := 2 16 | headers := []Header{ 17 | {Key: "header1", Value: []byte("value1")}, 18 | {Key: "header2", Value: []byte("value2")}, 19 | } 20 | highWaterMark := int64(100) 21 | 22 | // When 23 | builder := NewMessageBuilder(). 24 | WithTopic(topic). 25 | WithKey(key). 26 | WithValue(value). 27 | WithPartition(partition). 28 | WithHeaders(headers). 29 | WithHighWatermark(highWaterMark) 30 | message := builder.Build() 31 | 32 | // Then 33 | expectedMessage := Message{ 34 | Topic: topic, 35 | Key: key, 36 | Value: value, 37 | Partition: partition, 38 | Headers: headers, 39 | HighWaterMark: highWaterMark, 40 | } 41 | if !reflect.DeepEqual(message, expectedMessage) { 42 | t.Errorf("Expected: %+v, Actual: %+v", expectedMessage, message) 43 | } 44 | }) 45 | 46 | t.Run("Builds message with default values", func(t *testing.T) { 47 | // When 48 | builder := NewMessageBuilder() 49 | message := builder.Build() 50 | 51 | // Then 52 | expectedMessage := Message{} 53 | if !reflect.DeepEqual(message, expectedMessage) { 54 | t.Errorf("Expected: %+v, Actual: %+v", expectedMessage, message) 55 | } 56 | }) 57 | } 58 | 59 | func Test_WithTopic(t *testing.T) { 60 | // Given 61 | expected := "topic" 62 | messageBuilder := MessageBuilder{} 63 | // When 64 | actual := messageBuilder.WithTopic(expected).topic 65 | // Then 66 | if *actual != expected { 67 | t.Errorf("Expected: %s, Actual: %s", expected, *actual) 68 | } 69 | } 70 | 71 | func Test_WithKey(t *testing.T) { 72 | // Given 73 | expected := []byte("1") 74 | messageBuilder := MessageBuilder{} 75 | // When 76 | actual := messageBuilder.WithKey(expected).key 77 | // Then 78 | if !bytes.Equal(expected, actual) { 79 | t.Errorf("Expected: %s, Actual: %s", expected, actual) 80 | } 81 | } 82 | 83 | func Test_WithValue(t *testing.T) { 84 | // Given 85 | expected := []byte("1") 86 | messageBuilder := MessageBuilder{} 87 | // When 88 | actual := messageBuilder.WithValue(expected).value 89 | // Then 90 | if !bytes.Equal(expected, actual) { 91 | t.Errorf("Expected: %s, Actual: %s", expected, actual) 92 | } 93 | } 94 | 95 | func Test_WithPartition(t *testing.T) { 96 | // Given 97 | expected := 1 98 | messageBuilder := MessageBuilder{} 99 | // When 100 | actual := messageBuilder.WithPartition(expected).partition 101 | // Then 102 | if *actual != expected { 103 | t.Errorf("Expected: %d, Actual: %d", expected, *actual) 104 | } 105 | } 106 | 107 | func Test_WithHeaders(t *testing.T) { 108 | // Given 109 | expected := []Header{ 110 | {Key: "x-retry-count", Value: []byte("1")}, 111 | } 112 | messageBuilder := MessageBuilder{} 113 | // When 114 | actual := messageBuilder.WithHeaders(expected).headers 115 | // Then 116 | if !bytes.Equal(actual[0].Value, expected[0].Value) { 117 | t.Errorf("Expected: %s, Actual: %s", expected[0].Value, actual[0].Value) 118 | } 119 | } 120 | 121 | func Test_WithHighWatermark(t *testing.T) { 122 | // Given 123 | expected := int64(1) 124 | messageBuilder := MessageBuilder{} 125 | // When 126 | actual := messageBuilder.WithHighWatermark(expected).highWaterMark 127 | // Then 128 | if *actual != expected { 129 | t.Errorf("Expected: %d, Actual: %d", expected, *actual) 130 | } 131 | } 132 | 133 | func TestMessage_AddHeader(t *testing.T) { 134 | t.Run("When_New_Header_Comes", func(t *testing.T) { 135 | // Given 136 | m := Message{ 137 | Headers: []Header{ 138 | {Key: "foo", Value: []byte("fooValue")}, 139 | }, 140 | } 141 | 142 | // When 143 | m.AddHeader(Header{Key: "bar", Value: []byte("barValue")}) 144 | 145 | // Then 146 | headers := m.Headers 147 | if len(headers) != 2 { 148 | t.Fatalf("Header length must be equal to 2") 149 | } 150 | if headers[1].Key != "bar" { 151 | t.Fatalf("Header key must be equal to bar") 152 | } 153 | if !bytes.Equal(headers[1].Value, []byte("barValue")) { 154 | t.Fatalf("Header value must be equal to barValue") 155 | } 156 | }) 157 | t.Run("When_Same_Header_Comes", func(t *testing.T) { 158 | // Given 159 | m := Message{ 160 | Headers: []Header{ 161 | {Key: "foo", Value: []byte("fooValue")}, 162 | }, 163 | } 164 | 165 | // When 166 | m.AddHeader(Header{Key: "foo", Value: []byte("barValue")}) 167 | 168 | // Then 169 | headers := m.Headers 170 | if len(headers) != 1 { 171 | t.Fatalf("Header length must be equal to 1") 172 | } 173 | if headers[0].Key != "foo" { 174 | t.Fatalf("Header key must be equal to foo") 175 | } 176 | if !bytes.Equal(headers[0].Value, []byte("barValue")) { 177 | t.Fatalf("Header value must be equal to barValue") 178 | } 179 | }) 180 | } 181 | 182 | func TestHeaders_Pretty(t *testing.T) { 183 | // Given 184 | headers := Headers{ 185 | {Key: "key1", Value: []byte("value1")}, 186 | {Key: "key2", Value: []byte("value2")}, 187 | } 188 | 189 | // When 190 | result := headers.Pretty() 191 | 192 | // Then 193 | if result != "key1: value1, key2: value2" { 194 | t.Error("result must be `key1: value1, key2: value2`") 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /pkg/logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | type Level string 4 | 5 | const ( 6 | Debug Level = "debug" 7 | Info Level = "info" 8 | Warn Level = "warn" 9 | Error Level = "error" 10 | ) 11 | 12 | // Interface is a logger that supports log levels, context and structured logging. 13 | type Interface interface { 14 | // With returns a logger based off the root logger and decorates it with the given context and arguments. 15 | With(args ...interface{}) Interface 16 | 17 | // Debug uses fmt.Sprint to construct and log a message at DEBUG level 18 | Debug(args ...interface{}) 19 | // Info uses fmt.Sprint to construct and log a message at INFO level 20 | Info(args ...interface{}) 21 | // Warn uses fmt.Sprint to construct and log a message at ERROR level 22 | Warn(args ...interface{}) 23 | // Error uses fmt.Sprint to construct and log a message at ERROR level 24 | Error(args ...interface{}) 25 | 26 | // Debugf uses fmt.Sprintf to construct and log a message at DEBUG level 27 | Debugf(format string, args ...interface{}) 28 | // Infof uses fmt.Sprintf to construct and log a message at INFO level 29 | Infof(format string, args ...interface{}) 30 | // Warnf uses fmt.Sprintf to construct and log a message at WARN level 31 | Warnf(format string, args ...interface{}) 32 | // Errorf uses fmt.Sprintf to construct and log a message at ERROR level 33 | Errorf(format string, args ...interface{}) 34 | 35 | Infow(msg string, keysAndValues ...interface{}) 36 | Errorw(msg string, keysAndValues ...interface{}) 37 | Warnw(msg string, keysAndValues ...interface{}) 38 | } 39 | -------------------------------------------------------------------------------- /pkg/logger/zap.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "go.uber.org/zap/zapcore" 6 | ) 7 | 8 | type logger struct { 9 | *zap.SugaredLogger 10 | } 11 | 12 | func New(level Level) Interface { 13 | if level == "" { 14 | level = Info 15 | } 16 | 17 | l, _ := newLogger(level) 18 | return &logger{l.Sugar()} 19 | } 20 | 21 | func (l *logger) With(args ...interface{}) Interface { 22 | if len(args) > 0 { 23 | return &logger{l.SugaredLogger.With(args...)} 24 | } 25 | return l 26 | } 27 | 28 | func newLogger(level Level) (*zap.Logger, error) { 29 | encoderConfig := zapcore.EncoderConfig{ 30 | TimeKey: "time", 31 | LevelKey: "level", 32 | NameKey: "logger", 33 | CallerKey: "sourceLocation", 34 | FunctionKey: zapcore.OmitKey, 35 | MessageKey: "message", 36 | StacktraceKey: "stacktrace", 37 | LineEnding: zapcore.DefaultLineEnding, 38 | EncodeLevel: zapcore.CapitalLevelEncoder, 39 | EncodeTime: zapcore.TimeEncoderOfLayout("2006-01-02T15:04:05.999Z"), 40 | EncodeDuration: zapcore.SecondsDurationEncoder, 41 | EncodeCaller: zapcore.ShortCallerEncoder, 42 | } 43 | 44 | // default log level is Info 45 | lvl := zapcore.InfoLevel 46 | _ = lvl.Set(string(level)) 47 | 48 | const initial = 100 49 | config := zap.Config{ 50 | Level: zap.NewAtomicLevelAt(lvl), 51 | Development: false, 52 | Sampling: &zap.SamplingConfig{ 53 | Initial: initial, 54 | Thereafter: initial, 55 | }, 56 | Encoding: "json", 57 | EncoderConfig: encoderConfig, 58 | OutputPaths: []string{"stdout"}, 59 | ErrorOutputPaths: []string{"stderr"}, 60 | } 61 | 62 | return config.Build(zap.AddStacktrace(zap.FatalLevel)) 63 | } 64 | -------------------------------------------------------------------------------- /test/integration/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | redpanda: 4 | image: docker.redpanda.com/redpandadata/redpanda 5 | container_name: redpanda-1 6 | command: 7 | - redpanda 8 | - start 9 | - --smp 10 | - '1' 11 | - --reserve-memory 12 | - 0M 13 | - --overprovisioned 14 | - --node-id 15 | - '0' 16 | - --kafka-addr 17 | - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092 18 | - --advertise-kafka-addr 19 | - PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092 20 | - --pandaproxy-addr 21 | - PLAINTEXT://0.0.0.0:28082,OUTSIDE://0.0.0.0:8082 22 | - --advertise-pandaproxy-addr 23 | - PLAINTEXT://redpanda:28082,OUTSIDE://localhost:8082 24 | ports: 25 | - 8081:8081 26 | - 8082:8082 27 | - 9092:9092 28 | - 28082:28082 29 | - 29092:29092 -------------------------------------------------------------------------------- /test/integration/go.mod: -------------------------------------------------------------------------------- 1 | module integration-test-example 2 | 3 | go 1.19 4 | 5 | replace github.com/Trendyol/kafka-consumer => ../.. 6 | 7 | require ( 8 | github.com/Trendyol/kafka-cronsumer v0.0.0-00010101000000-000000000000 9 | github.com/segmentio/kafka-go v0.4.42 10 | ) 11 | 12 | require ( 13 | github.com/beorn7/perks v1.0.1 // indirect 14 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 15 | github.com/golang/protobuf v1.5.3 // indirect 16 | github.com/klauspost/compress v1.16.4 // indirect 17 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 18 | github.com/pierrec/lz4/v4 v4.1.17 // indirect 19 | github.com/prometheus/client_golang v1.16.0 // indirect 20 | github.com/prometheus/client_model v0.3.0 // indirect 21 | github.com/prometheus/common v0.42.0 // indirect 22 | github.com/prometheus/procfs v0.10.1 // indirect 23 | github.com/robfig/cron/v3 v3.0.1 // indirect 24 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 25 | github.com/xdg-go/scram v1.1.2 // indirect 26 | github.com/xdg-go/stringprep v1.0.4 // indirect 27 | go.uber.org/atomic v1.7.0 // indirect 28 | go.uber.org/multierr v1.6.0 // indirect 29 | go.uber.org/zap v1.24.0 // indirect 30 | golang.org/x/sys v0.8.0 // indirect 31 | golang.org/x/text v0.7.0 // indirect 32 | google.golang.org/protobuf v1.30.0 // indirect 33 | ) 34 | 35 | replace github.com/Trendyol/kafka-cronsumer => ../../. 36 | -------------------------------------------------------------------------------- /test/integration/go.sum: -------------------------------------------------------------------------------- 1 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= 2 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 3 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 4 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 5 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 8 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 10 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 11 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 12 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 13 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 14 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 15 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 16 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 17 | github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= 18 | github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 19 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 20 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 21 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 22 | github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= 23 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 24 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 25 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 26 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 27 | github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= 28 | github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= 29 | github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= 30 | github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= 31 | github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= 32 | github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= 33 | github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= 34 | github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= 35 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 36 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 37 | github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= 38 | github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= 39 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 40 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 41 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 42 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 43 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 44 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 45 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 46 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 47 | github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= 48 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 49 | github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= 50 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 51 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 52 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 53 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 54 | go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= 55 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 56 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 57 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 58 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 59 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 60 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 61 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 62 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 63 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 64 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 65 | golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= 66 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 67 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 68 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 69 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 70 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 71 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 72 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 73 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 74 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 75 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 76 | golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= 77 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 78 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 79 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 80 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 81 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 82 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 83 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 84 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 85 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 86 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 87 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 88 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 89 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 90 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 91 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 92 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 93 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 94 | google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= 95 | google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 96 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 97 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 98 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 99 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 100 | -------------------------------------------------------------------------------- /testdata/message.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 7000, 3 | "item_number": 1240, 4 | "ratio": 0.8 5 | } --------------------------------------------------------------------------------