├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── dependabot.yml └── workflows │ ├── build.yml │ ├── release.yml │ └── scorecard.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── config └── config.go ├── connector.go ├── docs └── production_tutorial.md ├── elasticsearch ├── action.go ├── bulk │ ├── bulk.go │ ├── metric.go │ └── option.go ├── client │ ├── client.go │ ├── fasthttp_transport.go │ └── logger.go └── response_handler.go ├── example ├── script-update │ ├── README.md │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ └── main.go └── simple │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ └── main.go ├── go.mod ├── go.sum ├── handler.go ├── internal ├── bytes │ ├── bytes.go │ ├── escape.go │ ├── escape_test.go │ ├── reader.go │ └── reader_test.go └── slices │ └── slices.go ├── message.go └── option.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 16 | 1. Do the following '...' 17 | 2. Run the project with '....' 18 | 3. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Version (please complete the following information):** 27 | 28 | - OS: [e.g. macOS] 29 | - Golang version [e.g. 1.17] 30 | - Postgres version 31 | 32 | **Additional context** 33 | Add any other context about the problem here. 34 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | open-pull-requests-limit: 10 8 | commit-message: 9 | prefix: "chore: update version" 10 | allow: 11 | - dependency-name: github.com/jackc/pgx/v5 12 | - dependency-name: github.com/lib/pq 13 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | types: [opened, reopened, synchronize] 9 | workflow_dispatch: 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | build: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | with: 22 | fetch-depth: 1 23 | 24 | - name: Set up Go 25 | uses: actions/setup-go@v4 26 | with: 27 | go-version: '1.22.4' 28 | 29 | - name: Lint 30 | run: | 31 | make lint 32 | 33 | - name: Install dependencies 34 | run: | 35 | make tidy 36 | 37 | - name: Build 38 | run: | 39 | make build/linux 40 | 41 | - name: Unit Test 42 | run: go test -v ./... 43 | 44 | security-gates: 45 | uses: Trendyol/security-actions/.github/workflows/security-gates.yml@master 46 | needs: build 47 | permissions: 48 | actions: read 49 | contents: read 50 | security-events: write 51 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v4 14 | with: 15 | fetch-depth: 0 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@v4 19 | with: 20 | go-version: '1.22.4' 21 | 22 | - name: Run GoReleaser 23 | uses: goreleaser/goreleaser-action@v4 24 | with: 25 | version: latest 26 | args: release --clean 27 | env: 28 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/scorecard.yml: -------------------------------------------------------------------------------- 1 | 2 | name: Scorecard supply-chain security 3 | 4 | on: 5 | branch_protection_rule: 6 | schedule: 7 | - cron: '29 23 * * 3' 8 | push: 9 | branches: [ "main", "master"] 10 | pull_request: 11 | branches: ["main", "master"] 12 | 13 | permissions: read-all 14 | 15 | jobs: 16 | visibility-check: 17 | # Bu job, deponun public/private olduğunu belirler 18 | outputs: 19 | visibility: ${{ steps.drv.outputs.visibility }} 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Determine repository visibility 23 | id: drv 24 | run: | 25 | visibility=$(gh api /repos/$GITHUB_REPOSITORY --jq '.visibility') 26 | echo "visibility=$visibility" >> $GITHUB_OUTPUT 27 | env: 28 | GH_TOKEN: ${{ github.token }} 29 | 30 | analysis: 31 | if: ${{ needs.visibility-check.outputs.visibility == 'public' }} 32 | needs: visibility-check 33 | runs-on: ubuntu-latest 34 | permissions: 35 | security-events: write 36 | id-token: write 37 | steps: 38 | - name: "Checkout code" 39 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 40 | with: 41 | persist-credentials: false 42 | 43 | - name: "Run analysis" 44 | uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 45 | with: 46 | results_file: results.sarif 47 | results_format: sarif 48 | publish_results: true 49 | 50 | - name: "Upload artifact" 51 | uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 52 | with: 53 | name: SARIF file 54 | path: results.sarif 55 | retention-days: 5 56 | 57 | # Upload the results to GitHub's code scanning dashboard (optional). 58 | # Commenting out will disable upload of results to your repo's Code Scanning dashboard 59 | - name: "Upload to code-scanning" 60 | uses: github/codeql-action/upload-sarif@f6091c0113d1dcf9b98e269ee48e8a7e51b7bdd4 # v3.28.5 61 | with: 62 | sarif_file: results.sarif 63 | 64 | 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | vendor 3 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | funlen: 3 | lines: 70 4 | 5 | linters: 6 | disable-all: true 7 | enable: 8 | - bodyclose 9 | - dogsled 10 | - dupl 11 | - errcheck 12 | - errorlint 13 | - exportloopref 14 | - funlen 15 | - gocheckcompilerdirectives 16 | - gochecknoinits 17 | - goconst 18 | - gocritic 19 | - gocyclo 20 | - godox 21 | - gofmt 22 | - goimports 23 | - goprintffuncname 24 | - gosec 25 | - gosimple 26 | - govet 27 | - ineffassign 28 | - misspell 29 | - nakedret 30 | - noctx 31 | - nolintlint 32 | - revive 33 | - staticcheck 34 | - stylecheck 35 | - testifylint 36 | - unconvert 37 | - unparam 38 | - unused 39 | - whitespace 40 | 41 | issues: 42 | exclude-rules: 43 | - path: (.+)_test.go 44 | linters: 45 | - funlen 46 | - goconst 47 | - dupl 48 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: go-pq-cdc-elasticsearch 2 | 3 | release: 4 | github: 5 | name: go-pq-cdc-elasticsearch 6 | owner: Trendyol 7 | 8 | before: 9 | hooks: 10 | - go mod tidy 11 | 12 | builds: 13 | - skip: true 14 | 15 | changelog: 16 | sort: asc 17 | use: github 18 | filters: 19 | exclude: 20 | - '^test:' 21 | - '^docs:' 22 | - '^chore:' 23 | - 'merge conflict' 24 | - Merge pull request 25 | - Merge remote-tracking branch 26 | - Merge branch 27 | - go mod tidy -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contribution Guidelines 2 | 3 | Thank you for your interest in go-pq-cdc-elasticsearch! 4 | 5 | This project welcomes contributions and suggestions. Most contributions require you to signoff on your commits. Please 6 | follow the instructions provided below. 7 | 8 | Contributions come in many forms: submitting issues, writing code, participating in discussions and community calls. 9 | 10 | This document provides the guidelines for how to contribute to the project. 11 | 12 | ### Issues 13 | 14 | This section describes the guidelines for submitting issues 15 | 16 | **Issue Types** 17 | 18 | There are 2 types of issues: 19 | 20 | - Issue/Bug: You've found a bug with the code, and want to report it, or create an issue to track the bug. 21 | - Issue/Feature: You have something on your mind, which requires input form others in a discussion, before it eventually 22 | manifests as a proposal. 23 | 24 | ### Before You File 25 | 26 | Before you file an issue, make sure you've checked the following: 27 | 28 | 1. Check for existing issues 29 | Before you create a new issue, please do a search in open issues to see if the issue or feature request has already 30 | been filed. 31 | 32 | If you find your issue already exists, make relevant comments and add your reaction. Use a reaction: 33 | 👍 up-vote 34 | 👎 down-vote 35 | 36 | 2. For bugs 37 | Check it's not an environment issue. For example, if your configurations correct or network connections is alive. 38 | 39 | ### Contributing to go-pq-cdc-elasticsearch 40 | 41 | Pull Requests 42 | All contributions come through pull requests. To submit a proposed change, we recommend following this workflow: 43 | 44 | - Make sure there's an issue (bug or feature) raised, which sets the expectations for the contribution you are about to 45 | make. 46 | - Fork the relevant repo and create a new branch 47 | - Create your change 48 | - Code changes require tests 49 | - Update relevant documentation for the change 50 | - Commit sign-off and open a PR 51 | - Wait for the CI process to finish and make sure all checks are green 52 | - A maintainer of the project will be assigned, and you can expect a review within a few days 53 | 54 | ### Use work-in-progress PRs for early feedback 55 | 56 | A good way to communicate before investing too much time is to create a "Work-in-progress" PR and share it with your 57 | reviewers. The standard way of doing this is to add a "[WIP]" prefix in your PR's title and assign the do-not-merge 58 | label. This will let people looking at your PR know that it is not well baked yet. 59 | 60 | ### Developer Certificate of Origin: Signing your work 61 | 62 | **Every commit needs to be signed** 63 | 64 | The Developer Certificate of Origin (DCO) is a lightweight way for contributors to certify that they wrote or otherwise 65 | have the right to submit the code they are contributing to the project. Here is the full text of the DCO, reformatted 66 | for readability: 67 | 68 | By making a contribution to this project, I certify that: 69 | 70 | (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or 71 | 72 | (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or 73 | 74 | (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. 75 | 76 | (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. 77 | 78 | Contributors sign-off that they adhere to these requirements by adding a Signed-off-by line to commit messages. 79 | 80 | This is my commit message 81 | 82 | Signed-off-by: Random X Developer 83 | 84 | Git even has a -s command line option to append this automatically to your commit message: 85 | 86 | $ git commit -s -m 'This is my commit message' 87 | 88 | Each Pull Request is checked whether or not commits in a Pull Request do contain a valid Signed-off-by line. 89 | 90 | I didn't sign my commit, now what?! 91 | No worries - You can easily replay your changes, sign them and force push them! 92 | 93 | git checkout 94 | git commit --amend --no-edit --signoff 95 | git push --force-with-lease 96 | 97 | ### Use of Third-party code 98 | 99 | - Third-party code must include licenses. 100 | 101 | **Thank You!** - Your contributions to open source, large or small, make projects like this possible. Thank you for 102 | taking the time to contribute. 103 | 104 | ### Code of Conduct 105 | 106 | This project has adopted the Contributor Covenant Code of Conduct -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Trendyol 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: init 2 | 3 | .PHONY: init 4 | init: init/lint 5 | 6 | .PHONY: init/lint init/vulnCheck 7 | init/lint: 8 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.1 9 | go install golang.org/x/tools/go/analysis/passes/fieldalignment/cmd/fieldalignment@v0.22.0 10 | 11 | .PHONY: init/vulnCheck 12 | init/vulnCheck: 13 | go install golang.org/x/vuln/cmd/govulncheck@latest 14 | 15 | .PHONY: audit 16 | audit: 17 | @echo 'Formatting code...' 18 | fieldalignment -fix ./... 19 | golangci-lint run -c .golangci.yml --timeout=5m -v --fix 20 | @echo 'Vetting code...' 21 | go vet ./... 22 | @echo 'Vulnerability scanning...' 23 | govulncheck ./... 24 | 25 | .PHONY: tidy 26 | tidy: 27 | @echo 'Tidying and verifying module dependencies...' 28 | go mod tidy -compat=1.22.4 29 | go mod verify 30 | 31 | .PHONY: tidy/all 32 | tidy/all: 33 | go mod tidy 34 | cd example/simple && go mod tidy && cd ../.. 35 | 36 | .PHONY: test/integration 37 | test/integration: 38 | cd integration_test && go test -race -p=1 -v ./... 39 | 40 | .PHONY: lint 41 | lint: init/lint 42 | @echo 'Formatting code...' 43 | fieldalignment -fix ./... 44 | golangci-lint run -c .golangci.yml --timeout=5m -v --fix 45 | 46 | .PHONY: build 47 | build/linux: 48 | GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -trimpath -a -v -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-pq-cdc-elasticsearch [![Go Reference](https://pkg.go.dev/badge/github.com/Trendyol/go-dcp.svg)](https://pkg.go.dev/github.com/Trendyol/go-pq-cdc-elasticsearch) [![Go Report Card](https://goreportcard.com/badge/github.com/Trendyol/go-pq-cdc-elasticsearch)](https://goreportcard.com/report/github.com/Trendyol/go-pq-cdc-elasticsearch) [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/Trendyol/go-pq-cdc-elasticsearch/badge)](https://scorecard.dev/viewer/?uri=github.com/Trendyol/go-pq-cdc-elasticsearch) 2 | 3 | Elasticsearch connector for [go-pq-cdc](https://github.com/Trendyol/go-pq-cdc). 4 | 5 | go-pq-cdc-elasticsearch streams documents from PostgreSql and writes to Elasticsearch index in near real-time. 6 | 7 | ### Contents 8 | 9 | * [Usage](#usage) 10 | * [Examples](#examples) 11 | * [Availability](#availability) 12 | * [Configuration](#configuration) 13 | * [API](#api) 14 | * [Exposed Metrics](#exposed-metrics) 15 | * [Compatibility](#compatibility) 16 | * [Breaking Changes](#breaking-changes) 17 | 18 | ### Usage 19 | 20 | > ### ⚠️ For production usage check the [production tutorial](./docs/production_tutorial.md) doc 21 | 22 | > ### ⚠️ For other usages check the dockerfile and code at [examples](./example). 23 | 24 | ```sh 25 | go get github.com/Trendyol/go-pq-cdc-elasticsearch 26 | ``` 27 | 28 | ```go 29 | package main 30 | 31 | import ( 32 | "context" 33 | "encoding/json" 34 | cdc "github.com/Trendyol/go-pq-cdc-elasticsearch" 35 | "github.com/Trendyol/go-pq-cdc-elasticsearch/config" 36 | "github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch" 37 | cdcconfig "github.com/Trendyol/go-pq-cdc/config" 38 | "github.com/Trendyol/go-pq-cdc/pq/publication" 39 | "github.com/Trendyol/go-pq-cdc/pq/slot" 40 | "log/slog" 41 | "os" 42 | "strconv" 43 | "time" 44 | ) 45 | 46 | func main() { 47 | slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) 48 | ctx := context.TODO() 49 | cfg := config.Config{ 50 | CDC: cdcconfig.Config{ 51 | Host: "127.0.0.1", 52 | Username: "es_cdc_user", 53 | Password: "es_cdc_pass", 54 | Database: "es_cdc_db", 55 | DebugMode: false, 56 | Publication: publication.Config{ 57 | Name: "es_cdc_publication", 58 | CreateIfNotExists: true, 59 | Operations: publication.Operations{ 60 | publication.OperationInsert, 61 | publication.OperationDelete, 62 | publication.OperationTruncate, 63 | publication.OperationUpdate, 64 | }, 65 | Tables: publication.Tables{publication.Table{ 66 | Name: "users", 67 | ReplicaIdentity: publication.ReplicaIdentityFull, 68 | }}, 69 | }, 70 | Slot: slot.Config{ 71 | Name: "es_cdc_slot", 72 | CreateIfNotExists: true, 73 | SlotActivityCheckerInterval: 3000, 74 | }, 75 | Metric: cdcconfig.MetricConfig{ 76 | Port: 8081, 77 | }, 78 | }, 79 | Elasticsearch: config.Elasticsearch{ 80 | BatchSizeLimit: 10000, 81 | BatchTickerDuration: time.Millisecond * 100, 82 | TableIndexMapping: map[string]string{ 83 | "public.users": "users", 84 | }, 85 | TypeName: "_doc", 86 | URLs: []string{"http://127.0.0.1:9200"}, 87 | }, 88 | } 89 | 90 | connector, err := cdc.NewConnector(ctx, cfg, Handler) 91 | if err != nil { 92 | slog.Error("new connector", "error", err) 93 | os.Exit(1) 94 | } 95 | 96 | defer connector.Close() 97 | connector.Start(ctx) 98 | } 99 | 100 | func Handler(msg cdc.Message) []elasticsearch.Action { 101 | switch msg.Type { 102 | case cdc.InsertMessage: 103 | b, _ := json.Marshal(msg.NewData) 104 | return []elasticsearch.Action{ 105 | elasticsearch.NewIndexAction([]byte(strconv.Itoa(int(msg.NewData["id"].(int32)))), b, nil), 106 | } 107 | case cdc.DeleteMessage: 108 | return []elasticsearch.Action{ 109 | elasticsearch.NewDeleteAction([]byte(strconv.Itoa(int(msg.OldData["id"].(int32)))), nil), 110 | } 111 | case cdc.UpdateMessage: 112 | msg.NewData["old_name"] = msg.OldData["name"] 113 | b, _ := json.Marshal(msg.NewData) 114 | return []elasticsearch.Action{ 115 | elasticsearch.NewIndexAction([]byte(strconv.Itoa(int(msg.NewData["id"].(int32)))), b, nil), 116 | } 117 | default: 118 | return nil 119 | } 120 | } 121 | 122 | 123 | ``` 124 | 125 | ### Examples 126 | 127 | * [Simple](./example/simple) 128 | 129 | ### Availability 130 | 131 | The go-pq-cdc operates in passive/active modes for PostgreSQL change data capture (CDC). Here's how it ensures 132 | availability: 133 | 134 | * **Active Mode:** When the PostgreSQL replication slot (slot.name) is active, go-pq-cdc continuously monitors changes 135 | and streams them to downstream systems as configured. 136 | * **Passive Mode:** If the PostgreSQL replication slot becomes inactive (detected via slot.slotActivityCheckerInterval), 137 | go-pq-cdc automatically captures the slot again and resumes data capturing. Other deployments also monitor slot 138 | activity, 139 | and when detected as inactive, they initiate data capturing. 140 | 141 | This setup ensures continuous data synchronization and minimal downtime in capturing database changes. 142 | 143 | ### Configuration 144 | 145 | | Variable | Type | Required | Default | Description | Options | 146 | |----------------------------------------------------------------------------------------------|:-----------------:|:-----------------------------:|:-------:|-------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 147 | | `cdc.host` | string | yes | - | PostgreSQL host | Should be a valid hostname or IP address. Example: `localhost`. | 148 | | `cdc.username` | string | yes | - | PostgreSQL username | Should have sufficient privileges to perform required database operations. | 149 | | `cdc.password` | string | yes | - | PostgreSQL password | Keep secure and avoid hardcoding in the source code. | 150 | | `cdc.database` | string | yes | - | PostgreSQL database | The database must exist and be accessible by the specified user. | 151 | | `cdc.debugMode` | bool | no | false | For debugging purposes | Enables pprof for trace. | 152 | | `cdc.metric.port` | int | no | 8080 | Set API port | Choose a port that is not in use by other applications. | 153 | | `cdc.logger.logLevel` | string | no | info | Set logging level | [`DEBUG`, `WARN`, `INFO`, `ERROR`] | 154 | | `cdc.logger.logger` | Logger | no | slog | Set logger | Can be customized with other logging frameworks if `slog` is not used. | 155 | | `cdc.publication.createIfNotExists` | bool | no | - | Create publication if not exists. Otherwise, return `publication is not exists` error. | | 156 | | `cdc.publication.name` | string | yes | - | Set PostgreSQL publication name | Should be unique within the database. | 157 | | `cdc.publication.operations` | []string | yes | - | Set PostgreSQL publication operations. List of operations to track; all or a subset can be specified. | **INSERT:** Track insert operations.
**UPDATE:** Track update operations.
**DELETE:** Track delete operations. | 158 | | `cdc.publication.tables` | []Table | yes | - | Set tables which are tracked by data change capture | Define multiple tables as needed. | 159 | | `cdc.publication.tables[i].name` | string | yes | - | Set the data change captured table name | Must be a valid table name in the specified database. | 160 | | `cdc.publication.tables[i].replicaIdentity` | string | yes | - | Set the data change captured table replica identity [`FULL`, `DEFAULT`] | **FULL:** Captures all columns of old row when a row is updated or deleted.
**DEFAULT:** Captures only the primary key of old row when a row is updated or deleted. | 161 | | `publication.tables[i].schema` | string | no | public | Set the data change captured table schema name | Must be a valid table name in the specified database. | 162 | | `cdc.slot.createIfNotExists` | bool | no | - | Create replication slot if not exists. Otherwise, return `replication slot is not exists` error. | | 163 | | `cdc.slot.name` | string | yes | - | Set the logical replication slot name | Should be unique and descriptive. | 164 | | `cdc.slot.slotActivityCheckerInterval` | int | yes | 1000 | Set the slot activity check interval time in milliseconds | Specify as an integer value in milliseconds (e.g., `1000` for 1 second). | 165 | | `elasticsearch.username` | string | no (yes, if the auth enabled) | - | The username for authenticating to Elasticsearch. | Maps table names to Elasticsearch indices. | 166 | | `elasticsearch.password` | string | no (yes, if the auth enabled) | - | The password associated with the elasticsearch.username for authenticating to Elasticsearch. | Maps table names to Elasticsearch indices. | 167 | | `elasticsearch.tableIndexMapping` | map[string]string | yes | - | Mapping of PostgreSQL table events to Elasticsearch indices | Maps table names to Elasticsearch indices. | 168 | | `elasticsearch.urls` | []string | yes | - | Elasticsearch connection URLs | List of URLs to connect to Elasticsearch instances. | 169 | | `elasticsearch.batchSizeLimit` | int | no | 1000 | Maximum message count per batch | Flush is triggered if this limit is exceeded. | 170 | | `elasticsearch.batchTickerDuration` | time.Duration | no | 10 sec | Automatic batch flush interval | Specify in a human-readable format, e.g., `10s` for 10 seconds. | 171 | | `elasticsearch.batchByteSizeLimit` | string | no | 10mb | Maximum size (bytes) per batch | Flush is triggered if this size is exceeded. | 172 | | `elasticsearch.maxConnsPerHost` | int | no | 512 | Maximum connections per host | Limits connections to each Elasticsearch host. | 173 | | `elasticsearch.maxIdleConnDuration` | time.Duration | no | 10 sec | Duration to keep idle connections alive | Specify in a human-readable format, e.g., `10s` for 10 seconds. | 174 | | `elasticsearch.typeName` | string | no | - | Elasticsearch index type name | Typically used in Elasticsearch for index types. | 175 | | `elasticsearch.concurrentRequest` | int | no | 1 | Number of concurrent bulk requests | Specify the count of bulk requests that can run concurrently. | 176 | | `elasticsearch.compressionEnabled` | bool | no | false | Enable compression for large messages | Useful if message sizes are large, but may increase CPU usage. | 177 | | `elasticsearch.disableDiscoverNodesOnStart` | bool | no | false | Disable node discovery on client initialization | Skips node discovery when the client starts. | 178 | | `elasticsearch.discoverNodesInterval` | time.Duration | no | 5 min | Periodic node discovery interval | Specify in a human-readable format, e.g., `5m` for 5 minutes. | 179 | | `elasticsearch.version` | string | no | - | Elasticsearch version to determine compatibility features | Used to handle version-specific behaviors, such as `_type` parameter support (removed in ES 8.0+). If not specified, version is automatically detected from the cluster. | 180 | 181 | ### API 182 | 183 | | Endpoint | Description | 184 | |----------------------|-------------------------------------------------------------------------------------------| 185 | | `GET /status` | Returns a 200 OK status if the client is able to ping the PostgreSQL server successfully. | 186 | | `GET /metrics` | Prometheus metric endpoint. | 187 | | `GET /debug/pprof/*` | (Only for `debugMode=true`) [pprof](https://pkg.go.dev/net/http/pprof) | 188 | 189 | ### Exposed Metrics 190 | 191 | The client collects relevant metrics related to PostgreSQL change data capture (CDC) and makes them available at 192 | the `/metrics` endpoint. 193 | 194 | | Metric Name | Description | Labels | Value Type | 195 | |--------------------------------------------------------------|---------------------------------------------------------------------------------|-----------------------------|------------| 196 | | go_pq_cdc_elasticsearch_process_latency_current | The latest elasticsearch connector process latency in nanoseconds. | slot_name, host | Gauge | 197 | | go_pq_cdc_elasticsearch_bulk_request_process_latency_current | The latest elasticsearch connector bulk request process latency in nanoseconds. | slot_name, host | Gauge | 198 | | go_pq_cdc_elasticsearch_index_total | Total number of index operation. | slot_name, host, index_name | Counter | 199 | | go_pq_cdc_elasticsearch_delete_total | Total number of delete operation. | slot_name, host, index_name | Counter | 200 | 201 | You can also use all cdc related metrics explained [here](https://github.com/Trendyol/go-pq-cdc#exposed-metrics). 202 | All cdc related metrics are automatically injected. It means you don't need to do anything. 203 | 204 | ### Compatibility 205 | 206 | | go-pq-cdc Version | Minimum PostgreSQL Server Version | 207 | |-------------------|-----------------------------------| 208 | | 0.0.2 or higher | 14 | 209 | 210 | ### Elasticsearch Version Compatibility 211 | 212 | The connector supports different versions of Elasticsearch through the `elasticsearch.version` configuration parameter: 213 | 214 | | Elasticsearch Version | Type Parameter Behavior | 215 | |-----------------------|--------------------------------------------------------| 216 | | Below 8.0 | `_type` parameter is included in the index requests | 217 | | 8.0 and above | `_type` parameter is automatically omitted | 218 | 219 | If no version is specified, the connector will automatically detect the Elasticsearch cluster version by querying the Info API after connection. This eliminates the need to manually configure the version. 220 | 221 | ### Breaking Changes 222 | 223 | | Date taking effect | Version | Change | How to check | 224 | |--------------------|---------|--------|--------------| 225 | | - | - | - | - | 226 | 227 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/Trendyol/go-pq-cdc/config" 7 | ) 8 | 9 | type Elasticsearch struct { 10 | Username string `yaml:"username"` 11 | Password string `yaml:"password"` 12 | BatchByteSizeLimit string `yaml:"batchByteSizeLimit"` 13 | TableIndexMapping map[string]string `yaml:"tableIndexMapping"` 14 | MaxConnsPerHost *int `yaml:"maxConnsPerHost"` 15 | MaxIdleConnDuration *time.Duration `yaml:"maxIdleConnDuration"` 16 | DiscoverNodesInterval *time.Duration `yaml:"discoverNodesInterval"` 17 | TypeName string `yaml:"typeName"` 18 | Version string `yaml:"version"` 19 | URLs []string `yaml:"urls"` 20 | BatchSizeLimit int `yaml:"batchSizeLimit"` 21 | BatchTickerDuration time.Duration `yaml:"batchTickerDuration"` 22 | ConcurrentRequest int `yaml:"concurrentRequest"` 23 | CompressionEnabled bool `yaml:"compressionEnabled"` 24 | DisableDiscoverNodesOnStart bool `yaml:"disableDiscoverNodesOnStart"` 25 | } 26 | 27 | type RejectionLog struct { 28 | Index string `yaml:"index"` 29 | IncludeSource bool `yaml:"includeSource"` 30 | } 31 | 32 | type Config struct { 33 | CDC config.Config 34 | Elasticsearch Elasticsearch 35 | } 36 | 37 | func (c *Config) SetDefault() { 38 | if c.Elasticsearch.BatchTickerDuration == 0 { 39 | c.Elasticsearch.BatchTickerDuration = 10 * time.Second 40 | } 41 | 42 | if c.Elasticsearch.BatchSizeLimit == 0 { 43 | c.Elasticsearch.BatchSizeLimit = 1000 44 | } 45 | 46 | if c.Elasticsearch.BatchByteSizeLimit == "" { 47 | c.Elasticsearch.BatchByteSizeLimit = "10mb" 48 | } 49 | 50 | if c.Elasticsearch.ConcurrentRequest == 0 { 51 | c.Elasticsearch.ConcurrentRequest = 1 52 | } 53 | 54 | if c.Elasticsearch.DiscoverNodesInterval == nil { 55 | duration := 5 * time.Minute 56 | c.Elasticsearch.DiscoverNodesInterval = &duration 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /connector.go: -------------------------------------------------------------------------------- 1 | package cdc 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "sync" 8 | 9 | "github.com/Trendyol/go-pq-cdc/pq/timescaledb" 10 | 11 | cdc "github.com/Trendyol/go-pq-cdc" 12 | "github.com/Trendyol/go-pq-cdc-elasticsearch/config" 13 | "github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch" 14 | "github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch/bulk" 15 | "github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch/client" 16 | "github.com/Trendyol/go-pq-cdc-elasticsearch/internal/slices" 17 | "github.com/Trendyol/go-pq-cdc/logger" 18 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 19 | "github.com/Trendyol/go-pq-cdc/pq/replication" 20 | es "github.com/elastic/go-elasticsearch/v7" 21 | "github.com/go-playground/errors" 22 | "github.com/prometheus/client_golang/prometheus" 23 | ) 24 | 25 | type Connector interface { 26 | Start(ctx context.Context) 27 | Close() 28 | } 29 | 30 | type connector struct { 31 | partitionCache sync.Map 32 | handler Handler 33 | responseHandler elasticsearch.ResponseHandler 34 | cfg *config.Config 35 | esClient *es.Client 36 | cdc cdc.Connector 37 | bulk bulk.Indexer 38 | metrics []prometheus.Collector 39 | } 40 | 41 | func NewConnector(ctx context.Context, cfg config.Config, handler Handler, options ...Option) (Connector, error) { 42 | cfg.SetDefault() 43 | 44 | esConnector := &connector{ 45 | cfg: &cfg, 46 | handler: handler, 47 | } 48 | 49 | Options(options).Apply(esConnector) 50 | 51 | pqCDC, err := cdc.NewConnector(ctx, esConnector.cfg.CDC, esConnector.listener) 52 | if err != nil { 53 | return nil, err 54 | } 55 | esConnector.cdc = pqCDC 56 | esConnector.cfg.CDC = *pqCDC.GetConfig() 57 | 58 | esClient, err := client.NewClient(esConnector.cfg) 59 | if err != nil { 60 | return nil, errors.Wrap(err, "elasticsearch new client") 61 | } 62 | esConnector.esClient = esClient 63 | 64 | esConnector.bulk, err = bulk.NewBulk( 65 | esConnector.cfg, 66 | esClient, 67 | pqCDC, 68 | bulk.WithResponseHandler(esConnector.responseHandler), 69 | ) 70 | if err != nil { 71 | return nil, errors.Wrap(err, "elasticsearch new bulk") 72 | } 73 | pqCDC.SetMetricCollectors(esConnector.bulk.GetMetric().PrometheusCollectors()...) 74 | pqCDC.SetMetricCollectors(esConnector.metrics...) 75 | 76 | return esConnector, nil 77 | } 78 | 79 | func (c *connector) Start(ctx context.Context) { 80 | go func() { 81 | logger.Info("waiting for connector start...") 82 | if err := c.cdc.WaitUntilReady(ctx); err != nil { 83 | panic(err) 84 | } 85 | logger.Info("bulk process started") 86 | c.bulk.StartBulk() 87 | }() 88 | c.cdc.Start(ctx) 89 | } 90 | 91 | func (c *connector) Close() { 92 | c.cdc.Close() 93 | c.bulk.Close() 94 | } 95 | 96 | func (c *connector) listener(ctx *replication.ListenerContext) { 97 | var msg Message 98 | switch m := ctx.Message.(type) { 99 | case *format.Insert: 100 | msg = NewInsertMessage(c.esClient, m) 101 | case *format.Update: 102 | msg = NewUpdateMessage(c.esClient, m) 103 | case *format.Delete: 104 | msg = NewDeleteMessage(c.esClient, m) 105 | default: 106 | return 107 | } 108 | 109 | fullTableName := c.getFullTableName(msg.TableNamespace, msg.TableName) 110 | 111 | indexName := c.resolveTableToIndexName(fullTableName, msg.TableNamespace, msg.TableName) 112 | if indexName == "" { 113 | if err := ctx.Ack(); err != nil { 114 | logger.Error("ack", "error", err) 115 | } 116 | return 117 | } 118 | 119 | actions := c.handler(msg) 120 | if len(actions) == 0 { 121 | if err := ctx.Ack(); err != nil { 122 | logger.Error("ack", "error", err) 123 | } 124 | return 125 | } 126 | 127 | batchSizeLimit := c.cfg.Elasticsearch.BatchSizeLimit 128 | if len(actions) > batchSizeLimit { 129 | chunks := slices.ChunkWithSize[elasticsearch.Action](actions, batchSizeLimit) 130 | lastChunkIndex := len(chunks) - 1 131 | for idx, chunk := range chunks { 132 | c.bulk.AddActions(ctx, msg.EventTime, chunk, indexName, idx == lastChunkIndex) 133 | } 134 | } else { 135 | c.bulk.AddActions(ctx, msg.EventTime, actions, indexName, true) 136 | } 137 | } 138 | 139 | func (c *connector) resolveTableToIndexName(fullTableName, tableNamespace, tableName string) string { 140 | tableIndexMapping := c.cfg.Elasticsearch.TableIndexMapping 141 | if len(tableIndexMapping) == 0 { 142 | return "" 143 | } 144 | 145 | if indexName, exists := tableIndexMapping[fullTableName]; exists { 146 | return indexName 147 | } 148 | 149 | if t, ok := timescaledb.HyperTables.Load(fullTableName); ok { 150 | parentName := t.(string) 151 | if indexName, exists := tableIndexMapping[parentName]; exists { 152 | return indexName 153 | } 154 | } 155 | 156 | parentTableName := c.getParentTableName(fullTableName, tableNamespace, tableName) 157 | if parentTableName != "" { 158 | if indexName, exists := tableIndexMapping[parentTableName]; exists { 159 | return indexName 160 | } 161 | } 162 | 163 | return "" 164 | } 165 | 166 | func (c *connector) getParentTableName(fullTableName, tableNamespace, tableName string) string { 167 | if cachedValue, found := c.partitionCache.Load(fullTableName); found { 168 | parentName, ok := cachedValue.(string) 169 | if !ok { 170 | logger.Error("invalid cache value type for table", "table", fullTableName) 171 | return "" 172 | } 173 | 174 | if parentName != "" { 175 | logger.Debug("matched partition table to parent from cache", 176 | "partition", fullTableName, 177 | "parent", parentName) 178 | } 179 | return parentName 180 | } 181 | 182 | parentTableName := c.findParentTable(tableNamespace, tableName) 183 | c.partitionCache.Store(fullTableName, parentTableName) 184 | 185 | if parentTableName != "" { 186 | logger.Debug("matched partition table to parent", 187 | "partition", fullTableName, 188 | "parent", parentTableName) 189 | } 190 | 191 | return parentTableName 192 | } 193 | 194 | func (c *connector) getFullTableName(tableNamespace, tableName string) string { 195 | return fmt.Sprintf("%s.%s", tableNamespace, tableName) 196 | } 197 | 198 | func (c *connector) findParentTable(tableNamespace, tableName string) string { 199 | tableParts := strings.Split(tableName, "_") 200 | if len(tableParts) <= 1 { 201 | return "" 202 | } 203 | 204 | for i := 1; i < len(tableParts); i++ { 205 | parentNameCandidate := strings.Join(tableParts[:i], "_") 206 | fullParentName := c.getFullTableName(tableNamespace, parentNameCandidate) 207 | 208 | if _, exists := c.cfg.Elasticsearch.TableIndexMapping[fullParentName]; exists { 209 | return fullParentName 210 | } 211 | } 212 | 213 | return "" 214 | } 215 | -------------------------------------------------------------------------------- /docs/production_tutorial.md: -------------------------------------------------------------------------------- 1 | # Production Tutorial 2 | 3 | In this tutorial, we will guide you through the setup process for using PostgreSQL logical replication as a change data 4 | capture (CDC) source to stream documents to Elasticsearch using the `go-pq-cdc-elasticsearch` tool. 5 | 6 | ## Database Settings 7 | 8 | Before setting up logical replication, you need to configure several PostgreSQL settings. Add or update the following 9 | parameters in your PostgreSQL configuration file (`postgresql.conf`): 10 | 11 | * `wal_level=logical`: 12 | * Explanation: This setting enables logical replication by generating a log of changes in a format that allows them 13 | to be replicated to other systems, including non-PostgreSQL systems like Elasticsearch. 14 | * `max_wal_senders=5`: 15 | * Explanation: This parameter sets the maximum number of concurrent connections that can be used for sending WAL 16 | (Write-Ahead Logging) data to replicas. Each replication slot or subscription uses one WAL sender, so ensure this number 17 | is sufficient for your replication needs. 18 | * `max_replication_slots=5`: 19 | * Explanation: This setting specifies the maximum number of replication slots that PostgreSQL can use. Replication 20 | slots ensure that the server retains WAL files until they have been processed by all subscribers. For logical 21 | replication, each slot corresponds to a replication source. 22 | 23 | After modifying these settings, restart your PostgreSQL server to apply the changes. 24 | 25 | ## Create User 26 | For production use, it's recommended to use predefined replication slots and publications to minimize the permissions 27 | required by the CDC user. This section outlines the steps to set up a superuser to create the publication and slot, and 28 | a dedicated user with minimal permissions for CDC operations. 29 | 30 | 31 | - **Create a publication** that specifies which tables and changes to replicate with SUPERUSER: 32 | ```sql 33 | CREATE PUBLICATION es_cdc_publication FOR TABLE users WITH (publish = 'INSERT,DELETE,UPDATE'); 34 | ``` 35 | 36 | - **Create replication slot** for the CDC process with SUPERUSER: 37 | ```sql 38 | SELECT * FROM pg_create_logical_replication_slot('es_cdc_slot', 'pgoutput'); 39 | ``` 40 | 41 | - Ensure the table is configured to capture necessary columns for updates and deletions. Choose `FULL` or `DEFAULT` 42 | based on your replication needs: 43 | ```sql 44 | ALTER TABLE users REPLICA IDENTITY FULL; 45 | ``` 46 | 47 | - Create a user with minimal permissions needed for CDC operations: 48 | ```sql 49 | CREATE USER es_cdc_user WITH REPLICATION LOGIN PASSWORD 'es_cdc_pass'; 50 | ``` 51 | 52 | ## Configuration 53 | You can Check Configs detailed explanations [here](../README.md/#configuration) 54 | You only need to configure the following fields to use the `go-pq-cdc-elasticsearch` application: 55 | ```go 56 | cfg := config.Config{ 57 | CDC: cdcconfig.Config{ 58 | Host: "127.0.0.1", 59 | Username: "es_cdc_user", 60 | Password: "es_cdc_pass", 61 | Database: "es_cdc_db", 62 | DebugMode: false, 63 | Publication: publication.Config{ 64 | Name: "es_cdc_publication", 65 | }, 66 | Slot: slot.Config{ 67 | Name: "es_cdc_slot", 68 | SlotActivityCheckerInterval: 3000, 69 | }, 70 | Metric: cdcconfig.MetricConfig{ 71 | Port: 8081, 72 | }, 73 | }, 74 | Elasticsearch: config.Elasticsearch{ 75 | BatchSizeLimit: 10000, 76 | BatchTickerDuration: time.Millisecond * 100, 77 | TableIndexMapping: map[string]string{ 78 | "public.users": "users", 79 | }, 80 | TypeName: "_doc", 81 | URLs: []string{"http://127.0.0.1:9200"}, 82 | }, 83 | } 84 | ``` 85 | 86 | ## Handler 87 | 88 | The `go-pq-cdc-elasticsearch` library supports handling `insert`, `delete`, and `update` messages (if you need other message types, feel free to open issue).
89 | Here is an example handler function: 90 | 91 | ```go 92 | func Handler(msg cdc.Message) []elasticsearch.Action { 93 | switch msg.Type { 94 | case cdc.InsertMessage: 95 | b, _ := json.Marshal(msg.NewData) 96 | return []elasticsearch.Action{ 97 | elasticsearch.NewIndexAction([]byte(strconv.Itoa(int(msg.NewData["id"].(int32)))), b, nil), 98 | } 99 | case cdc.DeleteMessage: 100 | return []elasticsearch.Action{ 101 | elasticsearch.NewDeleteAction([]byte(strconv.Itoa(int(msg.OldData["id"].(int32)))), nil), 102 | } 103 | case cdc.UpdateMessage: 104 | msg.NewData["old_name"] = msg.OldData["name"] // if table replica identity is not full, OldData will be nil 105 | b, _ := json.Marshal(msg.NewData) 106 | return []elasticsearch.Action{ 107 | elasticsearch.NewIndexAction([]byte(strconv.Itoa(int(msg.NewData["id"].(int32)))), b, nil), 108 | } 109 | default: 110 | return nil 111 | } 112 | } 113 | ``` 114 | 115 | ## Deploy Strategy 116 | 117 | The go-pq-cdc operates in passive/active modes for PostgreSQL change data capture (CDC). Here's how it ensures 118 | availability: 119 | 120 | * **Active Mode:** When the PostgreSQL replication slot (slot.name) is active, go-pq-cdc continuously monitors changes 121 | and streams them to downstream systems as configured. 122 | * **Passive Mode:** If the PostgreSQL replication slot becomes inactive (detected via slot.slotActivityCheckerInterval), 123 | go-pq-cdc automatically captures the slot again and resumes data capturing. Other deployments also monitor slot 124 | activity, 125 | and when detected as inactive, they initiate data capturing. 126 | 127 | Deploy go-pq-cdc with **`maximum two instances per cluster`** to ensure one active deployment and another ready to take over 128 | if the active slot becomes inactive. This setup provides redundancy and helps maintain continuous data capture without 129 | interruption. -------------------------------------------------------------------------------- /elasticsearch/action.go: -------------------------------------------------------------------------------- 1 | package elasticsearch 2 | 3 | import "encoding/json" 4 | 5 | type ActionType string 6 | 7 | const ( 8 | Index ActionType = "Index" 9 | Delete ActionType = "Delete" 10 | ScriptUpdate ActionType = "ScriptUpdate" 11 | ) 12 | 13 | type Action struct { 14 | Routing *string 15 | IndexName string 16 | Type ActionType 17 | Source []byte 18 | ID []byte 19 | } 20 | 21 | type Script struct { 22 | Params map[string]interface{} `json:"params,omitempty"` 23 | Source string `json:"source"` 24 | } 25 | 26 | func NewDeleteAction(key []byte, routing *string) Action { 27 | return Action{ 28 | ID: key, 29 | Routing: routing, 30 | Type: Delete, 31 | } 32 | } 33 | 34 | func NewIndexAction(key []byte, source []byte, routing *string) Action { 35 | return Action{ 36 | ID: key, 37 | Routing: routing, 38 | Source: source, 39 | Type: Index, 40 | } 41 | } 42 | 43 | func NewScriptUpdateAction(id []byte, script Script, routing *string) Action { 44 | scriptBytes, _ := json.Marshal(script) 45 | return Action{ 46 | ID: id, 47 | Type: ScriptUpdate, 48 | Source: scriptBytes, 49 | Routing: routing, 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /elasticsearch/bulk/bulk.go: -------------------------------------------------------------------------------- 1 | package bulk 2 | 3 | import ( 4 | gobytes "bytes" 5 | "context" 6 | "fmt" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | cdc "github.com/Trendyol/go-pq-cdc" 12 | "github.com/elastic/go-elasticsearch/v7" 13 | 14 | "github.com/Trendyol/go-pq-cdc-elasticsearch/config" 15 | elasticsearch2 "github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch" 16 | "github.com/Trendyol/go-pq-cdc-elasticsearch/internal/bytes" 17 | "github.com/Trendyol/go-pq-cdc-elasticsearch/internal/slices" 18 | "github.com/Trendyol/go-pq-cdc/logger" 19 | "github.com/Trendyol/go-pq-cdc/pq/replication" 20 | "github.com/go-playground/errors" 21 | 22 | "golang.org/x/sync/errgroup" 23 | 24 | "github.com/elastic/go-elasticsearch/v7/esapi" 25 | 26 | jsoniter "github.com/json-iterator/go" 27 | ) 28 | 29 | type Indexer interface { 30 | StartBulk() 31 | AddActions( 32 | ctx *replication.ListenerContext, 33 | eventTime time.Time, 34 | actions []elasticsearch2.Action, 35 | indexName string, 36 | isLastChunk bool, 37 | ) 38 | GetMetric() Metric 39 | Close() 40 | } 41 | 42 | type Bulk struct { 43 | metric Metric 44 | responseHandler elasticsearch2.ResponseHandler 45 | config *config.Config 46 | batchKeys map[string]int 47 | batchTicker *time.Ticker 48 | isClosed chan bool 49 | esClient *elasticsearch.Client 50 | readers []*bytes.MultiDimensionReader 51 | typeName []byte 52 | batch []BatchItem 53 | batchIndex int 54 | batchSize int 55 | batchSizeLimit int 56 | batchTickerDuration time.Duration 57 | batchByteSizeLimit int 58 | batchByteSize int 59 | concurrentRequest int 60 | flushLock sync.Mutex 61 | } 62 | 63 | type BatchItem struct { 64 | Action *elasticsearch2.Action 65 | Bytes []byte 66 | } 67 | 68 | func NewBulk( 69 | config *config.Config, 70 | esClient *elasticsearch.Client, 71 | pqCDC cdc.Connector, 72 | options ...Option, 73 | ) (*Bulk, error) { 74 | readers := make([]*bytes.MultiDimensionReader, config.Elasticsearch.ConcurrentRequest) 75 | for i := 0; i < config.Elasticsearch.ConcurrentRequest; i++ { 76 | readers[i] = bytes.NewMultiDimReader(nil) 77 | } 78 | 79 | batchByteSizeLimit, err := bytes.ParseSize(config.Elasticsearch.BatchByteSizeLimit) 80 | if err != nil { 81 | return nil, errors.Wrap(err, "batch byte size limit") 82 | } 83 | 84 | bulk := &Bulk{ 85 | batchTickerDuration: config.Elasticsearch.BatchTickerDuration, 86 | batchTicker: time.NewTicker(config.Elasticsearch.BatchTickerDuration), 87 | batchSizeLimit: config.Elasticsearch.BatchSizeLimit, 88 | batchByteSizeLimit: int(batchByteSizeLimit), 89 | isClosed: make(chan bool, 1), 90 | esClient: esClient, 91 | metric: NewMetric(pqCDC, config.CDC.Slot.Name), 92 | config: config, 93 | typeName: []byte(config.Elasticsearch.TypeName), 94 | readers: readers, 95 | concurrentRequest: config.Elasticsearch.ConcurrentRequest, 96 | batchKeys: make(map[string]int, config.Elasticsearch.BatchSizeLimit), 97 | } 98 | 99 | if config.Elasticsearch.TypeName == "" { 100 | bulk.typeName = nil 101 | } 102 | 103 | Options(options).Apply(bulk) 104 | 105 | return bulk, nil 106 | } 107 | 108 | func (b *Bulk) StartBulk() { 109 | for range b.batchTicker.C { 110 | b.flushMessages() 111 | } 112 | } 113 | 114 | func (b *Bulk) AddActions( 115 | ctx *replication.ListenerContext, 116 | eventTime time.Time, 117 | actions []elasticsearch2.Action, 118 | indexName string, 119 | isLastChunk bool, 120 | ) { 121 | b.flushLock.Lock() 122 | for i, action := range actions { 123 | indexName := b.getIndexName(indexName, action.IndexName) 124 | actions[i].IndexName = indexName 125 | value := getEsActionJSON( 126 | action.ID, 127 | action.Type, 128 | actions[i].IndexName, 129 | action.Routing, 130 | action.Source, 131 | b.typeName, 132 | b.config.Elasticsearch.Version, 133 | ) 134 | 135 | b.metric.incrementOp(action.Type, indexName) 136 | 137 | key := getActionKey(actions[i]) 138 | if batchIndex, ok := b.batchKeys[key]; ok { 139 | b.batchByteSize += len(value) - len(b.batch[batchIndex].Bytes) 140 | b.batch[batchIndex] = BatchItem{ 141 | Action: &actions[i], 142 | Bytes: value, 143 | } 144 | } else { 145 | b.batch = append(b.batch, BatchItem{ 146 | Action: &actions[i], 147 | Bytes: value, 148 | }) 149 | b.batchKeys[key] = b.batchIndex 150 | b.batchIndex++ 151 | b.batchSize++ 152 | b.batchByteSize += len(value) 153 | } 154 | } 155 | if isLastChunk { 156 | if err := ctx.Ack(); err != nil { 157 | logger.Error("ack", "error", err) 158 | } 159 | } 160 | 161 | b.flushLock.Unlock() 162 | 163 | if isLastChunk { 164 | b.metric.SetProcessLatency(time.Now().UTC().Sub(eventTime).Nanoseconds()) 165 | } 166 | if b.batchSize >= b.batchSizeLimit || b.batchByteSize >= b.batchByteSizeLimit { 167 | b.flushMessages() 168 | } 169 | } 170 | 171 | var ( 172 | indexPrefix = []byte(`{"index":{"_index":"`) 173 | deletePrefix = []byte(`{"delete":{"_index":"`) 174 | updatePrefix = []byte(`{"update":{"_index":"`) 175 | scriptPrefix = []byte(`{"script":`) 176 | idPrefix = []byte(`","_id":"`) 177 | typePrefix = []byte(`","_type":"`) 178 | routingPrefix = []byte(`","routing":"`) 179 | postFix = []byte(`"}}`) 180 | scriptPostfix = []byte(`,"scripted_upsert":true}`) 181 | ) 182 | 183 | var metaPool = sync.Pool{ 184 | New: func() interface{} { 185 | return []byte{} 186 | }, 187 | } 188 | 189 | func isTypeSupported(version string) bool { 190 | if version == "" { 191 | return true 192 | } 193 | 194 | parts := strings.Split(version, ".") 195 | if len(parts) == 0 { 196 | return true 197 | } 198 | 199 | majorVersion := parts[0] 200 | return majorVersion < "8" 201 | } 202 | 203 | func getEsActionJSON(docID []byte, action elasticsearch2.ActionType, indexName string, routing *string, source []byte, typeName []byte, esVersion string) []byte { 204 | meta := metaPool.Get().([]byte)[:0] 205 | 206 | switch action { 207 | case elasticsearch2.Index: 208 | meta = append(meta, indexPrefix...) 209 | case elasticsearch2.Delete: 210 | meta = append(meta, deletePrefix...) 211 | case elasticsearch2.ScriptUpdate: 212 | meta = append(meta, updatePrefix...) 213 | } 214 | 215 | meta = append(meta, []byte(indexName)...) 216 | meta = append(meta, idPrefix...) 217 | meta = append(meta, bytes.EscapePredefinedBytes(docID)...) 218 | if routing != nil { 219 | meta = append(meta, routingPrefix...) 220 | meta = append(meta, []byte(*routing)...) 221 | } 222 | if typeName != nil && isTypeSupported(esVersion) { 223 | meta = append(meta, typePrefix...) 224 | meta = append(meta, typeName...) 225 | } 226 | meta = append(meta, postFix...) 227 | 228 | switch action { 229 | case elasticsearch2.Index: 230 | meta = append(meta, '\n') 231 | meta = append(meta, source...) 232 | case elasticsearch2.ScriptUpdate: 233 | meta = append(meta, '\n') 234 | meta = append(meta, scriptPrefix...) 235 | meta = append(meta, source...) 236 | meta = append(meta, scriptPostfix...) 237 | } 238 | meta = append(meta, '\n') 239 | return meta 240 | } 241 | 242 | func (b *Bulk) Close() { 243 | b.batchTicker.Stop() 244 | b.flushMessages() 245 | close(b.isClosed) 246 | } 247 | 248 | func (b *Bulk) flushMessages() { 249 | b.flushLock.Lock() 250 | defer b.flushLock.Unlock() 251 | if len(b.batch) > 0 { 252 | err := b.bulkRequest() 253 | if err != nil && b.responseHandler == nil { 254 | panic(err) 255 | } 256 | b.batchTicker.Reset(b.batchTickerDuration) 257 | for _, batch := range b.batch { 258 | //nolint:staticcheck 259 | metaPool.Put(batch.Bytes) 260 | } 261 | b.batch = b.batch[:0] 262 | b.batchKeys = make(map[string]int, b.batchSizeLimit) 263 | b.batchIndex = 0 264 | b.batchSize = 0 265 | b.batchByteSize = 0 266 | } 267 | } 268 | 269 | func (b *Bulk) requestFunc(concurrentRequestIndex int, batchItems []BatchItem) func() error { 270 | return func() error { 271 | reader := b.readers[concurrentRequestIndex] 272 | reader.Reset(getBytes(batchItems)) 273 | r, err := b.esClient.Bulk(reader) 274 | if err != nil { 275 | return err 276 | } 277 | errorData, err := hasResponseError(r) 278 | b.handleResponse(getActions(batchItems), errorData) 279 | if err != nil { 280 | return err 281 | } 282 | return nil 283 | } 284 | } 285 | 286 | func (b *Bulk) bulkRequest() error { 287 | eg, _ := errgroup.WithContext(context.Background()) 288 | 289 | chunks := slices.Chunk(b.batch, b.concurrentRequest) 290 | 291 | startedTime := time.Now() 292 | 293 | for i, chunk := range chunks { 294 | if len(chunk) > 0 { 295 | eg.Go(b.requestFunc(i, chunk)) 296 | } 297 | } 298 | 299 | err := eg.Wait() 300 | 301 | b.metric.SetBulkRequestProcessLatency(time.Since(startedTime).Nanoseconds()) 302 | 303 | return err 304 | } 305 | 306 | func (b *Bulk) GetMetric() Metric { 307 | return b.metric 308 | } 309 | 310 | func hasResponseError(r *esapi.Response) (map[string]string, error) { 311 | if r == nil { 312 | return nil, fmt.Errorf("esapi response is nil") 313 | } 314 | if r.IsError() { 315 | return nil, fmt.Errorf("bulk request has error %v", r.String()) 316 | } 317 | rb := new(gobytes.Buffer) 318 | 319 | defer r.Body.Close() 320 | _, err := rb.ReadFrom(r.Body) 321 | if err != nil { 322 | return nil, err 323 | } 324 | b := make(map[string]any) 325 | err = jsoniter.Unmarshal(rb.Bytes(), &b) 326 | if err != nil { 327 | return nil, err 328 | } 329 | hasError, ok := b["errors"].(bool) 330 | if !ok || !hasError { 331 | return nil, nil 332 | } 333 | return joinErrors(b) 334 | } 335 | 336 | func joinErrors(body map[string]any) (map[string]string, error) { 337 | var sb strings.Builder 338 | ivd := make(map[string]string) 339 | sb.WriteString("bulk request has error. Errors will be listed below:\n") 340 | 341 | items, ok := body["items"].([]any) 342 | if !ok { 343 | return nil, nil 344 | } 345 | 346 | for _, i := range items { 347 | item, ok := i.(map[string]any) 348 | if !ok { 349 | continue 350 | } 351 | 352 | for _, v := range item { 353 | iv, ok := v.(map[string]any) 354 | if !ok { 355 | continue 356 | } 357 | 358 | if iv["error"] != nil { 359 | itemValue := fmt.Sprintf("%v\n", i) 360 | sb.WriteString(itemValue) 361 | itemValueDataKey := fmt.Sprintf("%s:%s", iv["_id"].(string), iv["_index"].(string)) 362 | ivd[itemValueDataKey] = itemValue 363 | } 364 | } 365 | } 366 | return ivd, fmt.Errorf(sb.String()) 367 | } 368 | 369 | func (b *Bulk) getIndexName(indexName, actionIndexName string) string { 370 | if actionIndexName != "" { 371 | return actionIndexName 372 | } 373 | 374 | if indexName == "" { 375 | panic(fmt.Sprintf("there is no index mapping for table: %s on your configuration", indexName)) 376 | } 377 | 378 | return indexName 379 | } 380 | 381 | func (b *Bulk) handleResponse(batchActions []*elasticsearch2.Action, errs map[string]string) { 382 | if b.responseHandler == nil { 383 | return 384 | } 385 | 386 | for _, a := range batchActions { 387 | key := getActionKey(*a) 388 | if _, ok := errs[key]; ok { 389 | b.responseHandler.OnError(&elasticsearch2.ResponseHandlerContext{ 390 | Action: a, 391 | Err: fmt.Errorf(errs[key]), 392 | }) 393 | continue 394 | } 395 | 396 | b.responseHandler.OnSuccess(&elasticsearch2.ResponseHandlerContext{ 397 | Action: a, 398 | }) 399 | } 400 | } 401 | 402 | func getActionKey(action elasticsearch2.Action) string { 403 | if action.Routing != nil { 404 | return fmt.Sprintf("%s:%s:%s", action.ID, action.IndexName, *action.Routing) 405 | } 406 | return fmt.Sprintf("%s:%s", action.ID, action.IndexName) 407 | } 408 | 409 | func getBytes(batchItems []BatchItem) [][]byte { 410 | batchBytes := make([][]byte, 0, len(batchItems)) 411 | for _, batchItem := range batchItems { 412 | batchBytes = append(batchBytes, batchItem.Bytes) 413 | } 414 | return batchBytes 415 | } 416 | 417 | func getActions(batchItems []BatchItem) []*elasticsearch2.Action { 418 | batchActions := make([]*elasticsearch2.Action, 0, len(batchItems)) 419 | for _, batchItem := range batchItems { 420 | batchActions = append(batchActions, batchItem.Action) 421 | } 422 | return batchActions 423 | } 424 | -------------------------------------------------------------------------------- /elasticsearch/bulk/metric.go: -------------------------------------------------------------------------------- 1 | package bulk 2 | 3 | import ( 4 | "os" 5 | 6 | cdc "github.com/Trendyol/go-pq-cdc" 7 | "github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch" 8 | "github.com/prometheus/client_golang/prometheus" 9 | ) 10 | 11 | const namespace = "go_pq_cdc_elasticsearch" 12 | 13 | type Metric interface { 14 | SetProcessLatency(latency int64) 15 | SetBulkRequestProcessLatency(latency int64) 16 | PrometheusCollectors() []prometheus.Collector 17 | incrementOp(opType elasticsearch.ActionType, index string) 18 | } 19 | 20 | var hostname, _ = os.Hostname() 21 | 22 | type metric struct { 23 | pqCDC cdc.Connector 24 | processLatencyMs prometheus.Gauge 25 | bulkRequestProcessLatencyMs prometheus.Gauge 26 | totalIndex map[string]prometheus.Counter 27 | totalDelete map[string]prometheus.Counter 28 | slotName string 29 | } 30 | 31 | func NewMetric(pqCDC cdc.Connector, slotName string) Metric { 32 | return &metric{ 33 | slotName: slotName, 34 | pqCDC: pqCDC, 35 | processLatencyMs: prometheus.NewGauge(prometheus.GaugeOpts{ 36 | Namespace: namespace, 37 | Subsystem: "process_latency", 38 | Name: "current", 39 | Help: "latest elasticsearch connector process latency in nanoseconds", 40 | ConstLabels: prometheus.Labels{ 41 | "host": hostname, 42 | "slot_name": slotName, 43 | }, 44 | }), 45 | bulkRequestProcessLatencyMs: prometheus.NewGauge(prometheus.GaugeOpts{ 46 | Namespace: namespace, 47 | Subsystem: "bulk_request_process_latency", 48 | Name: "current", 49 | Help: "latest elasticsearch connector bulk request process latency in nanoseconds", 50 | ConstLabels: prometheus.Labels{ 51 | "host": hostname, 52 | "slot_name": slotName, 53 | }, 54 | }), 55 | totalIndex: make(map[string]prometheus.Counter), 56 | totalDelete: make(map[string]prometheus.Counter), 57 | } 58 | } 59 | 60 | func (m *metric) PrometheusCollectors() []prometheus.Collector { 61 | return []prometheus.Collector{ 62 | m.processLatencyMs, 63 | m.bulkRequestProcessLatencyMs, 64 | } 65 | } 66 | 67 | func (m *metric) SetProcessLatency(latency int64) { 68 | m.processLatencyMs.Set(float64(latency)) 69 | } 70 | 71 | func (m *metric) SetBulkRequestProcessLatency(latency int64) { 72 | m.bulkRequestProcessLatencyMs.Set(float64(latency)) 73 | } 74 | 75 | func (m *metric) incrementOp(opType elasticsearch.ActionType, index string) { 76 | switch opType { 77 | case elasticsearch.Index: 78 | if _, exists := m.totalIndex[index]; !exists { 79 | m.totalIndex[index] = prometheus.NewCounter(prometheus.CounterOpts{ 80 | Namespace: namespace, 81 | Subsystem: "index", 82 | Name: "total", 83 | Help: "total number of index operation in elasticsearch", 84 | ConstLabels: prometheus.Labels{ 85 | "slot_name": m.slotName, 86 | "index_name": index, 87 | "host": hostname, 88 | }, 89 | }) 90 | m.pqCDC.SetMetricCollectors(m.totalIndex[index]) 91 | } 92 | 93 | m.totalIndex[index].Add(1) 94 | case elasticsearch.Delete: 95 | if _, exists := m.totalDelete[index]; !exists { 96 | m.totalDelete[index] = prometheus.NewCounter(prometheus.CounterOpts{ 97 | Namespace: namespace, 98 | Subsystem: "delete", 99 | Name: "total", 100 | Help: "total number of delete operation in elasticsearch", 101 | ConstLabels: prometheus.Labels{ 102 | "slot_name": m.slotName, 103 | "index_name": index, 104 | "host": hostname, 105 | }, 106 | }) 107 | m.pqCDC.SetMetricCollectors(m.totalDelete[index]) 108 | } 109 | 110 | m.totalDelete[index].Add(1) 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /elasticsearch/bulk/option.go: -------------------------------------------------------------------------------- 1 | package bulk 2 | 3 | import ( 4 | "github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch" 5 | ) 6 | 7 | type Option func(*Bulk) 8 | 9 | type Options []Option 10 | 11 | func (ops Options) Apply(c *Bulk) { 12 | for _, op := range ops { 13 | op(c) 14 | } 15 | } 16 | 17 | func WithResponseHandler(respHandler elasticsearch.ResponseHandler) Option { 18 | return func(b *Bulk) { 19 | if respHandler == nil { 20 | return 21 | } 22 | 23 | b.responseHandler = respHandler 24 | b.responseHandler.OnInit(&elasticsearch.ResponseHandlerInitContext{ 25 | Config: b.config, 26 | ElasticsearchClient: b.esClient, 27 | }) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /elasticsearch/client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/Trendyol/go-pq-cdc-elasticsearch/config" 7 | "github.com/Trendyol/go-pq-cdc/logger" 8 | "github.com/elastic/go-elasticsearch/v7" 9 | "github.com/go-playground/errors" 10 | ) 11 | 12 | func NewClient(config *config.Config) (*elasticsearch.Client, error) { 13 | client, err := elasticsearch.NewClient(elasticsearch.Config{ 14 | Username: config.Elasticsearch.Username, 15 | Password: config.Elasticsearch.Password, 16 | MaxRetries: 5, 17 | Addresses: config.Elasticsearch.URLs, 18 | Transport: NewTransport(config.Elasticsearch), 19 | CompressRequestBody: config.Elasticsearch.CompressionEnabled, 20 | DiscoverNodesOnStart: !config.Elasticsearch.DisableDiscoverNodesOnStart, 21 | DiscoverNodesInterval: *config.Elasticsearch.DiscoverNodesInterval, 22 | Logger: &LoggerAdapter{}, 23 | }) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | r, err := client.Ping() 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | if r.StatusCode == 401 { 34 | return nil, errors.New("unauthorized") 35 | } 36 | 37 | if config.Elasticsearch.Version == "" { 38 | version, err := detectElasticsearchVersion(client) 39 | if err != nil { 40 | logger.Warn("elasticsearch version detection failed", "error", err, "fallback_version", "7.0.0", "hint", "specify 'elasticsearch.version' in config to set manually") 41 | config.Elasticsearch.Version = "7.0.0" 42 | } else { 43 | logger.Info("elasticsearch version detected", "version", version) 44 | config.Elasticsearch.Version = version 45 | } 46 | } 47 | 48 | return client, nil 49 | } 50 | 51 | func detectElasticsearchVersion(client *elasticsearch.Client) (string, error) { 52 | info, err := client.Info() 53 | if err != nil { 54 | return "", err 55 | } 56 | 57 | var response map[string]interface{} 58 | if err := json.NewDecoder(info.Body).Decode(&response); err != nil { 59 | return "", err 60 | } 61 | defer info.Body.Close() 62 | 63 | version, ok := response["version"].(map[string]interface{}) 64 | if !ok { 65 | return "", errors.New("version info not found in Elasticsearch response") 66 | } 67 | 68 | number, ok := version["number"].(string) 69 | if !ok { 70 | return "", errors.New("version number not found in Elasticsearch response") 71 | } 72 | 73 | return number, nil 74 | } 75 | -------------------------------------------------------------------------------- /elasticsearch/client/fasthttp_transport.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | "strings" 7 | 8 | "github.com/Trendyol/go-pq-cdc-elasticsearch/config" 9 | 10 | "github.com/valyala/fasthttp" 11 | ) 12 | 13 | type Transporter interface { 14 | RoundTrip(req *http.Request) (*http.Response, error) 15 | } 16 | 17 | // transport implements the elastictransport interface with 18 | // the github.com/valyala/fasthttp HTTP client. 19 | type transport struct { 20 | client *fasthttp.Client 21 | } 22 | 23 | func NewTransport(cfg config.Elasticsearch) Transporter { 24 | client := &fasthttp.Client{ 25 | MaxConnsPerHost: fasthttp.DefaultMaxConnsPerHost, 26 | MaxIdleConnDuration: fasthttp.DefaultMaxIdleConnDuration, 27 | } 28 | 29 | if cfg.MaxConnsPerHost != nil { 30 | client.MaxConnsPerHost = *cfg.MaxConnsPerHost 31 | } 32 | 33 | if cfg.MaxIdleConnDuration != nil { 34 | client.MaxIdleConnDuration = *cfg.MaxIdleConnDuration 35 | } 36 | 37 | return &transport{client: client} 38 | } 39 | 40 | // RoundTrip performs the request and returns a response or error 41 | func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { 42 | freq := fasthttp.AcquireRequest() 43 | defer fasthttp.ReleaseRequest(freq) 44 | 45 | fres := fasthttp.AcquireResponse() 46 | defer fasthttp.ReleaseResponse(fres) 47 | 48 | t.copyRequest(freq, req) 49 | 50 | err := t.client.Do(freq, fres) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | res := &http.Response{Header: make(http.Header)} 56 | t.copyResponse(res, fres) 57 | 58 | return res, nil 59 | } 60 | 61 | // copyRequest converts a http.Request to fasthttp.Request 62 | func (t *transport) copyRequest(dst *fasthttp.Request, src *http.Request) *fasthttp.Request { 63 | if src.Method == http.MethodGet && src.Body != nil { 64 | src.Method = http.MethodPost 65 | } 66 | dst.SetHost(src.Host) 67 | dst.SetRequestURI(src.URL.String()) 68 | dst.Header.SetRequestURI(src.URL.String()) 69 | dst.Header.SetMethod(src.Method) 70 | 71 | for k, vv := range src.Header { 72 | for _, v := range vv { 73 | dst.Header.Set(k, v) 74 | } 75 | } 76 | 77 | if src.Body != nil { 78 | dst.SetBodyStream(src.Body, -1) 79 | } 80 | 81 | return dst 82 | } 83 | 84 | // copyResponse converts a http.Response to fasthttp.Response 85 | func (t *transport) copyResponse(dst *http.Response, src *fasthttp.Response) *http.Response { 86 | dst.StatusCode = src.StatusCode() 87 | 88 | src.Header.VisitAll(func(k, v []byte) { 89 | dst.Header.Set(string(k), string(v)) 90 | }) 91 | 92 | // Cast to a string to make a copy seeing as src.Body() won't 93 | // be valid after the response is released back to the pool (fasthttp.ReleaseResponse). 94 | dst.Body = io.NopCloser(strings.NewReader(string(src.Body()))) 95 | 96 | return dst 97 | } 98 | -------------------------------------------------------------------------------- /elasticsearch/client/logger.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/Trendyol/go-pq-cdc/logger" 8 | ) 9 | 10 | type LoggerAdapter struct{} 11 | 12 | func (l *LoggerAdapter) LogRoundTrip(_ *http.Request, _ *http.Response, err error, _ time.Time, _ time.Duration) error { 13 | if err == nil { 14 | return nil 15 | } 16 | 17 | logger.Error("elasticsearch error", "error", err) 18 | return nil 19 | } 20 | 21 | func (l *LoggerAdapter) RequestBodyEnabled() bool { 22 | return true 23 | } 24 | 25 | func (l *LoggerAdapter) ResponseBodyEnabled() bool { 26 | return true 27 | } 28 | -------------------------------------------------------------------------------- /elasticsearch/response_handler.go: -------------------------------------------------------------------------------- 1 | package elasticsearch 2 | 3 | import ( 4 | "github.com/Trendyol/go-pq-cdc-elasticsearch/config" 5 | es "github.com/elastic/go-elasticsearch/v7" 6 | ) 7 | 8 | type ResponseHandler interface { 9 | OnSuccess(ctx *ResponseHandlerContext) 10 | OnError(ctx *ResponseHandlerContext) 11 | OnInit(ctx *ResponseHandlerInitContext) 12 | } 13 | 14 | type ResponseHandlerContext struct { 15 | Action *Action 16 | Err error 17 | } 18 | 19 | type ResponseHandlerInitContext struct { 20 | Config *config.Config 21 | ElasticsearchClient *es.Client 22 | } 23 | -------------------------------------------------------------------------------- /example/script-update/README.md: -------------------------------------------------------------------------------- 1 | # Script Update Example 2 | 3 | This example demonstrates how to use painless scripts and partial updates with go-pq-cdc-elasticsearch. 4 | 5 | ## Features Demonstrated 6 | 7 | 1. Script Updates 8 | - Simple field updates 9 | - Conditional updates 10 | - Using script parameters 11 | 12 | 2. Doc Updates 13 | - Partial document updates 14 | - Field-specific updates 15 | 16 | ## Prerequisites 17 | 18 | - Docker and Docker Compose 19 | - Go 1.20 or later 20 | 21 | ## Setup 22 | 23 | 1. Start the required services: 24 | 25 | ```bash 26 | export STACK_VERSION=7.17.11 27 | docker-compose up -d 28 | ``` 29 | 30 | 2. Wait for all services to be healthy (check with `docker-compose ps`) 31 | 32 | 3. Create the database table and insert sample data: 33 | 34 | ```sql 35 | psql "postgres://script_cdc_user:script_cdc_pass@127.0.0.1/script_cdc_db?replication=database" 36 | 37 | CREATE TABLE products ( 38 | id serial PRIMARY KEY, 39 | name text NOT NULL, 40 | price decimal(10,2), 41 | stock integer, 42 | last_updated timestamptz 43 | ); 44 | 45 | -- Insert sample data 46 | INSERT INTO products (name, price, stock, last_updated) 47 | VALUES 48 | ('Product 1', 99.99, 100, NOW()), 49 | ('Product 2', 149.99, 50, NOW()), 50 | ('Product 3', 199.99, 25, NOW()); 51 | ``` 52 | 53 | ## Running the Example 54 | 55 | 1. Build and run the example: 56 | 57 | ```bash 58 | go mod tidy 59 | go run main.go 60 | ``` 61 | 62 | 2. Test different update scenarios: 63 | 64 | ```sql 65 | -- Update stock using script 66 | UPDATE products SET stock = stock + 10 WHERE id = 1; 67 | 68 | -- Update price conditionally 69 | UPDATE products SET price = 89.99 WHERE id = 2; 70 | 71 | -- Update multiple fields 72 | UPDATE products SET stock = stock - 5, last_updated = NOW() WHERE id = 3; 73 | ``` 74 | 75 | ## Understanding the Code 76 | 77 | The example demonstrates three types of updates: 78 | 79 | 1. **Script Updates for Stock Changes**: 80 | 81 | ```go 82 | script := map[string]interface{}{ 83 | "source": "ctx._source.stock = params.new_stock", 84 | "params": map[string]interface{}{ 85 | "new_stock": newStock, 86 | }, 87 | } 88 | ``` 89 | 90 | 2. **Conditional Script Updates for Price Changes**: 91 | 92 | ```go 93 | script := map[string]interface{}{ 94 | "source": "if (ctx._source.price != params.new_price) { ctx._source.price = params.new_price }", 95 | "params": map[string]interface{}{ 96 | "new_price": newPrice, 97 | }, 98 | } 99 | ``` 100 | 101 | 3. **Doc Updates for Other Fields**: 102 | 103 | ```go 104 | updateData := map[string]interface{}{ 105 | "name": msg.NewData["name"], 106 | "last_updated": msg.NewData["last_updated"], 107 | } 108 | ``` 109 | 110 | ## Monitoring 111 | 112 | - Check Elasticsearch indices: http://localhost:9200/products/_search 113 | - View logs: `docker-compose logs -f` 114 | - Monitor metrics: http://localhost:8081/metrics 115 | 116 | ## Cleanup 117 | 118 | ```bash 119 | docker-compose down -v 120 | ``` -------------------------------------------------------------------------------- /example/script-update/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | postgres: 4 | image: postgres:16.2 5 | restart: always 6 | command: ["-c", "wal_level=logical", "-c", "max_wal_senders=10", "-c", "max_replication_slots=10"] 7 | environment: 8 | POSTGRES_USER: "script_cdc_user" 9 | POSTGRES_PASSWORD: "script_cdc_pass" 10 | POSTGRES_DB: "script_cdc_db" 11 | POSTGRES_HOST_AUTH_METHOD: trust 12 | ports: 13 | - 5432:5432 14 | 15 | es01: 16 | image: docker.elastic.co/elasticsearch/elasticsearch:7.17.11 17 | labels: 18 | co.elastic.logs/module: elasticsearch 19 | volumes: 20 | - es01_data:/usr/share/elasticsearch/data 21 | networks: 22 | - cdc_net 23 | ports: 24 | - 9200:9200 25 | environment: 26 | - node.name=es01 27 | - cluster.name=script_cdc_cluster 28 | - discovery.type=single-node 29 | - ELASTIC_PASSWORD=script_cdc_es_pass 30 | - bootstrap.memory_lock=true 31 | - xpack.security.enabled=true 32 | - xpack.security.http.ssl.enabled=false 33 | - xpack.security.transport.ssl.enabled=false 34 | ulimits: 35 | memlock: 36 | soft: -1 37 | hard: -1 38 | healthcheck: 39 | test: 40 | [ 41 | "CMD-SHELL", 42 | "curl -u elastic:script_cdc_es_pass -I -o /dev/null http://localhost:9200 -w '%{http_code}' | grep -q '200'", 43 | ] 44 | interval: 10s 45 | timeout: 10s 46 | retries: 120 47 | 48 | kibana: 49 | depends_on: 50 | es01: 51 | condition: service_healthy 52 | image: docker.elastic.co/kibana/kibana:7.17.11 53 | networks: 54 | - cdc_net 55 | labels: 56 | co.elastic.logs/module: kibana 57 | volumes: 58 | - kibana1_data:/usr/share/kibana/data 59 | ports: 60 | - "5601:5601" 61 | environment: 62 | - SERVERNAME=kibana 63 | - ELASTICSEARCH_HOSTS=http://es01:9200 64 | - ELASTICSEARCH_USERNAME=elastic 65 | - ELASTICSEARCH_PASSWORD=script_cdc_es_pass 66 | healthcheck: 67 | test: 68 | [ 69 | "CMD-SHELL", 70 | "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", 71 | ] 72 | interval: 10s 73 | timeout: 10s 74 | retries: 120 75 | 76 | volumes: 77 | postgres_data: null 78 | kibana1_data: null 79 | es01_data: null 80 | 81 | networks: 82 | cdc_net: 83 | driver: bridge -------------------------------------------------------------------------------- /example/script-update/go.mod: -------------------------------------------------------------------------------- 1 | module script-update 2 | 3 | go 1.22.5 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/Trendyol/go-pq-cdc v0.0.12 9 | github.com/Trendyol/go-pq-cdc-elasticsearch v0.0.0 10 | ) 11 | 12 | require ( 13 | github.com/andybalholm/brotli v1.1.0 // indirect 14 | github.com/avast/retry-go/v4 v4.6.0 // indirect 15 | github.com/beorn7/perks v1.0.1 // indirect 16 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 17 | github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect 18 | github.com/go-playground/errors v3.3.0+incompatible // indirect 19 | github.com/jackc/pgpassfile v1.0.0 // indirect 20 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect 21 | github.com/jackc/pgx/v5 v5.6.0 // indirect 22 | github.com/json-iterator/go v1.1.12 // indirect 23 | github.com/klauspost/compress v1.17.9 // indirect 24 | github.com/lib/pq v1.10.9 // indirect 25 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 26 | github.com/modern-go/reflect2 v1.0.2 // indirect 27 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 28 | github.com/prometheus/client_golang v1.19.1 // indirect 29 | github.com/prometheus/client_model v0.6.1 // indirect 30 | github.com/prometheus/common v0.55.0 // indirect 31 | github.com/prometheus/procfs v0.15.1 // indirect 32 | github.com/valyala/bytebufferpool v1.0.0 // indirect 33 | github.com/valyala/fasthttp v1.55.0 // indirect 34 | golang.org/x/crypto v0.24.0 // indirect 35 | golang.org/x/sync v0.7.0 // indirect 36 | golang.org/x/sys v0.21.0 // indirect 37 | golang.org/x/text v0.16.0 // indirect 38 | google.golang.org/protobuf v1.34.2 // indirect 39 | gopkg.in/yaml.v2 v2.4.0 // indirect 40 | ) 41 | 42 | replace github.com/Trendyol/go-pq-cdc-elasticsearch => ../../ 43 | -------------------------------------------------------------------------------- /example/script-update/go.sum: -------------------------------------------------------------------------------- 1 | github.com/Trendyol/go-pq-cdc v0.0.12 h1:VE38j8n47ALMJxtHWro67Fd6kyPuuMiDZc7Q3EUbomo= 2 | github.com/Trendyol/go-pq-cdc v0.0.12/go.mod h1:RIooS3DPOWkXxq7nhrOuGgkD4x3ondWEYOrOEHAHnxc= 3 | github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= 4 | github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= 5 | github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= 6 | github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= 7 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 8 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 9 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 10 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= 15 | github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= 16 | github.com/go-playground/errors v3.3.0+incompatible h1:w7qP6bdFXNmI86aV8VEfhXrGxoQWYHc/OX4Muw4FgW0= 17 | github.com/go-playground/errors v3.3.0+incompatible/go.mod h1:n+RcthKmtLxDczVHKkhqiUSOGtTjvRl+HB4Gga0vWSI= 18 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 19 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 20 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 21 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 22 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 23 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= 24 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 25 | github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= 26 | github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= 27 | github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= 28 | github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= 29 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 30 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 31 | github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= 32 | github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= 33 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 34 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 35 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 36 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 37 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 38 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 39 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 40 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 41 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 42 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 43 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 44 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 45 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 46 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 47 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 48 | github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= 49 | github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= 50 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 51 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 52 | github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= 53 | github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= 54 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 55 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 56 | github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= 57 | github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 58 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 59 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 60 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 61 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 62 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 63 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 64 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 65 | github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8= 66 | github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM= 67 | golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= 68 | golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= 69 | golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= 70 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 71 | golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= 72 | golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 73 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 74 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 75 | google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= 76 | google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= 77 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 78 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 79 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 80 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 81 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 82 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 83 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 84 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 85 | -------------------------------------------------------------------------------- /example/script-update/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log/slog" 8 | "os" 9 | "strconv" 10 | "time" 11 | 12 | cdc "github.com/Trendyol/go-pq-cdc-elasticsearch" 13 | "github.com/Trendyol/go-pq-cdc-elasticsearch/config" 14 | "github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch" 15 | cdcconfig "github.com/Trendyol/go-pq-cdc/config" 16 | "github.com/Trendyol/go-pq-cdc/pq/publication" 17 | "github.com/Trendyol/go-pq-cdc/pq/slot" 18 | ) 19 | 20 | /* 21 | psql "postgres://script_cdc_user:script_cdc_pass@127.0.0.1/script_cdc_db?replication=database" 22 | 23 | CREATE TABLE products ( 24 | id serial PRIMARY KEY, 25 | name text NOT NULL, 26 | price decimal(10,2), 27 | stock integer, 28 | last_updated timestamptz 29 | ); 30 | 31 | -- Insert sample data 32 | INSERT INTO products (name, price, stock, last_updated) 33 | VALUES 34 | ('Product 1', 99.99, 100, NOW()), 35 | ('Product 2', 149.99, 50, NOW()), 36 | ('Product 3', 199.99, 25, NOW()); 37 | 38 | -- Update stock using script 39 | UPDATE products SET stock = stock + 10 WHERE id = 1; 40 | 41 | -- Update price conditionally 42 | UPDATE products SET price = 89.99 WHERE id = 2; 43 | 44 | -- Update multiple fields 45 | UPDATE products SET stock = stock - 5, last_updated = NOW() WHERE id = 3; 46 | */ 47 | 48 | func main() { 49 | slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) 50 | ctx := context.TODO() 51 | cfg := config.Config{ 52 | CDC: cdcconfig.Config{ 53 | Host: "127.0.0.1", 54 | Username: "script_cdc_user", 55 | Password: "script_cdc_pass", 56 | Database: "script_cdc_db", 57 | DebugMode: false, 58 | Publication: publication.Config{ 59 | CreateIfNotExists: true, 60 | Name: "script_cdc_publication", 61 | Operations: publication.Operations{ 62 | publication.OperationInsert, 63 | publication.OperationDelete, 64 | publication.OperationTruncate, 65 | publication.OperationUpdate, 66 | }, 67 | Tables: publication.Tables{ 68 | publication.Table{ 69 | Name: "products", 70 | ReplicaIdentity: publication.ReplicaIdentityFull, 71 | }, 72 | }, 73 | }, 74 | Slot: slot.Config{ 75 | CreateIfNotExists: true, 76 | Name: "script_cdc_slot", 77 | SlotActivityCheckerInterval: 3000, 78 | }, 79 | Metric: cdcconfig.MetricConfig{ 80 | Port: 8081, 81 | }, 82 | }, 83 | Elasticsearch: config.Elasticsearch{ 84 | Username: "elastic", 85 | Password: "script_cdc_es_pass", 86 | BatchSizeLimit: 10000, 87 | BatchTickerDuration: time.Millisecond * 100, 88 | TableIndexMapping: map[string]string{ 89 | "public.products": "products", 90 | }, 91 | TypeName: "_doc", 92 | URLs: []string{"http://127.0.0.1:9200"}, 93 | DisableDiscoverNodesOnStart: true, 94 | CompressionEnabled: false, 95 | MaxConnsPerHost: &[]int{10}[0], 96 | MaxIdleConnDuration: &[]time.Duration{30 * time.Second}[0], 97 | }, 98 | } 99 | 100 | connector, err := cdc.NewConnector(ctx, cfg, Handler) 101 | if err != nil { 102 | slog.Error("new connector", "error", err) 103 | os.Exit(1) 104 | } 105 | 106 | defer connector.Close() 107 | connector.Start(ctx) 108 | } 109 | 110 | func Handler(msg cdc.Message) []elasticsearch.Action { 111 | slog.Info("message received", "type", msg.Type, "msg", fmt.Sprintf("%#v", msg)) 112 | 113 | switch msg.Type { 114 | case cdc.InsertMessage: 115 | // For inserts, we'll use a regular index action with initial version 116 | insertData := msg.NewData 117 | insertData["version"] = 1 118 | b, _ := json.Marshal(insertData) 119 | return []elasticsearch.Action{ 120 | elasticsearch.NewIndexAction([]byte(strconv.Itoa(int(msg.NewData["id"].(int32)))), b, nil), 121 | } 122 | 123 | case cdc.DeleteMessage: 124 | // For deletes, we'll use a regular delete action 125 | return []elasticsearch.Action{ 126 | elasticsearch.NewDeleteAction([]byte(strconv.Itoa(int(msg.OldData["id"].(int32)))), nil), 127 | } 128 | 129 | case cdc.UpdateMessage: 130 | id := strconv.Itoa(int(msg.NewData["id"].(int32))) 131 | 132 | if oldStock, newStock := msg.OldData["stock"], msg.NewData["stock"]; oldStock != newStock { 133 | script := elasticsearch.Script{ 134 | Source: ` 135 | if (ctx._source.version == null) { 136 | ctx._source.version = 1; 137 | } else { 138 | ctx._source.version += 1; 139 | } 140 | ctx._source.stock = params.new_stock; 141 | `, 142 | Params: map[string]interface{}{"new_stock": newStock}, 143 | } 144 | return []elasticsearch.Action{ 145 | elasticsearch.NewScriptUpdateAction([]byte(id), script, nil), 146 | } 147 | } 148 | 149 | if oldPrice, newPrice := msg.OldData["price"], msg.NewData["price"]; oldPrice != newPrice { 150 | script := elasticsearch.Script{ 151 | Source: ` 152 | if (ctx._source.version == null) { 153 | ctx._source.version = 1; 154 | } else { 155 | ctx._source.version += 1; 156 | } 157 | if (ctx._source.price != params.new_price) { 158 | ctx._source.price = params.new_price; 159 | } 160 | `, 161 | Params: map[string]interface{}{"new_price": newPrice}, 162 | } 163 | return []elasticsearch.Action{ 164 | elasticsearch.NewScriptUpdateAction([]byte(id), script, nil), 165 | } 166 | } 167 | 168 | script := elasticsearch.Script{ 169 | Source: ` 170 | if (ctx._source.version == null) { 171 | ctx._source.version = 1; 172 | } else { 173 | ctx._source.version += 1; 174 | } 175 | ctx._source.name = params.new_data.name; 176 | ctx._source.last_updated = params.new_data.last_updated; 177 | `, 178 | Params: map[string]interface{}{"new_data": msg.NewData}, 179 | } 180 | return []elasticsearch.Action{ 181 | elasticsearch.NewScriptUpdateAction([]byte(id), script, nil), 182 | } 183 | 184 | default: 185 | return nil 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /example/simple/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | postgres: 4 | image: postgres:16.2 5 | restart: always 6 | command: ["-c", "wal_level=logical", "-c", "max_wal_senders=10", "-c", "max_replication_slots=10"] 7 | environment: 8 | POSTGRES_USER: "es_cdc_user" 9 | POSTGRES_PASSWORD: "es_cdc_pass" 10 | POSTGRES_DB: "es_cdc_db" 11 | POSTGRES_HOST_AUTH_METHOD: trust 12 | network_mode: "host" 13 | 14 | es01: 15 | image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} 16 | labels: 17 | co.elastic.logs/module: elasticsearch 18 | volumes: 19 | - es01_data:/usr/share/elasticsearch/data 20 | networks: 21 | - cdc_net 22 | ports: 23 | - 9200:9200 24 | environment: 25 | - node.name=es01 26 | - cluster.name=es_cdc_cluster 27 | - discovery.type=single-node 28 | - ELASTIC_PASSWORD=es_cdc_es_pass 29 | - bootstrap.memory_lock=true 30 | - xpack.security.enabled=true 31 | - xpack.security.http.ssl.enabled=false 32 | - xpack.security.transport.ssl.enabled=false 33 | ulimits: 34 | memlock: 35 | soft: -1 36 | hard: -1 37 | healthcheck: 38 | test: 39 | [ 40 | "CMD-SHELL", 41 | "curl -u elastic:es_cdc_es_pass -I -o /dev/null http://localhost:9200 -w '%{http_code}' | grep -q '200'", 42 | ] 43 | interval: 10s 44 | timeout: 10s 45 | retries: 120 46 | 47 | kibana: 48 | depends_on: 49 | es01: 50 | condition: service_healthy 51 | image: docker.elastic.co/kibana/kibana:7.17.11 52 | networks: 53 | - cdc_net 54 | labels: 55 | co.elastic.logs/module: kibana 56 | volumes: 57 | - kibana1_data:/usr/share/kibana/data 58 | ports: 59 | - "5601:5601" 60 | environment: 61 | - SERVERNAME=kibana 62 | - ELASTICSEARCH_HOSTS=http://es01:9200 63 | - ELASTICSEARCH_USERNAME=elastic 64 | - ELASTICSEARCH_PASSWORD=es_cdc_es_pass 65 | healthcheck: 66 | test: 67 | [ 68 | "CMD-SHELL", 69 | "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", 70 | ] 71 | interval: 10s 72 | timeout: 10s 73 | retries: 120 74 | 75 | volumes: 76 | postgres_data: null 77 | kibana1_data: null 78 | es01_data: null 79 | 80 | networks: 81 | cdc_net: 82 | driver: bridge 83 | -------------------------------------------------------------------------------- /example/simple/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Trendyol/go-pq-cdc-elasticsearch/example/simple 2 | 3 | go 1.22.5 4 | 5 | replace github.com/Trendyol/go-pq-cdc-elasticsearch => ../.. 6 | 7 | require ( 8 | github.com/Trendyol/go-pq-cdc v0.0.12 9 | github.com/Trendyol/go-pq-cdc-elasticsearch v0.0.0-20240628144743-a0f06e9b6d86 10 | ) 11 | 12 | require ( 13 | github.com/andybalholm/brotli v1.1.0 // indirect 14 | github.com/avast/retry-go/v4 v4.6.0 // indirect 15 | github.com/beorn7/perks v1.0.1 // indirect 16 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 17 | github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect 18 | github.com/go-playground/errors v3.3.0+incompatible // indirect 19 | github.com/jackc/pgpassfile v1.0.0 // indirect 20 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect 21 | github.com/jackc/pgx/v5 v5.6.0 // indirect 22 | github.com/json-iterator/go v1.1.12 // indirect 23 | github.com/klauspost/compress v1.17.9 // indirect 24 | github.com/lib/pq v1.10.9 // indirect 25 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 26 | github.com/modern-go/reflect2 v1.0.2 // indirect 27 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 28 | github.com/prometheus/client_golang v1.19.1 // indirect 29 | github.com/prometheus/client_model v0.6.1 // indirect 30 | github.com/prometheus/common v0.55.0 // indirect 31 | github.com/prometheus/procfs v0.15.1 // indirect 32 | github.com/valyala/bytebufferpool v1.0.0 // indirect 33 | github.com/valyala/fasthttp v1.55.0 // indirect 34 | golang.org/x/crypto v0.24.0 // indirect 35 | golang.org/x/sync v0.7.0 // indirect 36 | golang.org/x/sys v0.21.0 // indirect 37 | golang.org/x/text v0.16.0 // indirect 38 | google.golang.org/protobuf v1.34.2 // indirect 39 | gopkg.in/yaml.v2 v2.4.0 // indirect 40 | ) 41 | -------------------------------------------------------------------------------- /example/simple/go.sum: -------------------------------------------------------------------------------- 1 | github.com/Trendyol/go-pq-cdc v0.0.12 h1:VE38j8n47ALMJxtHWro67Fd6kyPuuMiDZc7Q3EUbomo= 2 | github.com/Trendyol/go-pq-cdc v0.0.12/go.mod h1:RIooS3DPOWkXxq7nhrOuGgkD4x3ondWEYOrOEHAHnxc= 3 | github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= 4 | github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= 5 | github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= 6 | github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= 7 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 8 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 9 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 10 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= 15 | github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= 16 | github.com/go-playground/errors v3.3.0+incompatible h1:w7qP6bdFXNmI86aV8VEfhXrGxoQWYHc/OX4Muw4FgW0= 17 | github.com/go-playground/errors v3.3.0+incompatible/go.mod h1:n+RcthKmtLxDczVHKkhqiUSOGtTjvRl+HB4Gga0vWSI= 18 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 19 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 20 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 21 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 22 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 23 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= 24 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 25 | github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= 26 | github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= 27 | github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= 28 | github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= 29 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 30 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 31 | github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= 32 | github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= 33 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 34 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 35 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 36 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 37 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 38 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 39 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 40 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 41 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 42 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 43 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 44 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 45 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 46 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 47 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 48 | github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= 49 | github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= 50 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 51 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 52 | github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= 53 | github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= 54 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 55 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 56 | github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= 57 | github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 58 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 59 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 60 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 61 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 62 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 63 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 64 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 65 | github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8= 66 | github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM= 67 | golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= 68 | golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= 69 | golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= 70 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 71 | golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= 72 | golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 73 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 74 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 75 | google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= 76 | google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= 77 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 78 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 79 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 80 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 81 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 82 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 83 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 84 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 85 | -------------------------------------------------------------------------------- /example/simple/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log/slog" 8 | "os" 9 | "strconv" 10 | "time" 11 | 12 | cdc "github.com/Trendyol/go-pq-cdc-elasticsearch" 13 | "github.com/Trendyol/go-pq-cdc-elasticsearch/config" 14 | "github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch" 15 | cdcconfig "github.com/Trendyol/go-pq-cdc/config" 16 | "github.com/Trendyol/go-pq-cdc/pq/publication" 17 | "github.com/Trendyol/go-pq-cdc/pq/slot" 18 | ) 19 | 20 | /* 21 | psql "postgres://es_cdc_user:es_cdc_pass@127.0.0.1/es_cdc_db?replication=database" 22 | 23 | CREATE TABLE users ( 24 | id serial PRIMARY KEY, 25 | name text NOT NULL, 26 | created_on timestamptz 27 | ); 28 | 29 | CREATE TABLE books ( 30 | id serial PRIMARY KEY, 31 | name text NOT NULL, 32 | created_on timestamptz 33 | ); 34 | 35 | INSERT INTO users (name) 36 | SELECT 37 | 'Oyleli' || i 38 | FROM generate_series(1, 100) AS i; 39 | 40 | INSERT INTO books (name) 41 | SELECT 42 | 'Oyleli' || i 43 | FROM generate_series(1, 100) AS i; 44 | */ 45 | 46 | func main() { 47 | slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) 48 | ctx := context.TODO() 49 | cfg := config.Config{ 50 | CDC: cdcconfig.Config{ 51 | Host: "127.0.0.1", 52 | Username: "es_cdc_user", 53 | Password: "es_cdc_pass", 54 | Database: "es_cdc_db", 55 | DebugMode: false, 56 | Publication: publication.Config{ 57 | CreateIfNotExists: true, 58 | Name: "es_cdc_publication", 59 | Operations: publication.Operations{ 60 | publication.OperationInsert, 61 | publication.OperationDelete, 62 | publication.OperationTruncate, 63 | publication.OperationUpdate, 64 | }, 65 | Tables: publication.Tables{ 66 | publication.Table{ 67 | Name: "users", 68 | ReplicaIdentity: publication.ReplicaIdentityFull, 69 | }, 70 | publication.Table{ 71 | Name: "books", 72 | ReplicaIdentity: publication.ReplicaIdentityFull, 73 | }, 74 | }, 75 | }, 76 | Slot: slot.Config{ 77 | CreateIfNotExists: true, 78 | Name: "es_cdc_slot", 79 | SlotActivityCheckerInterval: 3000, 80 | }, 81 | Metric: cdcconfig.MetricConfig{ 82 | Port: 8081, 83 | }, 84 | }, 85 | Elasticsearch: config.Elasticsearch{ 86 | Username: "elastic", 87 | Password: "es_cdc_es_pass", 88 | BatchSizeLimit: 10000, 89 | BatchTickerDuration: time.Millisecond * 100, 90 | TableIndexMapping: map[string]string{ 91 | "public.users": "users", 92 | "public.books": "books", 93 | }, 94 | TypeName: "_doc", 95 | URLs: []string{"http://127.0.0.1:9200"}, 96 | }, 97 | } 98 | 99 | connector, err := cdc.NewConnector(ctx, cfg, Handler) 100 | if err != nil { 101 | slog.Error("new connector", "error", err) 102 | os.Exit(1) 103 | } 104 | 105 | defer connector.Close() 106 | connector.Start(ctx) 107 | } 108 | 109 | func Handler(msg cdc.Message) []elasticsearch.Action { 110 | slog.Info("message received", "type", msg.Type, "msg", fmt.Sprintf("%#v", msg)) 111 | switch msg.Type { 112 | case cdc.InsertMessage: 113 | b, _ := json.Marshal(msg.NewData) 114 | return []elasticsearch.Action{ 115 | elasticsearch.NewIndexAction([]byte(strconv.Itoa(int(msg.NewData["id"].(int32)))), b, nil), 116 | } 117 | case cdc.DeleteMessage: 118 | return []elasticsearch.Action{ 119 | elasticsearch.NewDeleteAction([]byte(strconv.Itoa(int(msg.OldData["id"].(int32)))), nil), 120 | } 121 | case cdc.UpdateMessage: 122 | msg.NewData["old_name"] = msg.OldData["name"] 123 | b, _ := json.Marshal(msg.NewData) 124 | return []elasticsearch.Action{ 125 | elasticsearch.NewIndexAction([]byte(strconv.Itoa(int(msg.NewData["id"].(int32)))), b, nil), 126 | } 127 | default: 128 | return nil 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Trendyol/go-pq-cdc-elasticsearch 2 | 3 | go 1.22.5 4 | 5 | require ( 6 | github.com/Trendyol/go-pq-cdc v0.0.12 7 | github.com/elastic/go-elasticsearch/v7 v7.17.10 8 | github.com/go-playground/errors v3.3.0+incompatible 9 | github.com/json-iterator/go v1.1.12 10 | github.com/prometheus/client_golang v1.19.1 11 | github.com/valyala/fasthttp v1.55.0 12 | golang.org/x/sync v0.7.0 13 | ) 14 | 15 | require ( 16 | github.com/andybalholm/brotli v1.1.0 // indirect 17 | github.com/avast/retry-go/v4 v4.6.0 // indirect 18 | github.com/beorn7/perks v1.0.1 // indirect 19 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 20 | github.com/jackc/pgpassfile v1.0.0 // indirect 21 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect 22 | github.com/jackc/pgx/v5 v5.6.0 // indirect 23 | github.com/klauspost/compress v1.17.9 // indirect 24 | github.com/lib/pq v1.10.9 // indirect 25 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 26 | github.com/modern-go/reflect2 v1.0.2 // indirect 27 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 28 | github.com/prometheus/client_model v0.6.1 // indirect 29 | github.com/prometheus/common v0.55.0 // indirect 30 | github.com/prometheus/procfs v0.15.1 // indirect 31 | github.com/valyala/bytebufferpool v1.0.0 // indirect 32 | golang.org/x/crypto v0.24.0 // indirect 33 | golang.org/x/sys v0.21.0 // indirect 34 | golang.org/x/text v0.16.0 // indirect 35 | google.golang.org/protobuf v1.34.2 // indirect 36 | gopkg.in/yaml.v2 v2.4.0 // indirect 37 | ) 38 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Trendyol/go-pq-cdc v0.0.12 h1:VE38j8n47ALMJxtHWro67Fd6kyPuuMiDZc7Q3EUbomo= 2 | github.com/Trendyol/go-pq-cdc v0.0.12/go.mod h1:RIooS3DPOWkXxq7nhrOuGgkD4x3ondWEYOrOEHAHnxc= 3 | github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= 4 | github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= 5 | github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= 6 | github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= 7 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 8 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 9 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 10 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= 15 | github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= 16 | github.com/go-playground/errors v3.3.0+incompatible h1:w7qP6bdFXNmI86aV8VEfhXrGxoQWYHc/OX4Muw4FgW0= 17 | github.com/go-playground/errors v3.3.0+incompatible/go.mod h1:n+RcthKmtLxDczVHKkhqiUSOGtTjvRl+HB4Gga0vWSI= 18 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 19 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 20 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 21 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 22 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 23 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= 24 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 25 | github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= 26 | github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= 27 | github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= 28 | github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= 29 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 30 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 31 | github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= 32 | github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= 33 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 34 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 35 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 36 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 37 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 38 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 39 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 40 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 41 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 42 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 43 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 44 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 45 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 46 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 47 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 48 | github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= 49 | github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= 50 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 51 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 52 | github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= 53 | github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= 54 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 55 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 56 | github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= 57 | github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 58 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 59 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 60 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 61 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 62 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 63 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 64 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 65 | github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8= 66 | github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM= 67 | golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= 68 | golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= 69 | golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= 70 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 71 | golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= 72 | golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 73 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 74 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 75 | google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= 76 | google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= 77 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 78 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 79 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 80 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 81 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 82 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 83 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 84 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 85 | -------------------------------------------------------------------------------- /handler.go: -------------------------------------------------------------------------------- 1 | package cdc 2 | 3 | import ( 4 | "github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch" 5 | ) 6 | 7 | type Handler func(msg Message) []elasticsearch.Action 8 | -------------------------------------------------------------------------------- /internal/bytes/bytes.go: -------------------------------------------------------------------------------- 1 | package bytes 2 | 3 | import ( 4 | "errors" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | type Size uint64 10 | 11 | const ( 12 | B Size = 1 13 | KB Size = 1 << (10 * iota) 14 | MB 15 | GB 16 | TB 17 | PB 18 | EB 19 | ) 20 | 21 | var unitMap = map[string]Size{ 22 | "B": B, 23 | "BYTE": B, 24 | "BYTES": B, 25 | 26 | "KB": KB, 27 | "KILOBYTE": KB, 28 | "KILOBYTES": KB, 29 | 30 | "MB": MB, 31 | "MEGABYTE": MB, 32 | "MEGABYTES": MB, 33 | 34 | "GB": GB, 35 | "GIGABYTE": GB, 36 | "GIGABYTES": GB, 37 | 38 | "TB": TB, 39 | "TERABYTE": TB, 40 | "TERABYTES": TB, 41 | 42 | "PB": PB, 43 | "PETABYTE": PB, 44 | "PETABYTES": PB, 45 | 46 | "EB": EB, 47 | "EXABYTE": EB, 48 | "EXABYTES": EB, 49 | } 50 | 51 | var ( 52 | ErrInvalidNumberFormat = errors.New("invalid number format") 53 | ErrUnknownUnit = errors.New("unknown unit") 54 | ErrInvalidSizeFormat = errors.New("invalid size format") 55 | ) 56 | 57 | func ParseSize(sizeStr string) (Size, error) { 58 | sizeStr = strings.TrimSpace(sizeStr) 59 | 60 | for i, c := range sizeStr { 61 | if c < '0' || c > '9' { 62 | numberStr := sizeStr[:i] 63 | unitStr := sizeStr[i:] 64 | 65 | number, err := strconv.ParseFloat(numberStr, 64) 66 | if err != nil { 67 | return 0, ErrInvalidNumberFormat 68 | } 69 | 70 | unitStr = strings.ToUpper(strings.TrimSpace(unitStr)) 71 | 72 | unitSize, ok := unitMap[unitStr] 73 | if !ok { 74 | return 0, ErrUnknownUnit 75 | } 76 | 77 | return Size(number) * unitSize, nil 78 | } 79 | } 80 | 81 | return 0, ErrInvalidSizeFormat 82 | } 83 | -------------------------------------------------------------------------------- /internal/bytes/escape.go: -------------------------------------------------------------------------------- 1 | package bytes 2 | 3 | var ( 4 | EscapeBytes = []byte{ 5 | 34, // quote 6 | } 7 | BackSlash byte = 92 8 | ) 9 | 10 | func EscapePredefinedBytes(docID []byte) []byte { 11 | newByteArr := make([]byte, 0, len(docID)) 12 | for _, byt := range docID { 13 | for _, escapeByte := range EscapeBytes { 14 | if escapeByte == byt { 15 | newByteArr = append(newByteArr, BackSlash) 16 | } 17 | } 18 | newByteArr = append(newByteArr, byt) 19 | } 20 | return newByteArr 21 | } 22 | -------------------------------------------------------------------------------- /internal/bytes/escape_test.go: -------------------------------------------------------------------------------- 1 | package bytes 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestEscapeQuote(t *testing.T) { 8 | input := "12345-999\"" 9 | byteArr := []byte(input) 10 | result := EscapePredefinedBytes(byteArr) 11 | if len(result) == 10 || result[len(result)-1] != 34 { 12 | t.Error("Expected backslash byte") 13 | } 14 | } 15 | 16 | func TestDoNotEscapeQuote(t *testing.T) { 17 | input := "12345-999" 18 | byteArr := []byte(input) 19 | 20 | result := EscapePredefinedBytes(byteArr) 21 | if len(result) != 9 { 22 | t.Error("Not expected any change related with length") 23 | } 24 | for _, b := range result { 25 | if b == 34 { 26 | t.Error("Not expected any backslash") 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /internal/bytes/reader.go: -------------------------------------------------------------------------------- 1 | package bytes 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | type MultiDimensionReader struct { 8 | s [][]byte // slices 9 | currentSliceIndex int // index of current slice 10 | currentIndexInCurrentSlice int // index of current byte in current slice 11 | currentSliceLen int // length of current slice 12 | totalLen int // total length of all slices 13 | } 14 | 15 | func (r *MultiDimensionReader) Read(b []byte) (n int, err error) { 16 | if r.currentSliceIndex >= r.totalLen { 17 | return 0, io.EOF 18 | } 19 | 20 | if r.currentIndexInCurrentSlice >= r.currentSliceLen { 21 | return 0, io.EOF 22 | } 23 | 24 | n = copy(b, r.s[r.currentSliceIndex][r.currentIndexInCurrentSlice:]) 25 | 26 | if r.currentIndexInCurrentSlice+n >= r.currentSliceLen { 27 | r.currentSliceIndex++ 28 | r.currentIndexInCurrentSlice = 0 29 | r.currentSliceLen = getLen(r.s, r.currentSliceIndex) 30 | } else { 31 | r.currentIndexInCurrentSlice += n 32 | } 33 | 34 | return 35 | } 36 | 37 | func getLen(b [][]byte, index int) int { 38 | if index >= len(b) { 39 | return 0 40 | } 41 | 42 | return len(b[index]) 43 | } 44 | 45 | func (r *MultiDimensionReader) Reset(b [][]byte) { 46 | *r = MultiDimensionReader{b, 0, 0, getLen(b, 0), len(b)} 47 | } 48 | 49 | func NewMultiDimReader(b [][]byte) *MultiDimensionReader { 50 | return &MultiDimensionReader{b, 0, 0, getLen(b, 0), len(b)} 51 | } 52 | -------------------------------------------------------------------------------- /internal/bytes/reader_test.go: -------------------------------------------------------------------------------- 1 | package bytes 2 | 3 | import "testing" 4 | 5 | func TestMultiDimByteReader(t *testing.T) { 6 | reader := NewMultiDimReader(nil) 7 | b := make([]byte, 10) 8 | n, err := reader.Read(b) 9 | 10 | if n != 0 || err == nil { 11 | t.Error("Expected 0, io.EOF") 12 | } 13 | 14 | c := make([][]byte, 2) 15 | c[0] = []byte("hello") 16 | c[1] = []byte("world") 17 | reader = NewMultiDimReader(c) 18 | 19 | b = make([]byte, 3) 20 | n, err = reader.Read(b) 21 | 22 | if n != 3 || err != nil { 23 | t.Error("Expected 3, nil") 24 | } 25 | 26 | d := make([]byte, 8) 27 | n, err = reader.Read(d) 28 | if n != 2 || err != nil { 29 | t.Error("Expected 2, nil") 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /internal/slices/slices.go: -------------------------------------------------------------------------------- 1 | package slices 2 | 3 | func ChunkWithSize[T any](slice []T, chunkSize int) [][]T { 4 | var chunks [][]T 5 | for i := 0; i < len(slice); i += chunkSize { 6 | end := i + chunkSize 7 | 8 | if end > len(slice) { 9 | end = len(slice) 10 | } 11 | 12 | chunks = append(chunks, slice[i:end]) 13 | } 14 | 15 | return chunks 16 | } 17 | 18 | func Chunk[T any](slice []T, chunks int) [][]T { 19 | maxChunkSize := ((len(slice) - 1) / chunks) + 1 20 | numFullChunks := chunks - (maxChunkSize*chunks - len(slice)) 21 | 22 | result := make([][]T, chunks) 23 | 24 | startIndex := 0 25 | 26 | for i := 0; i < chunks; i++ { 27 | endIndex := startIndex + maxChunkSize 28 | 29 | if i >= numFullChunks { 30 | endIndex-- 31 | } 32 | 33 | result[i] = slice[startIndex:endIndex] 34 | 35 | startIndex = endIndex 36 | } 37 | 38 | return result 39 | } 40 | -------------------------------------------------------------------------------- /message.go: -------------------------------------------------------------------------------- 1 | package cdc 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/elastic/go-elasticsearch/v7" 7 | 8 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 9 | ) 10 | 11 | type Message struct { 12 | ElasticsearchClient *elasticsearch.Client 13 | EventTime time.Time 14 | TableName string 15 | TableNamespace string 16 | 17 | OldData map[string]any 18 | NewData map[string]any 19 | 20 | Type MessageType 21 | } 22 | 23 | func NewInsertMessage(esClient *elasticsearch.Client, m *format.Insert) Message { 24 | return Message{ 25 | ElasticsearchClient: esClient, 26 | EventTime: m.MessageTime, 27 | TableName: m.TableName, 28 | TableNamespace: m.TableNamespace, 29 | OldData: nil, 30 | NewData: m.Decoded, 31 | Type: InsertMessage, 32 | } 33 | } 34 | 35 | func NewUpdateMessage(esClient *elasticsearch.Client, m *format.Update) Message { 36 | return Message{ 37 | ElasticsearchClient: esClient, 38 | EventTime: m.MessageTime, 39 | TableName: m.TableName, 40 | TableNamespace: m.TableNamespace, 41 | OldData: m.OldDecoded, 42 | NewData: m.NewDecoded, 43 | Type: UpdateMessage, 44 | } 45 | } 46 | 47 | func NewDeleteMessage(esClient *elasticsearch.Client, m *format.Delete) Message { 48 | return Message{ 49 | ElasticsearchClient: esClient, 50 | EventTime: m.MessageTime, 51 | TableName: m.TableName, 52 | TableNamespace: m.TableNamespace, 53 | OldData: m.OldDecoded, 54 | NewData: nil, 55 | Type: DeleteMessage, 56 | } 57 | } 58 | 59 | type MessageType string 60 | 61 | const ( 62 | InsertMessage MessageType = "INSERT" 63 | UpdateMessage MessageType = "UPDATE" 64 | DeleteMessage MessageType = "DELETE" 65 | ) 66 | 67 | func (m MessageType) IsInsert() bool { return m == InsertMessage } 68 | func (m MessageType) IsUpdate() bool { return m == UpdateMessage } 69 | func (m MessageType) IsDelete() bool { return m == DeleteMessage } 70 | -------------------------------------------------------------------------------- /option.go: -------------------------------------------------------------------------------- 1 | package cdc 2 | 3 | import ( 4 | "github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch" 5 | "github.com/Trendyol/go-pq-cdc/logger" 6 | "github.com/prometheus/client_golang/prometheus" 7 | ) 8 | 9 | type Option func(Connector) 10 | 11 | type Options []Option 12 | 13 | func (ops Options) Apply(c Connector) { 14 | for _, op := range ops { 15 | op(c) 16 | } 17 | } 18 | 19 | func WithResponseHandler(respHandler elasticsearch.ResponseHandler) Option { 20 | return func(c Connector) { 21 | c.(*connector).responseHandler = respHandler 22 | } 23 | } 24 | 25 | func WithPrometheusMetrics(collectors []prometheus.Collector) Option { 26 | return func(c Connector) { 27 | c.(*connector).metrics = collectors 28 | } 29 | } 30 | 31 | func WithLogger(l logger.Logger) Option { 32 | return func(c Connector) { 33 | c.(*connector).cfg.CDC.Logger.Logger = l 34 | } 35 | } 36 | --------------------------------------------------------------------------------