├── .gitignore ├── example ├── .DS_Store ├── simple │ ├── init.sql │ ├── docker-compose.yml │ └── main.go ├── snapshot-only-mode │ ├── init.sql │ ├── docker-compose.yml │ └── main.go ├── snapshot-initial-mode │ ├── init.sql │ ├── docker-compose.yml │ └── main.go ├── snapshot-with-scaling │ ├── init.sql │ ├── Dockerfile │ ├── go.mod │ ├── docker-compose.yml │ ├── main.go │ └── go.sum ├── simple-with-heartbeat │ ├── init.sql │ ├── high_init.sql │ ├── docker-compose.yml │ └── main.go ├── simple-file-config │ ├── docker-compose.yml │ ├── config.yml │ ├── go.mod │ ├── main.go │ └── go.sum └── postgresql │ ├── docker-compose.yml │ ├── go.mod │ ├── main.go │ └── go.sum ├── grafana └── dashboard.png ├── benchmark ├── benchmark_cdc │ ├── grafana │ │ ├── grafana.ini │ │ ├── datasource.yml │ │ └── dashboard.yml │ ├── 10m_test.png │ ├── dashboard.png │ ├── sql │ │ └── init.sql │ ├── prometheus.yml │ ├── go-pq-cdc-kafka │ │ ├── Dockerfile │ │ ├── go.mod │ │ └── main.go │ └── README.md └── benchmark_initial │ ├── grafana │ ├── grafana.ini │ ├── datasource.yml │ └── dashboard.yml │ ├── 10m_1x.png │ ├── 10m_3x.png │ ├── sql │ └── init.sql │ ├── go-pq-cdc-kafka │ ├── Dockerfile │ ├── go.mod │ └── main.go │ ├── prometheus.yml │ ├── README.md │ └── SCALING_GUIDE.md ├── internal ├── slice │ └── slice.go ├── retry │ └── retry.go ├── metric │ └── registry.go └── http │ └── server.go ├── .github ├── dependabot.yml ├── workflows │ ├── release.yml │ ├── integration_test.yml │ ├── build.yml │ └── scorecard.yml └── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── integration_test ├── test_connector.yaml ├── books.go ├── basic_functionality_test.go ├── copy_protocol_test.go ├── transactional_process_test.go ├── go.mod ├── snapshot_helpers_test.go ├── system_identity_full_test.go ├── heartbeat_test.go └── main_test.go ├── .goreleaser.yml ├── pq ├── lsn.go ├── slot │ ├── info.go │ ├── config.go │ └── slot.go ├── message │ ├── format │ │ ├── snapshot.go │ │ ├── relation_test.go │ │ ├── insert_test.go │ │ ├── delete_test.go │ │ ├── delete.go │ │ ├── insert.go │ │ ├── update_test.go │ │ ├── relation.go │ │ └── update.go │ ├── message.go │ └── tuple │ │ └── data.go ├── replication │ ├── wal.go │ └── replication.go ├── publication │ ├── operation.go │ ├── table.go │ ├── config.go │ ├── replica_identity.go │ └── publication.go ├── snapshot │ ├── decoder_cache.go │ ├── transaction_snapshot.go │ ├── connection_pool.go │ ├── job.go │ └── helpers.go ├── connection.go ├── system.go └── timescaledb │ └── hypertable.go ├── config └── read.go ├── .golangci.yml ├── logger └── logger.go ├── LICENSE ├── go.mod ├── Makefile ├── CONTRIBUTING.md └── go.sum /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | vendor -------------------------------------------------------------------------------- /example/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Trendyol/go-pq-cdc/HEAD/example/.DS_Store -------------------------------------------------------------------------------- /grafana/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Trendyol/go-pq-cdc/HEAD/grafana/dashboard.png -------------------------------------------------------------------------------- /benchmark/benchmark_cdc/grafana/grafana.ini: -------------------------------------------------------------------------------- 1 | [security] 2 | admin_user = go-pq-cdc-user 3 | admin_password = go-pq-cdc-pass -------------------------------------------------------------------------------- /benchmark/benchmark_cdc/10m_test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Trendyol/go-pq-cdc/HEAD/benchmark/benchmark_cdc/10m_test.png -------------------------------------------------------------------------------- /benchmark/benchmark_cdc/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Trendyol/go-pq-cdc/HEAD/benchmark/benchmark_cdc/dashboard.png -------------------------------------------------------------------------------- /benchmark/benchmark_initial/grafana/grafana.ini: -------------------------------------------------------------------------------- 1 | [security] 2 | admin_user = go-pq-cdc-user 3 | admin_password = go-pq-cdc-pass -------------------------------------------------------------------------------- /benchmark/benchmark_initial/10m_1x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Trendyol/go-pq-cdc/HEAD/benchmark/benchmark_initial/10m_1x.png -------------------------------------------------------------------------------- /benchmark/benchmark_initial/10m_3x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Trendyol/go-pq-cdc/HEAD/benchmark/benchmark_initial/10m_3x.png -------------------------------------------------------------------------------- /example/simple/init.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE users ( 2 | id serial PRIMARY KEY, 3 | name text NOT NULL, 4 | created_on timestamptz 5 | ); -------------------------------------------------------------------------------- /benchmark/benchmark_cdc/sql/init.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE users ( 2 | id serial PRIMARY KEY, 3 | name text NOT NULL, 4 | created_on timestamptz 5 | ); -------------------------------------------------------------------------------- /benchmark/benchmark_cdc/grafana/datasource.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: Prometheus 5 | type: prometheus 6 | access: proxy 7 | url: http://prometheus:9090 8 | isDefault: true -------------------------------------------------------------------------------- /internal/slice/slice.go: -------------------------------------------------------------------------------- 1 | package slice 2 | 3 | func ConvertToInt(ss []byte) []int { 4 | r := make([]int, len(ss)) 5 | for i, s := range ss { 6 | r[i] = int(s) 7 | } 8 | 9 | return r 10 | } 11 | -------------------------------------------------------------------------------- /benchmark/benchmark_initial/grafana/datasource.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: Prometheus 5 | type: prometheus 6 | access: proxy 7 | url: http://prometheus:9090 8 | isDefault: true -------------------------------------------------------------------------------- /benchmark/benchmark_initial/sql/init.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE users ( 2 | id serial PRIMARY KEY, 3 | name text NOT NULL, 4 | created_on timestamptz 5 | ); 6 | 7 | INSERT INTO users (name) 8 | SELECT 'Oyleli' || i 9 | FROM generate_series(1, 10000000) AS i; -------------------------------------------------------------------------------- /example/snapshot-only-mode/init.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE users ( 2 | id serial PRIMARY KEY, 3 | name text NOT NULL, 4 | created_on timestamptz 5 | ); 6 | 7 | INSERT INTO users (name) 8 | SELECT 9 | 'Oyleli' || i 10 | FROM generate_series(1, 100) AS i; -------------------------------------------------------------------------------- /example/snapshot-initial-mode/init.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE users ( 2 | id serial PRIMARY KEY, 3 | name text NOT NULL, 4 | created_on timestamptz 5 | ); 6 | 7 | INSERT INTO users (name) 8 | SELECT 9 | 'Oyleli' || i 10 | FROM generate_series(1, 1000) AS i; -------------------------------------------------------------------------------- /example/snapshot-with-scaling/init.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE users ( 2 | id serial PRIMARY KEY, 3 | name text NOT NULL, 4 | created_on timestamptz 5 | ); 6 | 7 | INSERT INTO users (name) 8 | SELECT 9 | 'Oyleli' || i 10 | FROM generate_series(1, 1000) AS i; -------------------------------------------------------------------------------- /example/simple-with-heartbeat/init.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE public.test_heartbeat_table ( 2 | id bigserial primary key, 3 | txt text, 4 | ts timestamptz default now() 5 | ); 6 | 7 | CREATE TABLE public.users ( 8 | id serial PRIMARY KEY, 9 | name text NOT NULL, 10 | created_on timestamptz 11 | ); -------------------------------------------------------------------------------- /benchmark/benchmark_cdc/grafana/dashboard.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: "Dashboard provider" 5 | orgId: 1 6 | type: file 7 | disableDeletion: false 8 | updateIntervalSeconds: 10 9 | allowUiUpdates: false 10 | options: 11 | path: /var/lib/grafana/dashboards 12 | foldersFromFilesStructure: true -------------------------------------------------------------------------------- /benchmark/benchmark_initial/grafana/dashboard.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: "Dashboard provider" 5 | orgId: 1 6 | type: file 7 | disableDeletion: false 8 | updateIntervalSeconds: 10 9 | allowUiUpdates: false 10 | options: 11 | path: /var/lib/grafana/dashboards 12 | foldersFromFilesStructure: true -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | open-pull-requests-limit: 10 8 | commit-message: 9 | prefix: "chore: update version" 10 | allow: 11 | - dependency-name: github.com/jackc/pgx/v5 12 | - dependency-name: github.com/lib/pq 13 | -------------------------------------------------------------------------------- /example/simple-file-config/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | postgres: 4 | image: postgres:16.2 5 | restart: always 6 | command: ["-c", "wal_level=logical", "-c", "max_wal_senders=10", "-c", "max_replication_slots=10"] 7 | environment: 8 | POSTGRES_USER: "cdc_user" 9 | POSTGRES_PASSWORD: "cdc_pass" 10 | POSTGRES_DB: "cdc_db" 11 | POSTGRES_HOST_AUTH_METHOD: trust 12 | network_mode: "host" -------------------------------------------------------------------------------- /example/simple-with-heartbeat/high_init.sql: -------------------------------------------------------------------------------- 1 | -- High-traffic database used to simulate WAL pressure from another database 2 | -- on the same Postgres instance. 3 | -- 4 | -- This database is NOT used by the CDC connector. It only generates WAL. 5 | 6 | CREATE DATABASE high_db; 7 | 8 | \connect high_db 9 | 10 | CREATE TABLE IF NOT EXISTS public.hightraffic ( 11 | id serial PRIMARY KEY, 12 | value text NOT NULL 13 | ); 14 | 15 | 16 | -------------------------------------------------------------------------------- /integration_test/test_connector.yaml: -------------------------------------------------------------------------------- 1 | username: cdc_user 2 | password: cdc_pass 3 | database: cdc_db 4 | debugMode: false 5 | publication: 6 | createIfNotExists: true 7 | name: cdc_publication 8 | operations: 9 | - INSERT 10 | - UPDATE 11 | - DELETE 12 | - TRUNCATE 13 | tables: 14 | - name: books 15 | replicaIdentity: FULL 16 | slot: 17 | createIfNotExists: true 18 | logger: 19 | level: INFO 20 | metric: 21 | port: 8083 22 | 23 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: go-pq-cdc 2 | 3 | release: 4 | github: 5 | name: go-pq-cdc 6 | owner: Trendyol 7 | 8 | before: 9 | hooks: 10 | - go mod tidy 11 | 12 | builds: 13 | - skip: true 14 | 15 | changelog: 16 | sort: asc 17 | use: github 18 | filters: 19 | exclude: 20 | - '^test:' 21 | - '^docs:' 22 | - '^chore:' 23 | - 'merge conflict' 24 | - Merge pull request 25 | - Merge remote-tracking branch 26 | - Merge branch 27 | - go mod tidy -------------------------------------------------------------------------------- /example/simple-file-config/config.yml: -------------------------------------------------------------------------------- 1 | host: 127.0.0.1 2 | database: cdc_db 3 | username: cdc_user 4 | password: cdc_pass 5 | publication: 6 | createIfNotExists: true 7 | name: cdc_publication 8 | operations: 9 | - INSERT 10 | - UPDATE 11 | - DELETE 12 | tables: 13 | - name: users 14 | replicaIdentity: FULL 15 | schema: public 16 | slot: 17 | createIfNotExists: true 18 | name: cdc_slot 19 | slotActivityCheckerInterval: 2000 20 | metric: 21 | port: 8083 22 | logger: 23 | level: DEBUG 24 | debugMode: true -------------------------------------------------------------------------------- /integration_test/books.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "strconv" 5 | "sync/atomic" 6 | ) 7 | 8 | type Book struct { 9 | Name string `json:"name"` 10 | ID int `json:"id"` 11 | } 12 | 13 | func (b *Book) Map() map[string]any { 14 | return map[string]any{ 15 | "id": int32(b.ID), 16 | "name": b.Name, 17 | } 18 | } 19 | 20 | func CreateBooks(count int) []Book { 21 | var idCounter atomic.Int64 22 | res := make([]Book, count) 23 | for i := range count { 24 | id := int(idCounter.Add(1)) 25 | res[i] = Book{ 26 | ID: id, 27 | Name: "book-no-" + strconv.Itoa(id), 28 | } 29 | } 30 | return res 31 | } 32 | -------------------------------------------------------------------------------- /pq/lsn.go: -------------------------------------------------------------------------------- 1 | package pq 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/go-playground/errors" 7 | ) 8 | 9 | type LSN uint64 10 | 11 | func (lsn LSN) String() string { 12 | return fmt.Sprintf("%X/%X", uint32(lsn>>32), uint32(lsn)) 13 | } 14 | 15 | func ParseLSN(s string) (LSN, error) { 16 | var upperHalf, lowerHalf uint64 17 | var nparsed int 18 | 19 | nparsed, err := fmt.Sscanf(s, "%X/%X", &upperHalf, &lowerHalf) 20 | if err != nil { 21 | return 0, errors.Wrap(err, "lsn parse") 22 | } 23 | 24 | if nparsed != 2 { 25 | return 0, errors.Newf("lsn parse: %s", s) 26 | } 27 | 28 | return LSN((upperHalf << 32) + lowerHalf), nil 29 | } 30 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v4 14 | with: 15 | fetch-depth: 0 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@v4 19 | with: 20 | go-version: '1.22.4' 21 | 22 | - name: Run GoReleaser 23 | uses: goreleaser/goreleaser-action@v4 24 | with: 25 | version: latest 26 | args: release --clean 27 | env: 28 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /example/simple/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | postgres: 4 | image: postgres:16.2 5 | environment: 6 | POSTGRES_DB: cdc_db 7 | POSTGRES_USER: cdc_user 8 | POSTGRES_PASSWORD: cdc_pass 9 | ports: 10 | - "5433:5432" 11 | command: 12 | - postgres 13 | - -c 14 | - wal_level=logical 15 | - -c 16 | - max_replication_slots=20 17 | - -c 18 | - max_wal_senders=25 19 | - -c 20 | - max_connections=100 21 | healthcheck: 22 | test: [ "CMD-SHELL", "pg_isready -U testuser -d testdb" ] 23 | interval: 5s 24 | timeout: 5s 25 | retries: 5 26 | volumes: 27 | - ./init.sql:/docker-entrypoint-initdb.d/init.sql -------------------------------------------------------------------------------- /pq/slot/info.go: -------------------------------------------------------------------------------- 1 | package slot 2 | 3 | import ( 4 | "github.com/Trendyol/go-pq-cdc/pq" 5 | ) 6 | 7 | const ( 8 | Logical Type = "logical" 9 | Physical Type = "physical" 10 | ) 11 | 12 | type Type string 13 | 14 | type Info struct { 15 | Name string `json:"name"` 16 | Type Type `json:"type"` 17 | WalStatus string `json:"walStatus"` 18 | RestartLSN pq.LSN `json:"restartLSN"` 19 | ConfirmedFlushLSN pq.LSN `json:"confirmedFlushLSN"` 20 | CurrentLSN pq.LSN `json:"currentLSN"` 21 | RetainedWALSize pq.LSN `json:"retainedWALSize"` 22 | Lag pq.LSN `json:"lag"` 23 | ActivePID int32 `json:"activePID"` 24 | Active bool `json:"active"` 25 | } 26 | -------------------------------------------------------------------------------- /example/snapshot-initial-mode/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | postgres: 4 | image: postgres:16.2 5 | environment: 6 | POSTGRES_DB: cdc_db 7 | POSTGRES_USER: cdc_user 8 | POSTGRES_PASSWORD: cdc_pass 9 | ports: 10 | - "5433:5432" 11 | command: 12 | - postgres 13 | - -c 14 | - wal_level=logical 15 | - -c 16 | - max_replication_slots=20 17 | - -c 18 | - max_wal_senders=25 19 | - -c 20 | - max_connections=100 21 | healthcheck: 22 | test: [ "CMD-SHELL", "pg_isready -U cdc_user -d cdc_db" ] 23 | interval: 5s 24 | timeout: 5s 25 | retries: 5 26 | volumes: 27 | - ./init.sql:/docker-entrypoint-initdb.d/init.sql -------------------------------------------------------------------------------- /example/snapshot-only-mode/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | postgres: 4 | image: postgres:16.2 5 | environment: 6 | POSTGRES_DB: cdc_db 7 | POSTGRES_USER: cdc_user 8 | POSTGRES_PASSWORD: cdc_pass 9 | ports: 10 | - "5433:5432" 11 | command: 12 | - postgres 13 | - -c 14 | - wal_level=logical 15 | - -c 16 | - max_replication_slots=20 17 | - -c 18 | - max_wal_senders=25 19 | - -c 20 | - max_connections=100 21 | healthcheck: 22 | test: [ "CMD-SHELL", "pg_isready -U cdc_user -d cdc_db" ] 23 | interval: 5s 24 | timeout: 5s 25 | retries: 5 26 | volumes: 27 | - ./init.sql:/docker-entrypoint-initdb.d/init.sql -------------------------------------------------------------------------------- /benchmark/benchmark_cdc/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 100ms 3 | evaluation_interval: 100ms 4 | scrape_configs: 5 | - job_name: prometheus 6 | static_configs: 7 | - targets: ["localhost:9090"] 8 | - job_name: cadvisor 9 | static_configs: 10 | - targets: [ "cadvisor:8080" ] 11 | - job_name: postgres-exporter 12 | static_configs: 13 | - targets: ["postgres-exporter:9187"] 14 | - job_name: es-exporter 15 | static_configs: 16 | - targets: [ "es-exporter:9114" ] 17 | - job_name: go_pq_cdc_exporter 18 | static_configs: 19 | - targets: [ "go-pq-cdc-kafka:2112" ] 20 | - job_name: redpanda_exporter 21 | metrics_path: "/metrics" 22 | static_configs: 23 | - targets: [ "redpanda:9644" ] -------------------------------------------------------------------------------- /pq/message/format/snapshot.go: -------------------------------------------------------------------------------- 1 | package format 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/Trendyol/go-pq-cdc/pq" 7 | ) 8 | 9 | // SnapshotEventType represents the type of snapshot event 10 | type SnapshotEventType string 11 | 12 | const ( 13 | SnapshotEventTypeBegin SnapshotEventType = "BEGIN" // Snapshot started 14 | SnapshotEventTypeData SnapshotEventType = "DATA" // Row data 15 | SnapshotEventTypeEnd SnapshotEventType = "END" // Snapshot completed 16 | ) 17 | 18 | // Snapshot represents a snapshot event 19 | type Snapshot struct { 20 | ServerTime time.Time 21 | Data map[string]any 22 | EventType SnapshotEventType 23 | Table string 24 | Schema string 25 | LSN pq.LSN 26 | TotalRows int64 27 | IsLast bool 28 | } 29 | -------------------------------------------------------------------------------- /example/snapshot-with-scaling/Dockerfile: -------------------------------------------------------------------------------- 1 | #- Build Stage 2 | FROM --platform=linux/arm64 golang:1.23-alpine3.19 AS builder 3 | 4 | # Install git (required for go mod download) 5 | RUN apk add --no-cache git 6 | 7 | WORKDIR /build 8 | 9 | # Copy the entire project (context is project root) 10 | COPY . . 11 | 12 | # Change to the example app directory 13 | WORKDIR /build/example/snapshot-with-scaling 14 | 15 | # Download dependencies 16 | RUN go mod download 17 | 18 | # Build the application 19 | RUN CGO_ENABLED=0 go build -trimpath -a -v -o /go/bin/go-pq-cdc ./main.go 20 | 21 | #- Run Stage 22 | FROM --platform=linux/arm64 gcr.io/distroless/static-debian11 23 | 24 | WORKDIR /app 25 | 26 | COPY --from=builder /go/bin/go-pq-cdc ./ 27 | 28 | EXPOSE 2112 29 | 30 | CMD [ "./go-pq-cdc" ] -------------------------------------------------------------------------------- /internal/retry/retry.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/avast/retry-go/v4" 7 | ) 8 | 9 | var DefaultOptions = []retry.Option{ 10 | retry.LastErrorOnly(true), 11 | retry.Delay(time.Second), 12 | retry.DelayType(retry.FixedDelay), 13 | } 14 | 15 | type Config[T any] struct { 16 | If func(err error) bool 17 | Options []retry.Option 18 | } 19 | 20 | func (rc Config[T]) Do(f retry.RetryableFuncWithData[T]) (T, error) { 21 | return retry.DoWithData(f, rc.Options...) 22 | } 23 | 24 | func OnErrorConfig[T any](attemptCount uint, check func(error) bool) Config[T] { 25 | cfg := Config[T]{ 26 | If: check, 27 | Options: []retry.Option{retry.Attempts(attemptCount)}, 28 | } 29 | cfg.Options = append(cfg.Options, DefaultOptions...) 30 | return cfg 31 | } 32 | -------------------------------------------------------------------------------- /pq/slot/config.go: -------------------------------------------------------------------------------- 1 | package slot 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | "time" 7 | ) 8 | 9 | type Config struct { 10 | Name string `json:"name" yaml:"name"` 11 | SlotActivityCheckerInterval time.Duration `json:"slotActivityCheckerInterval" yaml:"slotActivityCheckerInterval"` 12 | CreateIfNotExists bool `json:"createIfNotExists" yaml:"createIfNotExists"` 13 | } 14 | 15 | func (c Config) Validate() error { 16 | var err error 17 | if strings.TrimSpace(c.Name) == "" { 18 | err = errors.Join(err, errors.New("slot name cannot be empty")) 19 | } 20 | 21 | if c.SlotActivityCheckerInterval < 1000 { 22 | err = errors.Join(err, errors.New("slot activity checker interval cannot be lower than 1000 ms")) 23 | } 24 | 25 | return err 26 | } 27 | -------------------------------------------------------------------------------- /example/simple-with-heartbeat/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | postgres: 4 | image: postgres:16.2 5 | environment: 6 | POSTGRES_DB: cdc_db 7 | POSTGRES_USER: cdc_user 8 | POSTGRES_PASSWORD: cdc_pass 9 | ports: 10 | - "5433:5432" 11 | command: 12 | - postgres 13 | - -c 14 | - wal_level=logical 15 | - -c 16 | - max_replication_slots=20 17 | - -c 18 | - max_wal_senders=25 19 | - -c 20 | - max_connections=100 21 | healthcheck: 22 | test: [ "CMD-SHELL", "pg_isready -U cdc_user -d cdc_db" ] 23 | interval: 5s 24 | timeout: 5s 25 | retries: 5 26 | volumes: 27 | - ./init.sql:/docker-entrypoint-initdb.d/init.sql 28 | - ./high_init.sql:/docker-entrypoint-initdb.d/high_init.sql -------------------------------------------------------------------------------- /benchmark/benchmark_cdc/go-pq-cdc-kafka/Dockerfile: -------------------------------------------------------------------------------- 1 | #- Build Stage 2 | FROM --platform=linux/arm64 golang:1.23-alpine3.19 AS builder 3 | 4 | # Install git (required for go mod download) 5 | RUN apk add --no-cache git 6 | 7 | WORKDIR /build 8 | 9 | # Copy the entire project (context is project root) 10 | COPY . . 11 | 12 | # Change to the benchmark app directory 13 | WORKDIR /build/benchmark/benchmark_cdc/go-pq-cdc-kafka 14 | 15 | # Download dependencies 16 | RUN go mod download 17 | 18 | # Build the application 19 | RUN CGO_ENABLED=0 go build -trimpath -a -v -o /go/bin/go-pq-cdc-kafka ./main.go 20 | 21 | #- Run Stage 22 | FROM --platform=linux/arm64 gcr.io/distroless/static-debian11 23 | 24 | WORKDIR /app 25 | 26 | COPY --from=builder /go/bin/go-pq-cdc-kafka ./ 27 | 28 | EXPOSE 2112 29 | 30 | CMD [ "./go-pq-cdc-kafka" ] -------------------------------------------------------------------------------- /benchmark/benchmark_initial/go-pq-cdc-kafka/Dockerfile: -------------------------------------------------------------------------------- 1 | #- Build Stage 2 | FROM --platform=linux/arm64 golang:1.23-alpine3.19 AS builder 3 | 4 | # Install git (required for go mod download) 5 | RUN apk add --no-cache git 6 | 7 | WORKDIR /build 8 | 9 | # Copy the entire project (context is project root) 10 | COPY . . 11 | 12 | # Change to the benchmark app directory 13 | WORKDIR /build/benchmark/benchmark_initial/go-pq-cdc-kafka 14 | 15 | # Download dependencies 16 | RUN go mod download 17 | 18 | # Build the application 19 | RUN CGO_ENABLED=0 go build -trimpath -a -v -o /go/bin/go-pq-cdc-kafka ./main.go 20 | 21 | #- Run Stage 22 | FROM --platform=linux/arm64 gcr.io/distroless/static-debian11 23 | 24 | WORKDIR /app 25 | 26 | COPY --from=builder /go/bin/go-pq-cdc-kafka ./ 27 | 28 | EXPOSE 2112 29 | 30 | CMD [ "./go-pq-cdc-kafka" ] -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 16 | 1. Do the following '...' 17 | 2. Run the project with '....' 18 | 3. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Version (please complete the following information):** 27 | 28 | - OS: [e.g. macOS] 29 | - Golang version [e.g. 1.17] 30 | - Postgres version 31 | 32 | **Additional context** 33 | Add any other context about the problem here. 34 | -------------------------------------------------------------------------------- /example/postgresql/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | postgres-source: 4 | image: postgres:16.2 5 | restart: always 6 | command: ["-c", "wal_level=logical", "-c", "max_wal_senders=10", "-c", "max_replication_slots=10"] 7 | environment: 8 | POSTGRES_USER: "cdc_user" 9 | POSTGRES_PASSWORD: "cdc_pass" 10 | POSTGRES_DB: "cdc_db" 11 | POSTGRES_HOST_AUTH_METHOD: trust 12 | network_mode: "host" 13 | postgres-target: 14 | image: postgres:16.2 15 | restart: always 16 | command: [ "-c", "wal_level=logical", "-c", "max_wal_senders=10", "-c", "max_replication_slots=10" ] 17 | environment: 18 | POSTGRES_USER: "cdc_user" 19 | POSTGRES_PASSWORD: "cdc_pass" 20 | POSTGRES_DB: "cdc_db" 21 | POSTGRES_HOST_AUTH_METHOD: trust 22 | ports: 23 | - "5433:5432" -------------------------------------------------------------------------------- /config/read.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | 7 | "github.com/go-playground/errors" 8 | "gopkg.in/yaml.v2" 9 | ) 10 | 11 | func ReadConfigYAML(path string) (Config, error) { 12 | b, err := os.ReadFile(path) 13 | if err != nil { 14 | return Config{}, errors.Wrap(err, "read yaml config") 15 | } 16 | 17 | c := Config{} 18 | 19 | err = yaml.Unmarshal(b, &c) 20 | if err != nil { 21 | return Config{}, errors.Wrap(err, "yaml config file parse") 22 | } 23 | 24 | return c, nil 25 | } 26 | 27 | func ReadConfigJSON(path string) (Config, error) { 28 | b, err := os.ReadFile(path) 29 | if err != nil { 30 | return Config{}, errors.Wrap(err, "read json config") 31 | } 32 | 33 | c := Config{} 34 | 35 | err = json.Unmarshal(b, &c) 36 | if err != nil { 37 | return Config{}, errors.Wrap(err, "json config file parse") 38 | } 39 | 40 | return c, nil 41 | } 42 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | funlen: 3 | lines: 85 4 | 5 | linters: 6 | disable-all: true 7 | enable: 8 | - bodyclose 9 | - dogsled 10 | - dupl 11 | - errcheck 12 | - errorlint 13 | - exportloopref 14 | - funlen 15 | - gocheckcompilerdirectives 16 | - gochecknoinits 17 | - goconst 18 | - gocritic 19 | - gocyclo 20 | - godox 21 | - gofmt 22 | - goimports 23 | - goprintffuncname 24 | - gosec 25 | - gosimple 26 | - govet 27 | - ineffassign 28 | - misspell 29 | - nakedret 30 | - noctx 31 | - nolintlint 32 | - revive 33 | - staticcheck 34 | - stylecheck 35 | - testifylint 36 | - unconvert 37 | - unparam 38 | - unused 39 | - whitespace 40 | 41 | issues: 42 | exclude-rules: 43 | - path: (.+)_test.go 44 | linters: 45 | - funlen 46 | - goconst 47 | - dupl 48 | -------------------------------------------------------------------------------- /logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "log/slog" 5 | "os" 6 | "sync" 7 | ) 8 | 9 | var _default Logger 10 | 11 | type Logger interface { 12 | Debug(msg string, args ...any) 13 | Info(msg string, args ...any) 14 | Warn(msg string, args ...any) 15 | Error(msg string, args ...any) 16 | } 17 | 18 | var once sync.Once 19 | 20 | func InitLogger(l Logger) { 21 | once.Do(func() { 22 | _default = l 23 | }) 24 | } 25 | 26 | func Debug(msg string, args ...any) { 27 | _default.Debug(msg, args...) 28 | } 29 | 30 | func Info(msg string, args ...any) { 31 | _default.Info(msg, args...) 32 | } 33 | 34 | func Warn(msg string, args ...any) { 35 | _default.Warn(msg, args...) 36 | } 37 | 38 | func Error(msg string, args ...any) { 39 | _default.Error(msg, args...) 40 | } 41 | 42 | func NewSlog(logLevel slog.Level) Logger { 43 | return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel})) 44 | } 45 | -------------------------------------------------------------------------------- /.github/workflows/integration_test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | types: [ opened, reopened, synchronize ] 9 | workflow_dispatch: 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | integration_test: 17 | runs-on: ubuntu-latest 18 | strategy: 19 | matrix: 20 | version: [ "15-alpine", "15.7-alpine", "16-alpine", "16.3-alpine" ] 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v4 24 | with: 25 | fetch-depth: 1 26 | 27 | - name: Set up Go 28 | uses: actions/setup-go@v4 29 | with: 30 | go-version: '1.22.4' 31 | 32 | - name: Install dependencies 33 | run: make tidy 34 | 35 | - name: Integration Test 36 | run: | 37 | make test/integration 38 | env: 39 | POSTGRES_TEST_IMAGE: ${{ matrix.version }} 40 | -------------------------------------------------------------------------------- /benchmark/benchmark_initial/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 100ms 3 | evaluation_interval: 100ms 4 | scrape_configs: 5 | - job_name: prometheus 6 | static_configs: 7 | - targets: ["localhost:9090"] 8 | - job_name: cadvisor 9 | static_configs: 10 | - targets: [ "cadvisor:8080" ] 11 | - job_name: postgres-exporter 12 | static_configs: 13 | - targets: ["postgres-exporter:9187"] 14 | - job_name: es-exporter 15 | static_configs: 16 | - targets: [ "es-exporter:9114" ] 17 | - job_name: go_pq_cdc_exporter 18 | # DNS service discovery ile tüm scaled instance'ları scrape edebiliriz 19 | dns_sd_configs: 20 | - names: 21 | - 'tasks.go-pq-cdc-kafka' 22 | type: 'A' 23 | port: 2112 24 | refresh_interval: 5s 25 | # Fallback için static config de ekleyelim 26 | static_configs: 27 | - targets: [ "go-pq-cdc-kafka:2112" ] 28 | - job_name: redpanda_exporter 29 | metrics_path: "/metrics" 30 | static_configs: 31 | - targets: [ "redpanda:9644" ] -------------------------------------------------------------------------------- /pq/message/format/relation_test.go: -------------------------------------------------------------------------------- 1 | package format 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/Trendyol/go-pq-cdc/pq/message/tuple" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestRelation_New(t *testing.T) { 11 | data := []byte{82, 0, 0, 64, 6, 112, 117, 98, 108, 105, 99, 0, 116, 0, 100, 0, 2, 1, 105, 100, 0, 0, 0, 0, 23, 255, 255, 255, 255, 0, 110, 97, 109, 101, 0, 0, 0, 0, 25, 255, 255, 255, 255} 12 | 13 | rel, err := NewRelation(data, false) 14 | if err != nil { 15 | t.Fatal(err) 16 | } 17 | 18 | expected := &Relation{ 19 | OID: 16390, 20 | XID: 0, 21 | Namespace: "public", 22 | Name: "t", 23 | ReplicaID: 100, 24 | ColumnNumbers: 2, 25 | Columns: []tuple.RelationColumn{ 26 | { 27 | Flags: 1, 28 | Name: "id", 29 | DataType: 23, 30 | TypeModifier: 4294967295, 31 | }, 32 | { 33 | Flags: 0, 34 | Name: "name", 35 | DataType: 25, 36 | TypeModifier: 4294967295, 37 | }, 38 | }, 39 | } 40 | 41 | assert.Equal(t, expected, rel) 42 | } 43 | -------------------------------------------------------------------------------- /internal/metric/registry.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/prometheus/client_golang/prometheus/collectors" 8 | ) 9 | 10 | type Registry interface { 11 | AddMetricCollectors(metricCollectors ...prometheus.Collector) 12 | Prometheus() *prometheus.Registry 13 | } 14 | 15 | type prometheusRegistry struct { 16 | registry *prometheus.Registry 17 | } 18 | 19 | func NewRegistry(m Metric) Registry { 20 | r := prometheus.NewRegistry() 21 | r.MustRegister(collectors.NewBuildInfoCollector()) 22 | r.MustRegister(collectors.NewGoCollector( 23 | collectors.WithGoCollectorRuntimeMetrics(collectors.GoRuntimeMetricsRule{Matcher: regexp.MustCompile("/.*")}), 24 | )) 25 | r.MustRegister(m.PrometheusCollectors()...) 26 | 27 | return &prometheusRegistry{ 28 | registry: r, 29 | } 30 | } 31 | 32 | func (r *prometheusRegistry) AddMetricCollectors(metricCollectors ...prometheus.Collector) { 33 | r.registry.MustRegister(metricCollectors...) 34 | } 35 | 36 | func (r *prometheusRegistry) Prometheus() *prometheus.Registry { 37 | return r.registry 38 | } 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Trendyol 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /example/snapshot-with-scaling/go.mod: -------------------------------------------------------------------------------- 1 | module basic-snapshot-with-scaling 2 | 3 | go 1.22.4 4 | 5 | replace github.com/Trendyol/go-pq-cdc => ../../ 6 | 7 | require github.com/Trendyol/go-pq-cdc v0.0.0-00010101000000-000000000000 8 | 9 | require ( 10 | github.com/avast/retry-go/v4 v4.6.0 // indirect 11 | github.com/beorn7/perks v1.0.1 // indirect 12 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 13 | github.com/go-playground/errors v3.3.0+incompatible // indirect 14 | github.com/jackc/pgpassfile v1.0.0 // indirect 15 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect 16 | github.com/jackc/pgx/v5 v5.6.0 // indirect 17 | github.com/lib/pq v1.10.9 // indirect 18 | github.com/prometheus/client_golang v1.19.1 // indirect 19 | github.com/prometheus/client_model v0.5.0 // indirect 20 | github.com/prometheus/common v0.48.0 // indirect 21 | github.com/prometheus/procfs v0.12.0 // indirect 22 | golang.org/x/crypto v0.22.0 // indirect 23 | golang.org/x/sys v0.19.0 // indirect 24 | golang.org/x/text v0.14.0 // indirect 25 | google.golang.org/protobuf v1.34.1 // indirect 26 | gopkg.in/yaml.v2 v2.4.0 // indirect 27 | ) 28 | -------------------------------------------------------------------------------- /example/simple-file-config/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Trendyol/go-pq-cdc/example/simple-file-config 2 | 3 | go 1.22.4 4 | 5 | replace github.com/Trendyol/go-pq-cdc => ../.. 6 | 7 | require github.com/Trendyol/go-pq-cdc v0.0.0-00010101000000-000000000000 8 | 9 | require ( 10 | github.com/avast/retry-go/v4 v4.6.0 // indirect 11 | github.com/beorn7/perks v1.0.1 // indirect 12 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 13 | github.com/go-playground/errors v3.3.0+incompatible // indirect 14 | github.com/jackc/pgpassfile v1.0.0 // indirect 15 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect 16 | github.com/jackc/pgx/v5 v5.6.0 // indirect 17 | github.com/lib/pq v1.10.9 // indirect 18 | github.com/prometheus/client_golang v1.19.1 // indirect 19 | github.com/prometheus/client_model v0.5.0 // indirect 20 | github.com/prometheus/common v0.48.0 // indirect 21 | github.com/prometheus/procfs v0.12.0 // indirect 22 | golang.org/x/crypto v0.22.0 // indirect 23 | golang.org/x/sys v0.19.0 // indirect 24 | golang.org/x/text v0.14.0 // indirect 25 | google.golang.org/protobuf v1.34.1 // indirect 26 | gopkg.in/yaml.v2 v2.4.0 // indirect 27 | ) 28 | -------------------------------------------------------------------------------- /pq/replication/wal.go: -------------------------------------------------------------------------------- 1 | package replication 2 | 3 | import ( 4 | "encoding/binary" 5 | "time" 6 | 7 | "github.com/go-playground/errors" 8 | 9 | "github.com/Trendyol/go-pq-cdc/pq" 10 | ) 11 | 12 | // The server's system clock at the time of transmission, as microseconds since midnight on 2000-01-01. 13 | // microSecFromUnixEpochToY2K is unix timestamp of 2000-01-01. 14 | var microSecFromUnixEpochToY2K = int64(946684800) 15 | 16 | type XLogData struct { 17 | ServerTime time.Time 18 | WALData []byte 19 | WALStart pq.LSN 20 | ServerWALEnd pq.LSN 21 | } 22 | 23 | func ParseXLogData(buf []byte) (XLogData, error) { 24 | var xld XLogData 25 | if len(buf) < 24 { 26 | return xld, errors.Newf("XLogData must be at least 24 bytes, got %d", len(buf)) 27 | } 28 | 29 | xld.WALStart = pq.LSN(binary.BigEndian.Uint64(buf)) 30 | xld.ServerWALEnd = pq.LSN(binary.BigEndian.Uint64(buf[8:])) 31 | xld.ServerTime = pgTimeToTime(int64(binary.BigEndian.Uint64(buf[16:]))) 32 | xld.WALData = buf[24:] 33 | 34 | return xld, nil 35 | } 36 | 37 | func pgTimeToTime(microSecSinceY2K int64) time.Time { 38 | return time.Unix(microSecFromUnixEpochToY2K+(microSecSinceY2K/1_000_000), (microSecSinceY2K%1_000_000)*1_000).UTC() 39 | } 40 | -------------------------------------------------------------------------------- /pq/publication/operation.go: -------------------------------------------------------------------------------- 1 | package publication 2 | 3 | import ( 4 | "slices" 5 | "strings" 6 | 7 | "github.com/go-playground/errors" 8 | ) 9 | 10 | var OperationOptions = Operations{"INSERT", "UPDATE", "DELETE", "TRUNCATE"} 11 | 12 | var ( 13 | OperationInsert Operation = "INSERT" 14 | OperationUpdate Operation = "UPDATE" 15 | OperationDelete Operation = "DELETE" 16 | OperationTruncate Operation = "TRUNCATE" 17 | ) 18 | 19 | type Operation string 20 | 21 | func (op Operation) Validate() error { 22 | if !slices.Contains(OperationOptions, op) { 23 | return errors.Newf("undefined operation. valid operations are: %v", OperationOptions) 24 | } 25 | 26 | return nil 27 | } 28 | 29 | type Operations []Operation 30 | 31 | func (ops Operations) Validate() error { 32 | if len(ops) == 0 { 33 | return errors.New("at least one operation must be defined") 34 | } 35 | 36 | for _, op := range ops { 37 | if err := op.Validate(); err != nil { 38 | return err 39 | } 40 | } 41 | 42 | return nil 43 | } 44 | 45 | func (ops Operations) String() string { 46 | res := make([]string, len(ops)) 47 | 48 | for i, op := range ops { 49 | res[i] = string(op) 50 | } 51 | 52 | return strings.Join(res, ", ") 53 | } 54 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | types: [opened, reopened, synchronize] 9 | workflow_dispatch: 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | build: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | with: 22 | fetch-depth: 1 23 | 24 | - name: Set up Go 25 | uses: actions/setup-go@v4 26 | with: 27 | go-version: '1.22.4' 28 | 29 | - name: Lint 30 | run: | 31 | make lint 32 | 33 | - name: Install dependencies 34 | run: | 35 | make tidy 36 | 37 | - name: Build 38 | run: | 39 | make build/linux 40 | 41 | - name: Unit Test 42 | run: go test -v ./... 43 | 44 | security-gates: 45 | uses: Trendyol/security-actions/.github/workflows/security-gates.yml@master 46 | needs: build 47 | permissions: 48 | actions: read 49 | contents: read 50 | security-events: write 51 | secrets: inherit 52 | -------------------------------------------------------------------------------- /example/postgresql/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Trendyol/go-pq-cdc/example/postgresql 2 | 3 | go 1.22.4 4 | 5 | replace github.com/Trendyol/go-pq-cdc => ../.. 6 | 7 | require ( 8 | github.com/Trendyol/go-pq-cdc v0.0.0-00010101000000-000000000000 9 | github.com/jackc/pgx/v5 v5.6.0 10 | ) 11 | 12 | require ( 13 | github.com/avast/retry-go/v4 v4.6.0 // indirect 14 | github.com/beorn7/perks v1.0.1 // indirect 15 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 16 | github.com/go-playground/errors v3.3.0+incompatible // indirect 17 | github.com/jackc/pgpassfile v1.0.0 // indirect 18 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect 19 | github.com/jackc/puddle/v2 v2.2.1 // indirect 20 | github.com/lib/pq v1.10.9 // indirect 21 | github.com/prometheus/client_golang v1.19.1 // indirect 22 | github.com/prometheus/client_model v0.5.0 // indirect 23 | github.com/prometheus/common v0.48.0 // indirect 24 | github.com/prometheus/procfs v0.12.0 // indirect 25 | golang.org/x/crypto v0.22.0 // indirect 26 | golang.org/x/sync v0.6.0 // indirect 27 | golang.org/x/sys v0.19.0 // indirect 28 | golang.org/x/text v0.14.0 // indirect 29 | google.golang.org/protobuf v1.34.1 // indirect 30 | gopkg.in/yaml.v2 v2.4.0 // indirect 31 | ) 32 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Trendyol/go-pq-cdc 2 | 3 | go 1.22.4 4 | 5 | require ( 6 | github.com/avast/retry-go/v4 v4.6.0 7 | github.com/go-playground/errors v3.3.0+incompatible 8 | github.com/jackc/pgx/v5 v5.6.0 9 | github.com/lib/pq v1.10.9 10 | github.com/prometheus/client_golang v1.19.1 11 | github.com/stretchr/testify v1.9.0 12 | gopkg.in/yaml.v2 v2.4.0 13 | ) 14 | 15 | require ( 16 | github.com/beorn7/perks v1.0.1 // indirect 17 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 18 | github.com/davecgh/go-spew v1.1.1 // indirect 19 | github.com/jackc/pgpassfile v1.0.0 // indirect 20 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect 21 | github.com/kr/text v0.2.0 // indirect 22 | github.com/pmezard/go-difflib v1.0.0 // indirect 23 | github.com/prometheus/client_model v0.5.0 // indirect 24 | github.com/prometheus/common v0.48.0 // indirect 25 | github.com/prometheus/procfs v0.12.0 // indirect 26 | github.com/rogpeppe/go-internal v1.12.0 // indirect 27 | golang.org/x/crypto v0.22.0 // indirect 28 | golang.org/x/sync v0.6.0 // indirect 29 | golang.org/x/sys v0.19.0 // indirect 30 | golang.org/x/text v0.14.0 // indirect 31 | google.golang.org/protobuf v1.34.1 // indirect 32 | gopkg.in/yaml.v3 v3.0.1 // indirect 33 | ) 34 | -------------------------------------------------------------------------------- /example/snapshot-with-scaling/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | go-pq-cdc: 4 | build: 5 | context: ../../ 6 | dockerfile: example/snapshot-with-scaling/Dockerfile 7 | restart: on-failure 8 | expose: 9 | - "2112" 10 | depends_on: 11 | postgres: 12 | condition: service_healthy 13 | networks: 14 | - scaling_net 15 | deploy: 16 | resources: 17 | limits: 18 | cpus: "1" 19 | memory: 512M 20 | reservations: 21 | cpus: '0.25' 22 | memory: 128M 23 | 24 | postgres: 25 | image: postgres:16.2 26 | environment: 27 | POSTGRES_DB: cdc_db 28 | POSTGRES_USER: cdc_user 29 | POSTGRES_PASSWORD: cdc_pass 30 | ports: 31 | - "5433:5432" 32 | command: 33 | - postgres 34 | - -c 35 | - wal_level=logical 36 | - -c 37 | - max_replication_slots=20 38 | - -c 39 | - max_wal_senders=25 40 | - -c 41 | - max_connections=100 42 | healthcheck: 43 | test: [ "CMD-SHELL", "pg_isready -U cdc_user -d cdc_db" ] 44 | interval: 5s 45 | timeout: 5s 46 | retries: 5 47 | volumes: 48 | - ./init.sql:/docker-entrypoint-initdb.d/init.sql 49 | networks: 50 | - scaling_net 51 | 52 | networks: 53 | scaling_net: 54 | driver: bridge -------------------------------------------------------------------------------- /example/simple-file-config/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | cdc "github.com/Trendyol/go-pq-cdc" 6 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 7 | "github.com/Trendyol/go-pq-cdc/pq/replication" 8 | "log/slog" 9 | "os" 10 | ) 11 | 12 | /* 13 | psql "postgres://cdc_user:cdc_pass@127.0.0.1/cdc_db?replication=database" 14 | 15 | CREATE TABLE users ( 16 | id serial PRIMARY KEY, 17 | name text NOT NULL, 18 | created_on timestamptz 19 | ); 20 | 21 | INSERT INTO users (name) 22 | SELECT 23 | 'Oyleli' || i 24 | FROM generate_series(1, 100) AS i; 25 | */ 26 | 27 | func main() { 28 | ctx := context.Background() 29 | 30 | connector, err := cdc.NewConnectorWithConfigFile(ctx, "./config.yml", Handler) 31 | if err != nil { 32 | slog.Error("new connector", "error", err) 33 | os.Exit(1) 34 | } 35 | 36 | defer connector.Close() 37 | connector.Start(ctx) 38 | } 39 | 40 | func Handler(ctx *replication.ListenerContext) { 41 | switch msg := ctx.Message.(type) { 42 | case *format.Insert: 43 | slog.Info("insert message received", "new", msg.Decoded) 44 | case *format.Delete: 45 | slog.Info("delete message received", "old", msg.OldDecoded) 46 | case *format.Update: 47 | slog.Info("update message received", "new", msg.NewDecoded, "old", msg.OldDecoded) 48 | } 49 | 50 | if err := ctx.Ack(); err != nil { 51 | slog.Error("ack", "error", err) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pq/publication/table.go: -------------------------------------------------------------------------------- 1 | package publication 2 | 3 | import ( 4 | "slices" 5 | "strings" 6 | 7 | "github.com/go-playground/errors" 8 | ) 9 | 10 | type Table struct { 11 | Name string `json:"name" yaml:"name"` 12 | ReplicaIdentity string `json:"replicaIdentity" yaml:"replicaIdentity"` 13 | Schema string `json:"schema,omitempty" yaml:"schema,omitempty"` 14 | } 15 | 16 | func (tc Table) Validate() error { 17 | if strings.TrimSpace(tc.Name) == "" { 18 | return errors.New("table name cannot be empty") 19 | } 20 | 21 | if !slices.Contains(ReplicaIdentityOptions, tc.ReplicaIdentity) { 22 | return errors.Newf("undefined replica identity option. valid identity options are: %v", ReplicaIdentityOptions) 23 | } 24 | 25 | return nil 26 | } 27 | 28 | type Tables []Table 29 | 30 | func (ts Tables) Validate() error { 31 | if len(ts) == 0 { 32 | return errors.New("at least one table must be defined") 33 | } 34 | 35 | for _, t := range ts { 36 | if err := t.Validate(); err != nil { 37 | return err 38 | } 39 | } 40 | 41 | return nil 42 | } 43 | 44 | func (ts Tables) Diff(tss Tables) Tables { 45 | res := Tables{} 46 | tssMap := make(map[string]Table) 47 | 48 | for _, t := range tss { 49 | tssMap[t.Name+t.ReplicaIdentity] = t 50 | } 51 | 52 | for _, t := range ts { 53 | if v, found := tssMap[t.Name+t.ReplicaIdentity]; !found || v.ReplicaIdentity != t.ReplicaIdentity { 54 | res = append(res, t) 55 | } 56 | } 57 | 58 | return res 59 | } 60 | -------------------------------------------------------------------------------- /pq/message/format/insert_test.go: -------------------------------------------------------------------------------- 1 | package format 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/Trendyol/go-pq-cdc/pq/message/tuple" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestInsert_New(t *testing.T) { 12 | data := []byte{73, 0, 0, 64, 6, 78, 0, 2, 116, 0, 0, 0, 3, 54, 48, 53, 116, 0, 0, 0, 3, 102, 111, 111} 13 | 14 | rel := map[uint32]*Relation{ 15 | 16390: { 16 | OID: 16390, 17 | XID: 0, 18 | Namespace: "public", 19 | Name: "t", 20 | ReplicaID: 100, 21 | ColumnNumbers: 2, 22 | Columns: []tuple.RelationColumn{ 23 | { 24 | Flags: 1, 25 | Name: "id", 26 | DataType: 23, 27 | TypeModifier: 4294967295, 28 | }, 29 | { 30 | Flags: 0, 31 | Name: "name", 32 | DataType: 25, 33 | TypeModifier: 4294967295, 34 | }, 35 | }, 36 | }, 37 | } 38 | 39 | now := time.Now() 40 | msg, err := NewInsert(data, false, rel, now) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | expected := &Insert{ 46 | OID: 16390, 47 | XID: 0, 48 | TupleData: &tuple.Data{ 49 | ColumnNumber: 2, 50 | Columns: tuple.DataColumns{ 51 | {DataType: 116, Length: 3, Data: []byte{'6', '0', '5'}}, 52 | {DataType: 116, Length: 3, Data: []byte{'f', 'o', 'o'}}, 53 | }, 54 | SkipByte: 24, 55 | }, 56 | Decoded: map[string]any{"id": int32(605), "name": "foo"}, 57 | TableNamespace: "public", 58 | TableName: "t", 59 | MessageTime: now, 60 | } 61 | 62 | assert.EqualValues(t, expected, msg) 63 | } 64 | -------------------------------------------------------------------------------- /benchmark/benchmark_cdc/go-pq-cdc-kafka/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Trendyol/go-pq-cdc/benchmark/go-pq-cdc-kafka 2 | 3 | go 1.22.4 4 | 5 | replace github.com/Trendyol/go-pq-cdc => ../../../ 6 | 7 | require ( 8 | github.com/Trendyol/go-pq-cdc v1.0.3 9 | github.com/Trendyol/go-pq-cdc-kafka v1.0.3 10 | github.com/segmentio/kafka-go v0.4.47 11 | ) 12 | 13 | require ( 14 | github.com/avast/retry-go/v4 v4.6.0 // indirect 15 | github.com/beorn7/perks v1.0.1 // indirect 16 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 17 | github.com/go-playground/errors v3.3.0+incompatible // indirect 18 | github.com/jackc/pgpassfile v1.0.0 // indirect 19 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect 20 | github.com/jackc/pgx/v5 v5.6.0 // indirect 21 | github.com/klauspost/compress v1.18.0 // indirect 22 | github.com/lib/pq v1.10.9 // indirect 23 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 24 | github.com/pierrec/lz4/v4 v4.1.21 // indirect 25 | github.com/pkg/errors v0.9.1 // indirect 26 | github.com/prometheus/client_golang v1.22.0 // indirect 27 | github.com/prometheus/client_model v0.6.1 // indirect 28 | github.com/prometheus/common v0.62.0 // indirect 29 | github.com/prometheus/procfs v0.15.1 // indirect 30 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 31 | github.com/xdg-go/scram v1.1.2 // indirect 32 | github.com/xdg-go/stringprep v1.0.4 // indirect 33 | golang.org/x/crypto v0.24.0 // indirect 34 | golang.org/x/sys v0.30.0 // indirect 35 | golang.org/x/text v0.21.0 // indirect 36 | google.golang.org/protobuf v1.36.5 // indirect 37 | gopkg.in/yaml.v2 v2.4.0 // indirect 38 | ) 39 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: init 2 | 3 | .PHONY: init 4 | init: init/lint 5 | 6 | .PHONY: init/lint init/vulnCheck 7 | init/lint: 8 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.1 9 | go install golang.org/x/tools/go/analysis/passes/fieldalignment/cmd/fieldalignment@v0.22.0 10 | 11 | .PHONY: init/vulnCheck 12 | init/vulnCheck: 13 | go install golang.org/x/vuln/cmd/govulncheck@latest 14 | 15 | .PHONY: audit 16 | audit: vendor 17 | @echo 'Formatting code...' 18 | fieldalignment -fix ./... 19 | golangci-lint run -c .golangci.yml --timeout=5m -v --fix 20 | @echo 'Vetting code...' 21 | go vet ./... 22 | @echo 'Vulnerability scanning...' 23 | govulncheck ./... 24 | 25 | .PHONY: tidy 26 | tidy: 27 | @echo 'Tidying and verifying module dependencies...' 28 | go mod tidy -compat=1.22.4 29 | go mod verify 30 | 31 | .PHONY: tidy/all 32 | tidy/all: 33 | go mod tidy 34 | cd example/elasticsearch && go mod tidy && cd ../.. 35 | cd example/kafka && go mod tidy && cd ../.. 36 | cd example/postgresql && go mod tidy && cd ../.. 37 | cd example/simple && go mod tidy && cd ../.. 38 | cd example/simple-file-config && go mod tidy && cd ../.. 39 | cd integration_test && go mod tidy && cd ../ 40 | cd benchmark/go-pq-cdc-kafka && go mod tidy && cd ../.. 41 | 42 | .PHONY: test/integration 43 | test/integration: 44 | cd integration_test && go test -race -p=1 -v ./... 45 | 46 | .PHONY: lint 47 | lint: init/lint 48 | @echo 'Formatting code...' 49 | fieldalignment -fix ./... 50 | golangci-lint run -c .golangci.yml --timeout=5m -v --fix 51 | 52 | .PHONY: build 53 | build/linux: 54 | GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -trimpath -a -v -------------------------------------------------------------------------------- /pq/message/format/delete_test.go: -------------------------------------------------------------------------------- 1 | package format 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/Trendyol/go-pq-cdc/pq/message/tuple" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestDelete_New(t *testing.T) { 12 | data := []byte{68, 0, 0, 64, 6, 79, 0, 2, 116, 0, 0, 0, 3, 54, 52, 53, 116, 0, 0, 0, 3, 102, 111, 111} 13 | 14 | rel := map[uint32]*Relation{ 15 | 16390: { 16 | OID: 16390, 17 | XID: 0, 18 | Namespace: "public", 19 | Name: "t", 20 | ReplicaID: 100, 21 | ColumnNumbers: 2, 22 | Columns: []tuple.RelationColumn{ 23 | { 24 | Flags: 1, 25 | Name: "id", 26 | DataType: 23, 27 | TypeModifier: 4294967295, 28 | }, 29 | { 30 | Flags: 0, 31 | Name: "name", 32 | DataType: 25, 33 | TypeModifier: 4294967295, 34 | }, 35 | }, 36 | }, 37 | } 38 | 39 | now := time.Now() 40 | msg, err := NewDelete(data, false, rel, now) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | expected := &Delete{ 46 | OID: 16390, 47 | XID: 0, 48 | OldTupleType: 79, 49 | OldTupleData: &tuple.Data{ 50 | ColumnNumber: 2, 51 | Columns: tuple.DataColumns{ 52 | { 53 | DataType: 116, 54 | Length: 3, 55 | Data: []byte("645"), 56 | }, 57 | { 58 | DataType: 116, 59 | Length: 3, 60 | Data: []byte("foo"), 61 | }, 62 | }, 63 | SkipByte: 24, 64 | }, 65 | OldDecoded: map[string]any{ 66 | "id": int32(645), 67 | "name": "foo", 68 | }, 69 | TableNamespace: "public", 70 | TableName: "t", 71 | MessageTime: now, 72 | } 73 | 74 | assert.Equal(t, expected, msg) 75 | } 76 | -------------------------------------------------------------------------------- /pq/message/message.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 7 | "github.com/go-playground/errors" 8 | ) 9 | 10 | const ( 11 | StreamAbortByte Type = 'A' 12 | BeginByte Type = 'B' 13 | CommitByte Type = 'C' 14 | DeleteByte Type = 'D' 15 | StreamStopByte Type = 'E' 16 | InsertByte Type = 'I' 17 | LogicalByte Type = 'M' 18 | OriginByte Type = 'O' 19 | RelationByte Type = 'R' 20 | StreamStartByte Type = 'S' 21 | TruncateByte Type = 'T' 22 | UpdateByte Type = 'U' 23 | TypeByte Type = 'Y' 24 | StreamCommitByte Type = 'c' 25 | ) 26 | 27 | const ( 28 | XLogDataByteID = 'w' 29 | PrimaryKeepaliveMessageByteID = 'k' 30 | ) 31 | 32 | var ErrorByteNotSupported = errors.New("message byte not supported") 33 | 34 | type Type uint8 35 | 36 | var streamedTransaction bool 37 | 38 | func New(data []byte, serverTime time.Time, relation map[uint32]*format.Relation) (any, error) { 39 | switch Type(data[0]) { 40 | case InsertByte: 41 | return format.NewInsert(data, streamedTransaction, relation, serverTime) 42 | case UpdateByte: 43 | return format.NewUpdate(data, streamedTransaction, relation, serverTime) 44 | case DeleteByte: 45 | return format.NewDelete(data, streamedTransaction, relation, serverTime) 46 | case StreamStopByte, StreamAbortByte, StreamCommitByte: 47 | streamedTransaction = false 48 | return nil, nil 49 | case RelationByte: 50 | msg, err := format.NewRelation(data, streamedTransaction) 51 | if err == nil { 52 | relation[msg.OID] = msg 53 | } 54 | return msg, err 55 | case StreamStartByte: 56 | streamedTransaction = true 57 | return nil, nil 58 | default: 59 | return nil, errors.Wrap(ErrorByteNotSupported, string(data[0])) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /benchmark/benchmark_initial/go-pq-cdc-kafka/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Trendyol/go-pq-cdc/benchmark/go-pq-cdc-kafka 2 | 3 | go 1.22.4 4 | 5 | replace github.com/Trendyol/go-pq-cdc => ../../../ 6 | 7 | require ( 8 | github.com/Trendyol/go-pq-cdc v1.0.3-0.20251116190457-05f4e2ce11e2 9 | github.com/Trendyol/go-pq-cdc-kafka v1.0.3-0.20251116202558-0e24ac547fe0 10 | github.com/json-iterator/go v1.1.12 11 | github.com/segmentio/kafka-go v0.4.47 12 | ) 13 | 14 | require ( 15 | github.com/avast/retry-go/v4 v4.6.0 // indirect 16 | github.com/beorn7/perks v1.0.1 // indirect 17 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 18 | github.com/go-playground/errors v3.3.0+incompatible // indirect 19 | github.com/jackc/pgpassfile v1.0.0 // indirect 20 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect 21 | github.com/jackc/pgx/v5 v5.6.0 // indirect 22 | github.com/klauspost/compress v1.18.0 // indirect 23 | github.com/lib/pq v1.10.9 // indirect 24 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 25 | github.com/modern-go/reflect2 v1.0.2 // indirect 26 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 27 | github.com/pierrec/lz4/v4 v4.1.21 // indirect 28 | github.com/pkg/errors v0.9.1 // indirect 29 | github.com/prometheus/client_golang v1.22.0 // indirect 30 | github.com/prometheus/client_model v0.6.1 // indirect 31 | github.com/prometheus/common v0.62.0 // indirect 32 | github.com/prometheus/procfs v0.15.1 // indirect 33 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 34 | github.com/xdg-go/scram v1.1.2 // indirect 35 | github.com/xdg-go/stringprep v1.0.4 // indirect 36 | golang.org/x/crypto v0.24.0 // indirect 37 | golang.org/x/sys v0.30.0 // indirect 38 | golang.org/x/text v0.21.0 // indirect 39 | google.golang.org/protobuf v1.36.5 // indirect 40 | gopkg.in/yaml.v2 v2.4.0 // indirect 41 | ) 42 | -------------------------------------------------------------------------------- /pq/snapshot/decoder_cache.go: -------------------------------------------------------------------------------- 1 | package snapshot 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/jackc/pgx/v5/pgtype" 7 | ) 8 | 9 | // TypeDecoder wraps type decoding functionality 10 | type TypeDecoder struct { 11 | oid uint32 12 | } 13 | 14 | // Decode decodes column data using the cached decoder 15 | func (d *TypeDecoder) Decode(typeMap *pgtype.Map, data []byte) (interface{}, error) { 16 | if dt, ok := typeMap.TypeForOID(d.oid); ok { 17 | return dt.Codec.DecodeValue(typeMap, d.oid, pgtype.TextFormatCode, data) 18 | } 19 | return string(data), nil 20 | } 21 | 22 | // DecoderCache caches type decoders by OID to avoid repeated reflection lookups 23 | // This eliminates the overhead of TypeForOID calls for every column in every row 24 | type DecoderCache struct { 25 | cache map[uint32]*TypeDecoder 26 | mu sync.RWMutex 27 | } 28 | 29 | // NewDecoderCache creates a new decoder cache 30 | func NewDecoderCache() *DecoderCache { 31 | return &DecoderCache{ 32 | cache: make(map[uint32]*TypeDecoder, 50), // Pre-allocate for common types 33 | } 34 | } 35 | 36 | // Get retrieves or creates a decoder for the given OID 37 | func (c *DecoderCache) Get(oid uint32) *TypeDecoder { 38 | // Fast path: read lock for cache hit 39 | c.mu.RLock() 40 | decoder, exists := c.cache[oid] 41 | c.mu.RUnlock() 42 | 43 | if exists { 44 | return decoder 45 | } 46 | 47 | // Slow path: write lock for cache miss 48 | c.mu.Lock() 49 | defer c.mu.Unlock() 50 | 51 | // Double-check after acquiring write lock (another goroutine may have added it) 52 | if decoder, exists := c.cache[oid]; exists { 53 | return decoder 54 | } 55 | 56 | // Create new decoder 57 | decoder = &TypeDecoder{oid: oid} 58 | c.cache[oid] = decoder 59 | return decoder 60 | } 61 | 62 | // Size returns the number of cached decoders 63 | func (c *DecoderCache) Size() int { 64 | c.mu.RLock() 65 | defer c.mu.RUnlock() 66 | return len(c.cache) 67 | } 68 | -------------------------------------------------------------------------------- /internal/http/server.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "net/http/pprof" 9 | "time" 10 | 11 | "github.com/Trendyol/go-pq-cdc/config" 12 | "github.com/Trendyol/go-pq-cdc/internal/metric" 13 | "github.com/Trendyol/go-pq-cdc/logger" 14 | "github.com/prometheus/client_golang/prometheus/promhttp" 15 | ) 16 | 17 | type Server interface { 18 | Listen() 19 | Shutdown() 20 | } 21 | 22 | type server struct { 23 | server http.Server 24 | cdcConfig config.Config 25 | closed bool 26 | } 27 | 28 | func NewServer(cfg config.Config, registry metric.Registry) Server { 29 | mux := http.NewServeMux() 30 | 31 | mux.Handle("GET /metrics", promhttp.HandlerFor(registry.Prometheus(), promhttp.HandlerOpts{EnableOpenMetrics: true})) 32 | 33 | mux.HandleFunc("GET /status", func(w http.ResponseWriter, _ *http.Request) { 34 | _, _ = w.Write([]byte("OK")) 35 | }) 36 | 37 | if cfg.DebugMode { 38 | mux.Handle("GET /pprof", pprof.Handler("go-pq-cdc")) 39 | } 40 | 41 | return &server{ 42 | server: http.Server{ 43 | Addr: fmt.Sprintf(":%d", cfg.Metric.Port), 44 | Handler: mux, 45 | ReadTimeout: 15 * time.Second, 46 | WriteTimeout: 15 * time.Second, 47 | }, 48 | cdcConfig: cfg, 49 | } 50 | } 51 | 52 | func (s *server) Listen() { 53 | logger.Info(fmt.Sprintf("server starting on port :%d", s.cdcConfig.Metric.Port)) 54 | 55 | err := s.server.ListenAndServe() 56 | if err != nil { 57 | if errors.Is(err, http.ErrServerClosed) && s.closed { 58 | logger.Info("server stopped") 59 | return 60 | } 61 | logger.Error("server cannot start", "port", s.cdcConfig.Metric.Port, "error", err) 62 | } 63 | } 64 | 65 | func (s *server) Shutdown() { 66 | if s == nil { 67 | return 68 | } 69 | s.closed = true 70 | err := s.server.Shutdown(context.Background()) 71 | if err != nil { 72 | logger.Error("error while api cannot be shutdown", "error", err) 73 | panic(err) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /pq/connection.go: -------------------------------------------------------------------------------- 1 | package pq 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/Trendyol/go-pq-cdc/internal/retry" 7 | "github.com/go-playground/errors" 8 | "github.com/jackc/pgx/v5/pgconn" 9 | "github.com/jackc/pgx/v5/pgproto3" 10 | ) 11 | 12 | type Connection interface { 13 | Connect(ctx context.Context) error 14 | IsClosed() bool 15 | Close(ctx context.Context) error 16 | ReceiveMessage(ctx context.Context) (pgproto3.BackendMessage, error) 17 | Frontend() *pgproto3.Frontend 18 | Exec(ctx context.Context, sql string) *pgconn.MultiResultReader 19 | } 20 | 21 | type connection struct { 22 | *pgconn.PgConn 23 | dsn string 24 | } 25 | 26 | func NewConnection(ctx context.Context, dsn string) (Connection, error) { 27 | conn := NewConnectionTemplate(dsn) 28 | if err := conn.Connect(ctx); err != nil { 29 | return nil, err 30 | } 31 | return conn, nil 32 | } 33 | 34 | func NewConnectionTemplate(dsn string) Connection { 35 | return &connection{ 36 | dsn: dsn, 37 | } 38 | } 39 | 40 | func (c *connection) Connect(ctx context.Context) error { 41 | if c.PgConn != nil && !c.IsClosed() { 42 | return nil 43 | } 44 | 45 | conn, err := connect(ctx, c.dsn) 46 | if err != nil { 47 | return errors.Wrap(err, "postgres connection") 48 | } 49 | c.PgConn = conn 50 | return nil 51 | } 52 | 53 | func (c *connection) IsClosed() bool { 54 | return c.PgConn == nil || c.PgConn.IsClosed() 55 | } 56 | 57 | func connect(ctx context.Context, dsn string) (*pgconn.PgConn, error) { 58 | retryConfig := retry.OnErrorConfig[*pgconn.PgConn](5, func(err error) bool { return err == nil }) 59 | conn, err := retryConfig.Do(func() (*pgconn.PgConn, error) { 60 | conn, err := pgconn.Connect(ctx, dsn) 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | if err = conn.Ping(ctx); err != nil { 66 | return nil, err 67 | } 68 | 69 | return conn, nil 70 | }) 71 | 72 | if err != nil { 73 | return nil, errors.Wrap(err, "postgres connection") 74 | } 75 | 76 | return conn, nil 77 | } 78 | -------------------------------------------------------------------------------- /pq/message/format/delete.go: -------------------------------------------------------------------------------- 1 | package format 2 | 3 | import ( 4 | "encoding/binary" 5 | "time" 6 | 7 | "github.com/Trendyol/go-pq-cdc/pq/message/tuple" 8 | "github.com/go-playground/errors" 9 | ) 10 | 11 | type Delete struct { 12 | MessageTime time.Time 13 | OldTupleData *tuple.Data 14 | OldDecoded map[string]any 15 | TableNamespace string 16 | TableName string 17 | OID uint32 18 | XID uint32 19 | OldTupleType uint8 20 | } 21 | 22 | func NewDelete(data []byte, streamedTransaction bool, relation map[uint32]*Relation, serverTime time.Time) (*Delete, error) { 23 | msg := &Delete{ 24 | MessageTime: serverTime, 25 | } 26 | if err := msg.decode(data, streamedTransaction); err != nil { 27 | return nil, err 28 | } 29 | 30 | rel, ok := relation[msg.OID] 31 | if !ok { 32 | return nil, errors.New("relation not found") 33 | } 34 | 35 | msg.TableNamespace = rel.Namespace 36 | msg.TableName = rel.Name 37 | 38 | var err error 39 | 40 | msg.OldDecoded, err = msg.OldTupleData.DecodeWithColumn(rel.Columns) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | return msg, nil 46 | } 47 | 48 | func (m *Delete) decode(data []byte, streamedTransaction bool) error { 49 | skipByte := 1 50 | 51 | if streamedTransaction { 52 | if len(data) < 11 { 53 | return errors.Newf("streamed transaction delete message length must be at least 11 byte, but got %d", len(data)) 54 | } 55 | 56 | m.XID = binary.BigEndian.Uint32(data[skipByte:]) 57 | skipByte += 4 58 | } 59 | 60 | if len(data) < 7 { 61 | return errors.Newf("delete message length must be at least 7 byte, but got %d", len(data)) 62 | } 63 | 64 | m.OID = binary.BigEndian.Uint32(data[skipByte:]) 65 | skipByte += 4 66 | 67 | m.OldTupleType = data[skipByte] 68 | 69 | var err error 70 | 71 | m.OldTupleData, err = tuple.NewData(data, m.OldTupleType, skipByte) 72 | if err != nil { 73 | return errors.Wrap(err, "delete message old tuple data") 74 | } 75 | 76 | return nil 77 | } 78 | -------------------------------------------------------------------------------- /pq/message/format/insert.go: -------------------------------------------------------------------------------- 1 | package format 2 | 3 | import ( 4 | "encoding/binary" 5 | "time" 6 | 7 | "github.com/Trendyol/go-pq-cdc/pq/message/tuple" 8 | "github.com/go-playground/errors" 9 | ) 10 | 11 | const ( 12 | InsertTupleDataType = 'N' 13 | ) 14 | 15 | type Insert struct { 16 | MessageTime time.Time 17 | TupleData *tuple.Data 18 | Decoded map[string]any 19 | TableNamespace string 20 | TableName string 21 | OID uint32 22 | XID uint32 23 | } 24 | 25 | func NewInsert(data []byte, streamedTransaction bool, relation map[uint32]*Relation, serverTime time.Time) (*Insert, error) { 26 | msg := &Insert{ 27 | MessageTime: serverTime, 28 | } 29 | if err := msg.decode(data, streamedTransaction); err != nil { 30 | return nil, err 31 | } 32 | 33 | rel, ok := relation[msg.OID] 34 | if !ok { 35 | return nil, errors.New("relation not found") 36 | } 37 | 38 | msg.TableNamespace = rel.Namespace 39 | msg.TableName = rel.Name 40 | 41 | msg.Decoded = make(map[string]any) 42 | 43 | var err error 44 | msg.Decoded, err = msg.TupleData.DecodeWithColumn(rel.Columns) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | return msg, nil 50 | } 51 | 52 | func (m *Insert) decode(data []byte, streamedTransaction bool) error { 53 | skipByte := 1 54 | 55 | if streamedTransaction { 56 | if len(data) < 13 { 57 | return errors.Newf("streamed transaction insert message length must be at least 13 byte, but got %d", len(data)) 58 | } 59 | 60 | m.XID = binary.BigEndian.Uint32(data[skipByte:]) 61 | skipByte += 4 62 | } 63 | 64 | if len(data) < 9 { 65 | return errors.Newf("insert message length must be at least 9 byte, but got %d", len(data)) 66 | } 67 | 68 | m.OID = binary.BigEndian.Uint32(data[skipByte:]) 69 | skipByte += 4 70 | 71 | var err error 72 | 73 | m.TupleData, err = tuple.NewData(data, InsertTupleDataType, skipByte) 74 | if err != nil { 75 | return errors.Wrap(err, "insert message") 76 | } 77 | 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /example/snapshot-only-mode/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | cdc "github.com/Trendyol/go-pq-cdc" 6 | "github.com/Trendyol/go-pq-cdc/config" 7 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 8 | "github.com/Trendyol/go-pq-cdc/pq/publication" 9 | "github.com/Trendyol/go-pq-cdc/pq/replication" 10 | "log" 11 | "log/slog" 12 | "os" 13 | "time" 14 | ) 15 | 16 | func main() { 17 | ctx := context.Background() 18 | cfg := config.Config{ 19 | Host: "127.0.0.1", 20 | Port: 5433, 21 | Username: "cdc_user", 22 | Password: "cdc_pass", 23 | Database: "cdc_db", 24 | DebugMode: false, 25 | Snapshot: config.SnapshotConfig{ 26 | Enabled: true, 27 | Mode: config.SnapshotModeSnapshotOnly, 28 | Tables: publication.Tables{ 29 | publication.Table{ 30 | Name: "users", 31 | ReplicaIdentity: publication.ReplicaIdentityDefault, 32 | Schema: "public", 33 | }, 34 | }, 35 | ChunkSize: 100, 36 | ClaimTimeout: 30 * time.Second, 37 | HeartbeatInterval: 5 * time.Second, 38 | }, 39 | Metric: config.MetricConfig{ 40 | Port: 8081, 41 | }, 42 | Logger: config.LoggerConfig{ 43 | LogLevel: slog.LevelInfo, 44 | }, 45 | } 46 | 47 | connector, err := cdc.NewConnector(ctx, cfg, Handler) 48 | if err != nil { 49 | slog.Error("new connector", "error", err) 50 | os.Exit(1) 51 | } 52 | 53 | defer connector.Close() 54 | connector.Start(ctx) 55 | } 56 | 57 | func Handler(ctx *replication.ListenerContext) { 58 | msg := ctx.Message.(*format.Snapshot) 59 | 60 | switch msg.EventType { 61 | case format.SnapshotEventTypeBegin: 62 | log.Printf("📸 SNAPSHOT BEGIN | LSN: %s | Time: %s", 63 | msg.LSN.String(), 64 | msg.ServerTime.Format("15:04:05")) 65 | 66 | case format.SnapshotEventTypeData: 67 | slog.Info("snapshot message received", "data", msg.Data) 68 | 69 | case format.SnapshotEventTypeEnd: 70 | log.Printf("📸 SNAPSHOT END | LSN: %s | Time: %s", 71 | msg.LSN.String(), 72 | msg.ServerTime.Format("15:04:05")) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /pq/snapshot/transaction_snapshot.go: -------------------------------------------------------------------------------- 1 | package snapshot 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/Trendyol/go-pq-cdc/pq" 9 | 10 | "github.com/Trendyol/go-pq-cdc/logger" 11 | "github.com/go-playground/errors" 12 | ) 13 | 14 | // exportSnapshot exports the current transaction snapshot for use by other connections 15 | // Uses snapshotTxConn which keeps transaction open until all chunks are processed 16 | // Requires: REPLICATION privilege, wal_level=logical, max_replication_slots>0 17 | func (s *Snapshotter) exportSnapshot(ctx context.Context, exportSnapshotConn pq.Connection) (string, error) { 18 | var snapshotID string 19 | 20 | err := s.retryDBOperation(ctx, func() error { 21 | results, err := s.execQuery(ctx, exportSnapshotConn, "SELECT pg_export_snapshot()") 22 | if err != nil { 23 | if strings.Contains(err.Error(), "permission denied") { 24 | return errors.New("pg_export_snapshot requires REPLICATION privilege. Run: ALTER USER your_user WITH REPLICATION") 25 | } 26 | if strings.Contains(err.Error(), "wal_level") { 27 | return errors.New("pg_export_snapshot requires wal_level='logical'. Set in postgresql.conf and restart") 28 | } 29 | return errors.Wrap(err, "export snapshot") 30 | } 31 | 32 | if len(results) == 0 || len(results[0].Rows) == 0 || len(results[0].Rows[0]) == 0 { 33 | return errors.New("no snapshot ID returned") 34 | } 35 | 36 | snapshotID = string(results[0].Rows[0][0]) 37 | return nil 38 | }) 39 | 40 | return snapshotID, err 41 | } 42 | 43 | // setTransactionSnapshot sets the current transaction to use an exported snapshot 44 | func (s *Snapshotter) setTransactionSnapshot(ctx context.Context, conn pq.Connection, snapshotID string) error { 45 | return s.retryDBOperation(ctx, func() error { 46 | query := fmt.Sprintf("SET TRANSACTION SNAPSHOT '%s'", snapshotID) 47 | if err := s.execSQL(ctx, conn, query); err != nil { 48 | return errors.Wrap(err, "set transaction snapshot") 49 | } 50 | 51 | logger.Debug("[worker] transaction snapshot set", "snapshotID", snapshotID) 52 | return nil 53 | }) 54 | } 55 | -------------------------------------------------------------------------------- /.github/workflows/scorecard.yml: -------------------------------------------------------------------------------- 1 | 2 | name: Scorecard supply-chain security 3 | 4 | on: 5 | branch_protection_rule: 6 | schedule: 7 | - cron: '29 23 * * 3' 8 | push: 9 | branches: [ "main", "master"] 10 | pull_request: 11 | branches: ["main", "master"] 12 | 13 | permissions: read-all 14 | 15 | jobs: 16 | visibility-check: 17 | # Bu job, deponun public/private olduğunu belirler 18 | outputs: 19 | visibility: ${{ steps.drv.outputs.visibility }} 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Determine repository visibility 23 | id: drv 24 | run: | 25 | visibility=$(gh api /repos/$GITHUB_REPOSITORY --jq '.visibility') 26 | echo "visibility=$visibility" >> $GITHUB_OUTPUT 27 | env: 28 | GH_TOKEN: ${{ github.token }} 29 | 30 | analysis: 31 | if: ${{ needs.visibility-check.outputs.visibility == 'public' }} 32 | needs: visibility-check 33 | runs-on: ubuntu-latest 34 | permissions: 35 | security-events: write 36 | id-token: write 37 | steps: 38 | - name: "Checkout code" 39 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 40 | with: 41 | persist-credentials: false 42 | 43 | - name: "Run analysis" 44 | uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 45 | with: 46 | results_file: results.sarif 47 | results_format: sarif 48 | publish_results: true 49 | 50 | - name: "Upload artifact" 51 | uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 52 | with: 53 | name: SARIF file 54 | path: results.sarif 55 | retention-days: 5 56 | 57 | # Upload the results to GitHub's code scanning dashboard (optional). 58 | # Commenting out will disable upload of results to your repo's Code Scanning dashboard 59 | - name: "Upload to code-scanning" 60 | uses: github/codeql-action/upload-sarif@f6091c0113d1dcf9b98e269ee48e8a7e51b7bdd4 # v3.28.5 61 | with: 62 | sarif_file: results.sarif 63 | 64 | 65 | -------------------------------------------------------------------------------- /example/snapshot-with-scaling/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "log/slog" 7 | "os" 8 | "time" 9 | 10 | cdc "github.com/Trendyol/go-pq-cdc" 11 | "github.com/Trendyol/go-pq-cdc/config" 12 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 13 | "github.com/Trendyol/go-pq-cdc/pq/publication" 14 | "github.com/Trendyol/go-pq-cdc/pq/replication" 15 | ) 16 | 17 | // docker-compose up --scale go-pq-cdc=3 -d 18 | func main() { 19 | ctx := context.Background() 20 | cfg := config.Config{ 21 | Host: "postgres", 22 | Username: "cdc_user", 23 | Password: "cdc_pass", 24 | Database: "cdc_db", 25 | DebugMode: false, 26 | Snapshot: config.SnapshotConfig{ 27 | Enabled: true, 28 | Mode: config.SnapshotModeSnapshotOnly, 29 | Tables: publication.Tables{ 30 | publication.Table{ 31 | Name: "users", 32 | ReplicaIdentity: publication.ReplicaIdentityDefault, 33 | Schema: "public", 34 | }, 35 | }, 36 | ChunkSize: 100, 37 | ClaimTimeout: 30 * time.Second, 38 | HeartbeatInterval: 5 * time.Second, 39 | }, 40 | Metric: config.MetricConfig{ 41 | Port: 8081, 42 | }, 43 | Logger: config.LoggerConfig{ 44 | LogLevel: slog.LevelInfo, 45 | }, 46 | } 47 | 48 | connector, err := cdc.NewConnector(ctx, cfg, Handler) 49 | if err != nil { 50 | slog.Error("new connector", "error", err) 51 | os.Exit(1) 52 | } 53 | 54 | defer connector.Close() 55 | connector.Start(ctx) 56 | } 57 | 58 | func Handler(ctx *replication.ListenerContext) { 59 | time.Sleep(500 * time.Millisecond) // simulate for work 60 | msg := ctx.Message.(*format.Snapshot) 61 | 62 | switch msg.EventType { 63 | case format.SnapshotEventTypeBegin: 64 | log.Printf("📸 SNAPSHOT BEGIN | LSN: %s | Time: %s", 65 | msg.LSN.String(), 66 | msg.ServerTime.Format("15:04:05")) 67 | 68 | case format.SnapshotEventTypeData: 69 | slog.Info("snapshot message received", "data", msg.Data) 70 | 71 | case format.SnapshotEventTypeEnd: 72 | log.Printf("📸 SNAPSHOT END | LSN: %s | Time: %s", 73 | msg.LSN.String(), 74 | msg.ServerTime.Format("15:04:05")) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /pq/system.go: -------------------------------------------------------------------------------- 1 | package pq 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "sync" 8 | 9 | "github.com/go-playground/errors" 10 | "github.com/jackc/pgx/v5/pgconn" 11 | ) 12 | 13 | type IdentifySystemResult struct { 14 | mu *sync.RWMutex 15 | SystemID string 16 | Database string 17 | xLogPos LSN 18 | Timeline int32 19 | } 20 | 21 | func IdentifySystem(ctx context.Context, conn Connection) (IdentifySystemResult, error) { 22 | res, err := ParseIdentifySystem(conn.Exec(ctx, "IDENTIFY_SYSTEM")) 23 | if err != nil { 24 | return IdentifySystemResult{}, errors.Wrap(err, "identify system command execute") 25 | } 26 | return res, nil 27 | } 28 | 29 | func ParseIdentifySystem(mrr *pgconn.MultiResultReader) (IdentifySystemResult, error) { 30 | var isr IdentifySystemResult 31 | results, err := mrr.ReadAll() 32 | if err != nil { 33 | return isr, err 34 | } 35 | 36 | if len(results) != 1 { 37 | return isr, fmt.Errorf("expected 1 result set, got %d", len(results)) 38 | } 39 | 40 | result := results[0] 41 | if len(result.Rows) != 1 { 42 | return isr, fmt.Errorf("expected 1 result row, got %d", len(result.Rows)) 43 | } 44 | 45 | row := result.Rows[0] 46 | if len(row) != 4 { 47 | return isr, fmt.Errorf("expected 4 result columns, got %d", len(row)) 48 | } 49 | 50 | isr.SystemID = string(row[0]) 51 | timeline, err := strconv.ParseInt(string(row[1]), 10, 32) 52 | if err != nil { 53 | return isr, fmt.Errorf("failed to parse timeline: %w", err) 54 | } 55 | isr.Timeline = int32(timeline) 56 | 57 | isr.xLogPos, err = ParseLSN(string(row[2])) 58 | if err != nil { 59 | return isr, fmt.Errorf("failed to parse xlogpos as LSN: %w", err) 60 | } 61 | 62 | isr.Database = string(row[3]) 63 | 64 | isr.mu = &sync.RWMutex{} 65 | return isr, nil 66 | } 67 | 68 | func (isr *IdentifySystemResult) SetXLogPos(l LSN) { 69 | isr.xLogPos = l 70 | } 71 | 72 | func (isr *IdentifySystemResult) UpdateXLogPos(l LSN) { 73 | isr.mu.Lock() 74 | defer isr.mu.Unlock() 75 | 76 | if isr.xLogPos < l { 77 | isr.xLogPos = l 78 | } 79 | } 80 | 81 | func (isr *IdentifySystemResult) LoadXLogPos() LSN { 82 | isr.mu.RLock() 83 | defer isr.mu.RUnlock() 84 | return isr.xLogPos 85 | } 86 | -------------------------------------------------------------------------------- /pq/message/format/update_test.go: -------------------------------------------------------------------------------- 1 | package format 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/Trendyol/go-pq-cdc/pq/message/tuple" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestUpdate_New(t *testing.T) { 12 | data := []byte{85, 0, 0, 64, 6, 79, 0, 2, 116, 0, 0, 0, 2, 53, 51, 116, 0, 0, 0, 4, 98, 97, 114, 50, 78, 0, 2, 116, 0, 0, 0, 2, 53, 51, 116, 0, 0, 0, 4, 98, 97, 114, 53} 13 | 14 | rel := map[uint32]*Relation{ 15 | 16390: { 16 | OID: 16390, 17 | XID: 0, 18 | Namespace: "public", 19 | Name: "t", 20 | ReplicaID: 100, 21 | ColumnNumbers: 2, 22 | Columns: []tuple.RelationColumn{ 23 | { 24 | Flags: 1, 25 | Name: "id", 26 | DataType: 23, 27 | TypeModifier: 4294967295, 28 | }, 29 | { 30 | Flags: 0, 31 | Name: "name", 32 | DataType: 25, 33 | TypeModifier: 4294967295, 34 | }, 35 | }, 36 | }, 37 | } 38 | 39 | now := time.Now() 40 | msg, err := NewUpdate(data, false, rel, now) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | expected := &Update{ 46 | OID: 16390, 47 | XID: 0, 48 | NewTupleData: &tuple.Data{ 49 | ColumnNumber: 2, 50 | Columns: tuple.DataColumns{ 51 | { 52 | DataType: 116, 53 | Length: 2, 54 | Data: []byte("53"), 55 | }, 56 | { 57 | DataType: 116, 58 | Length: 4, 59 | Data: []byte("bar5"), 60 | }, 61 | }, 62 | SkipByte: 43, 63 | }, 64 | NewDecoded: map[string]any{ 65 | "id": int32(53), 66 | "name": "bar5", 67 | }, 68 | OldTupleType: 79, 69 | OldTupleData: &tuple.Data{ 70 | ColumnNumber: 2, 71 | Columns: tuple.DataColumns{ 72 | { 73 | DataType: 116, 74 | Length: 2, 75 | Data: []byte("53"), 76 | }, 77 | { 78 | DataType: 116, 79 | Length: 4, 80 | Data: []byte("bar2"), 81 | }, 82 | }, 83 | SkipByte: 24, 84 | }, 85 | OldDecoded: map[string]any{ 86 | "id": int32(53), 87 | "name": "bar2", 88 | }, 89 | TableNamespace: "public", 90 | TableName: "t", 91 | MessageTime: now, 92 | } 93 | 94 | assert.Equal(t, expected, msg) 95 | } 96 | -------------------------------------------------------------------------------- /benchmark/benchmark_cdc/README.md: -------------------------------------------------------------------------------- 1 | ## 10 M Insert Test 2 | 3 | ### Hardware 4 | ```txt 5 | PC: Macbook Apple M1 Pro (2021) 6 | Memory: 32 GB 7 | 8 | go-pq-cdc: 9 | resources: 10 | limits: 11 | cpus: 1 12 | memory: 512M 13 | reservations: 14 | cpus: '0.25' 15 | memory: 128M 16 | 17 | Debezium: 18 | resources: 19 | limits: 20 | cpus: 2 21 | memory: 1024M 22 | reservations: 23 | cpus: '0.25' 24 | memory: 128M 25 | ``` 26 | 27 | ### Result 28 | | | go-pq-cdc | Debezium | 29 | |----------------------|------------|--------------| 30 | | Row Count | 10 m | 10 m | 31 | | Elapsed Time | 2.5 min | 21 min | 32 | | Cpu Usage Max | 44% | 181% | 33 | | Memory Usage Max | 130 MB | 1.07 GB | 34 | | Received Traffic Max | 4.36 MiB/s | 7.97 MiB/s | 35 | | Sent Traffic Max | 5.96 MiB/s | 6.27 MiB/s | 36 | 37 | ![10m_result](./10m_test.png) 38 | 39 | 40 | ## Requirements 41 | - [Docker](https://docs.docker.com/compose/install/) 42 | - [psql](https://www.postgresql.org/download/) 43 | 44 | ## Instructions 45 | 46 | - Start the containers 47 | ```sh 48 | docker compose up -d 49 | ``` 50 | - Connect to Postgres database: 51 | ```sh 52 | psql postgres://cdc_user:cdc_pass@127.0.0.1:5432/cdc_db 53 | ``` 54 | - Insert data to users table: 55 | ```sql 56 | INSERT INTO users (name) 57 | SELECT 58 | 'Oyleli' || i 59 | FROM generate_series(1, 1000000) AS i; 60 | ``` 61 | - Go to grafana dashboard: http://localhost:3000/d/edl1ybvsmc64gb/benchmark?orgId=1 62 | > **Grafana Credentials** 63 | Username: `go-pq-cdc-user` Password: `go-pq-cdc-pass` 64 | - Trace the process 65 | ![benchmark_dashboard](./dashboard.png) 66 | 67 | ## Ports 68 | 69 | - RedPanda Console: `8085` http://localhost:8085 70 | - RedPanda: `19092` http://localhost:19092 71 | - Grafana: `3000` http://localhost:3000 72 | - Prometheus: `9090` http://localhost:9090 73 | - cAdvisor: `8080` http://localhost:8080 74 | - PostgreSQL:`5432` http://localhost:5432 75 | - PostgreSQL Metric Exporter: `9187` http://localhost:9187 76 | - Debezium: `9093` http://localhost:9093 77 | - go-pq-cdc Metric: `2112` http://localhost:2112 78 | 79 | -------------------------------------------------------------------------------- /pq/publication/config.go: -------------------------------------------------------------------------------- 1 | package publication 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/lib/pq" 9 | ) 10 | 11 | type Config struct { 12 | Name string `json:"name" yaml:"name"` 13 | Operations Operations `json:"operations" yaml:"operations"` 14 | Tables Tables `json:"tables" yaml:"tables"` 15 | CreateIfNotExists bool `json:"createIfNotExists" yaml:"createIfNotExists"` 16 | } 17 | 18 | func (c Config) Validate() error { 19 | var err error 20 | if strings.TrimSpace(c.Name) == "" { 21 | err = errors.Join(err, errors.New("publication name cannot be empty")) 22 | } 23 | 24 | if !c.CreateIfNotExists { 25 | return err 26 | } 27 | 28 | if validateErr := c.Tables.Validate(); validateErr != nil { 29 | err = errors.Join(err, validateErr) 30 | } 31 | 32 | if validateErr := c.Operations.Validate(); validateErr != nil { 33 | err = errors.Join(err, validateErr) 34 | } 35 | 36 | return err 37 | } 38 | 39 | func (c Config) createQuery() string { 40 | sqlStatement := fmt.Sprintf(`CREATE PUBLICATION %s`, pq.QuoteIdentifier(c.Name)) 41 | 42 | quotedTables := make([]string, len(c.Tables)) 43 | for i, table := range c.Tables { 44 | quotedTables[i] = fmt.Sprintf("%s.%s", pq.QuoteIdentifier(table.Schema), pq.QuoteIdentifier(table.Name)) 45 | } 46 | sqlStatement += " FOR TABLE " + strings.Join(quotedTables, ", ") 47 | 48 | sqlStatement += fmt.Sprintf(" WITH (publish = '%s')", c.Operations.String()) 49 | 50 | return sqlStatement 51 | } 52 | 53 | func (c Config) infoQuery() string { 54 | q := fmt.Sprintf(`WITH publication_details AS ( 55 | SELECT 56 | p.oid AS pubid, 57 | p.pubname, 58 | p.puballtables, 59 | p.pubinsert, 60 | p.pubupdate, 61 | p.pubdelete, 62 | p.pubtruncate 63 | FROM pg_publication p 64 | WHERE p.pubname = '%s' 65 | ), 66 | expanded_tables AS ( 67 | SELECT 68 | pubname, 69 | array_agg(schemaname || '.' || tablename) AS tables 70 | FROM pg_publication_tables 71 | WHERE pubname = '%s' 72 | GROUP BY pubname 73 | ) 74 | SELECT 75 | pd.pubname, 76 | pd.puballtables, 77 | pd.pubinsert, 78 | pd.pubupdate, 79 | pd.pubdelete, 80 | pd.pubtruncate, 81 | COALESCE(et.tables, ARRAY[]::text[]) AS pubtables 82 | FROM publication_details pd 83 | LEFT JOIN expanded_tables et ON pd.pubname = et.pubname;`, c.Name, c.Name) 84 | return q 85 | } 86 | -------------------------------------------------------------------------------- /pq/replication/replication.go: -------------------------------------------------------------------------------- 1 | package replication 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | 9 | "github.com/Trendyol/go-pq-cdc/pq" 10 | "github.com/go-playground/errors" 11 | "github.com/jackc/pgx/v5/pgconn" 12 | "github.com/jackc/pgx/v5/pgproto3" 13 | ) 14 | 15 | type Replication struct { 16 | conn pq.Connection 17 | } 18 | 19 | func New(conn pq.Connection) *Replication { 20 | return &Replication{conn: conn} 21 | } 22 | 23 | func (r *Replication) Start(publicationName, slotName string, startLSN pq.LSN) error { 24 | pluginArguments := append([]string{ 25 | "proto_version '2'", 26 | "messages 'true'", 27 | "streaming 'true'", 28 | }, "publication_names '"+publicationName+"'") 29 | 30 | sql := fmt.Sprintf("START_REPLICATION SLOT %s LOGICAL %s (%s)", slotName, startLSN, strings.Join(pluginArguments, ",")) 31 | r.conn.Frontend().SendQuery(&pgproto3.Query{String: sql}) 32 | err := r.conn.Frontend().Flush() 33 | if err != nil { 34 | return errors.Wrap(err, "start replication") 35 | } 36 | return nil 37 | } 38 | 39 | func (r *Replication) Test(ctx context.Context) error { 40 | var ( 41 | nextTli int64 42 | nextTliStartPos pq.LSN 43 | ) 44 | for { 45 | msg, err := r.conn.ReceiveMessage(ctx) 46 | if err != nil { 47 | return errors.Newf("failed to receive message: %w", err) 48 | } 49 | 50 | switch msg := msg.(type) { 51 | case *pgproto3.NoticeResponse: 52 | case *pgproto3.ErrorResponse: 53 | return pgconn.ErrorResponseToPgError(msg) 54 | case *pgproto3.CopyBothResponse: 55 | return nil 56 | case *pgproto3.RowDescription: 57 | return errors.Newf("received row RowDescription message in logical replication") 58 | case *pgproto3.DataRow: 59 | if cnt := len(msg.Values); cnt != 2 { 60 | return errors.Newf("expected next_tli and next_tli_startpos, got %d fields", cnt) 61 | } 62 | tmpNextTli, tmpNextTliStartPos := string(msg.Values[0]), string(msg.Values[1]) 63 | nextTli, err = strconv.ParseInt(tmpNextTli, 10, 64) 64 | if err != nil { 65 | return err 66 | } 67 | nextTliStartPos, err = pq.ParseLSN(tmpNextTliStartPos) 68 | if err != nil { 69 | return err 70 | } 71 | case *pgproto3.CommandComplete: 72 | case *pgproto3.ReadyForQuery: 73 | if nextTli > 0 && nextTliStartPos > 0 { 74 | return errors.New("start replication with a switch point") 75 | } 76 | default: 77 | return errors.Newf("unexpected response type: %T", msg) 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /example/simple/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | cdc "github.com/Trendyol/go-pq-cdc" 6 | "github.com/Trendyol/go-pq-cdc/config" 7 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 8 | "github.com/Trendyol/go-pq-cdc/pq/publication" 9 | "github.com/Trendyol/go-pq-cdc/pq/replication" 10 | "github.com/Trendyol/go-pq-cdc/pq/slot" 11 | "log/slog" 12 | "os" 13 | ) 14 | 15 | /* 16 | psql "postgres://cdc_user:cdc_pass@127.0.0.1/cdc_db?replication=database" 17 | 18 | CREATE TABLE users ( 19 | id serial PRIMARY KEY, 20 | name text NOT NULL, 21 | created_on timestamptz 22 | ); 23 | 24 | INSERT INTO users (name) 25 | SELECT 26 | 'Oyleli' || i 27 | FROM generate_series(1, 100) AS i; 28 | */ 29 | 30 | func main() { 31 | ctx := context.Background() 32 | cfg := config.Config{ 33 | Host: "127.0.0.1", 34 | Port: 5433, 35 | Username: "cdc_user", 36 | Password: "cdc_pass", 37 | Database: "cdc_db", 38 | DebugMode: false, 39 | Publication: publication.Config{ 40 | CreateIfNotExists: true, 41 | Name: "cdc_publication", 42 | Operations: publication.Operations{ 43 | publication.OperationInsert, 44 | publication.OperationDelete, 45 | publication.OperationTruncate, 46 | publication.OperationUpdate, 47 | }, 48 | Tables: publication.Tables{ 49 | publication.Table{ 50 | Name: "users", 51 | ReplicaIdentity: publication.ReplicaIdentityDefault, 52 | Schema: "public", 53 | }, 54 | }, 55 | }, 56 | Slot: slot.Config{ 57 | CreateIfNotExists: true, 58 | Name: "cdc_slot", 59 | SlotActivityCheckerInterval: 3000, 60 | }, 61 | Metric: config.MetricConfig{ 62 | Port: 8081, 63 | }, 64 | Logger: config.LoggerConfig{ 65 | LogLevel: slog.LevelInfo, 66 | }, 67 | } 68 | 69 | connector, err := cdc.NewConnector(ctx, cfg, Handler) 70 | if err != nil { 71 | slog.Error("new connector", "error", err) 72 | os.Exit(1) 73 | } 74 | 75 | defer connector.Close() 76 | connector.Start(ctx) 77 | } 78 | 79 | func Handler(ctx *replication.ListenerContext) { 80 | switch msg := ctx.Message.(type) { 81 | case *format.Insert: 82 | slog.Info("insert message received", "new", msg.Decoded) 83 | case *format.Delete: 84 | slog.Info("delete message received", "old", msg.OldDecoded) 85 | case *format.Update: 86 | slog.Info("update message received", "new", msg.NewDecoded, "old", msg.OldDecoded) 87 | } 88 | 89 | if err := ctx.Ack(); err != nil { 90 | slog.Error("ack", "error", err) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /pq/snapshot/connection_pool.go: -------------------------------------------------------------------------------- 1 | package snapshot 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/Trendyol/go-pq-cdc/logger" 8 | "github.com/Trendyol/go-pq-cdc/pq" 9 | "github.com/go-playground/errors" 10 | ) 11 | 12 | // ConnectionPool manages a pool of database connections for efficient reuse 13 | // Eliminates the overhead of creating/destroying connections for each chunk 14 | type ConnectionPool struct { 15 | pool chan pq.Connection 16 | dsn string 17 | conns []pq.Connection 18 | mu sync.Mutex 19 | } 20 | 21 | // NewConnectionPool creates a new connection pool with the specified size 22 | func NewConnectionPool(ctx context.Context, dsn string, size int) (*ConnectionPool, error) { 23 | if size <= 0 { 24 | size = 5 // Default pool size 25 | } 26 | 27 | p := &ConnectionPool{ 28 | dsn: dsn, 29 | pool: make(chan pq.Connection, size), 30 | conns: make([]pq.Connection, 0, size), 31 | } 32 | 33 | // Pre-create connections 34 | logger.Info("[snapshot-connection-pool] creating connection pool", "size", size) 35 | for i := 0; i < size; i++ { 36 | conn, err := pq.NewConnection(ctx, dsn) 37 | if err != nil { 38 | // Cleanup already created connections 39 | p.Close(ctx) 40 | return nil, errors.Wrap(err, "create pool connection") 41 | } 42 | p.conns = append(p.conns, conn) 43 | p.pool <- conn 44 | } 45 | 46 | logger.Info("[snapshot-connection-pool] connection pool ready", "size", size) 47 | return p, nil 48 | } 49 | 50 | // Get retrieves a connection from the pool (blocks if pool is empty) 51 | func (p *ConnectionPool) Get(ctx context.Context) (pq.Connection, error) { 52 | select { 53 | case conn := <-p.pool: 54 | return conn, nil 55 | case <-ctx.Done(): 56 | return nil, ctx.Err() 57 | } 58 | } 59 | 60 | // Put returns a connection to the pool 61 | func (p *ConnectionPool) Put(conn pq.Connection) { 62 | select { 63 | case p.pool <- conn: 64 | // Connection returned to pool 65 | default: 66 | // Pool is full (shouldn't happen with proper usage) 67 | logger.Warn("[snapshot-connection-pool] pool is full, connection not returned") 68 | } 69 | } 70 | 71 | // Close closes all connections in the pool 72 | func (p *ConnectionPool) Close(ctx context.Context) { 73 | p.mu.Lock() 74 | defer p.mu.Unlock() 75 | 76 | logger.Info("[snapshot-connection-pool] closing all connections", "count", len(p.conns)) 77 | 78 | for _, conn := range p.conns { 79 | if err := conn.Close(ctx); err != nil { 80 | logger.Warn("[snapshot-connection-pool] error closing connection", "error", err) 81 | } 82 | } 83 | 84 | close(p.pool) 85 | p.conns = nil 86 | logger.Info("[snapshot-connection-pool] connection pool closed") 87 | } 88 | -------------------------------------------------------------------------------- /pq/message/format/relation.go: -------------------------------------------------------------------------------- 1 | package format 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | 7 | "github.com/Trendyol/go-pq-cdc/pq/message/tuple" 8 | "github.com/go-playground/errors" 9 | ) 10 | 11 | type Relation struct { 12 | Namespace string 13 | Name string 14 | Columns []tuple.RelationColumn 15 | OID uint32 16 | XID uint32 17 | ColumnNumbers uint16 18 | ReplicaID uint8 19 | } 20 | 21 | func NewRelation(data []byte, streamedTransaction bool) (*Relation, error) { 22 | msg := &Relation{} 23 | if err := msg.decode(data, streamedTransaction); err != nil { 24 | return nil, err 25 | } 26 | 27 | return msg, nil 28 | } 29 | 30 | func (m *Relation) decode(data []byte, streamedTransaction bool) error { 31 | skipByte := 1 32 | 33 | if streamedTransaction { 34 | if len(data) < 12 { 35 | return errors.Newf("streamed transaction relation message length must be at least 12 byte, but got %d", len(data)) 36 | } 37 | 38 | m.XID = binary.BigEndian.Uint32(data[skipByte:]) 39 | skipByte += 4 40 | } 41 | 42 | if len(data) < 8 { 43 | return errors.Newf("relation message length must be at least 8 byte, but got %d", len(data)) 44 | } 45 | 46 | m.OID = binary.BigEndian.Uint32(data[skipByte:]) 47 | skipByte += 4 48 | 49 | var usedByteCount int 50 | m.Namespace, usedByteCount = decodeString(data[skipByte:]) 51 | if usedByteCount < 0 { 52 | return errors.New("relation message namespace decode error") 53 | } 54 | skipByte += usedByteCount 55 | 56 | m.Name, usedByteCount = decodeString(data[skipByte:]) 57 | if usedByteCount < 0 { 58 | return errors.New("relation message namespace decode error") 59 | } 60 | skipByte += usedByteCount 61 | 62 | m.ReplicaID = data[skipByte] 63 | skipByte++ 64 | 65 | m.ColumnNumbers = binary.BigEndian.Uint16(data[skipByte:]) 66 | skipByte += 2 67 | 68 | m.Columns = make([]tuple.RelationColumn, m.ColumnNumbers) 69 | for i := range m.Columns { 70 | col := tuple.RelationColumn{} 71 | col.Flags = data[skipByte] 72 | skipByte++ 73 | 74 | col.Name, usedByteCount = decodeString(data[skipByte:]) 75 | if usedByteCount < 0 { 76 | return errors.Newf("relation message columns[%d].name decode error", i) 77 | } 78 | skipByte += usedByteCount 79 | 80 | col.DataType = binary.BigEndian.Uint32(data[skipByte:]) 81 | skipByte += 4 82 | 83 | col.TypeModifier = binary.BigEndian.Uint32(data[skipByte:]) 84 | skipByte += 4 85 | 86 | m.Columns[i] = col 87 | } 88 | 89 | return nil 90 | } 91 | 92 | func decodeString(data []byte) (string, int) { 93 | end := bytes.IndexByte(data, byte(0)) 94 | if end == -1 { 95 | return "", -1 96 | } 97 | 98 | return string(data[:end]), end + 1 99 | } 100 | -------------------------------------------------------------------------------- /pq/message/tuple/data.go: -------------------------------------------------------------------------------- 1 | package tuple 2 | 3 | import ( 4 | "encoding/binary" 5 | 6 | "github.com/go-playground/errors" 7 | "github.com/jackc/pgx/v5/pgtype" 8 | ) 9 | 10 | const ( 11 | DataTypeNull = uint8('n') 12 | DataTypeToast = uint8('u') 13 | DataTypeText = uint8('t') 14 | DataTypeBinary = uint8('b') 15 | ) 16 | 17 | var typeMap = pgtype.NewMap() 18 | 19 | type Data struct { 20 | Columns DataColumns 21 | SkipByte int 22 | ColumnNumber uint16 23 | } 24 | 25 | type DataColumns []*DataColumn 26 | 27 | type DataColumn struct { 28 | Data []byte 29 | Length uint32 30 | DataType uint8 31 | } 32 | 33 | type RelationColumn struct { 34 | Name string 35 | DataType uint32 36 | TypeModifier uint32 37 | Flags uint8 38 | } 39 | 40 | func NewData(data []byte, tupleDataType uint8, skipByteLength int) (*Data, error) { 41 | if data[skipByteLength] != tupleDataType { 42 | return nil, errors.New("invalid tuple data type: " + string(data[skipByteLength])) 43 | } 44 | skipByteLength++ 45 | 46 | d := &Data{} 47 | d.Decode(data, skipByteLength) 48 | 49 | return d, nil 50 | } 51 | 52 | func (d *Data) Decode(data []byte, skipByteLength int) { 53 | d.ColumnNumber = binary.BigEndian.Uint16(data[skipByteLength:]) 54 | skipByteLength += 2 55 | 56 | for range d.ColumnNumber { 57 | col := new(DataColumn) 58 | col.DataType = data[skipByteLength] 59 | skipByteLength++ 60 | 61 | switch col.DataType { 62 | case DataTypeNull, DataTypeToast: 63 | case DataTypeText, DataTypeBinary: 64 | col.Length = binary.BigEndian.Uint32(data[skipByteLength:]) 65 | skipByteLength += 4 66 | 67 | col.Data = make([]byte, int(col.Length)) 68 | copy(col.Data, data[skipByteLength:]) 69 | 70 | skipByteLength += int(col.Length) 71 | } 72 | 73 | d.Columns = append(d.Columns, col) 74 | } 75 | d.SkipByte = skipByteLength 76 | } 77 | 78 | func (d *Data) DecodeWithColumn(columns []RelationColumn) (map[string]any, error) { 79 | decoded := make(map[string]any, d.ColumnNumber) 80 | for idx, col := range d.Columns { 81 | colName := columns[idx].Name 82 | switch col.DataType { 83 | case DataTypeNull: 84 | decoded[colName] = nil 85 | case DataTypeText: 86 | val, err := decodeTextColumnData(col.Data, columns[idx].DataType) 87 | if err != nil { 88 | return nil, errors.Wrap(err, "decode column") 89 | } 90 | decoded[colName] = val 91 | } 92 | } 93 | 94 | return decoded, nil 95 | } 96 | 97 | func decodeTextColumnData(data []byte, dataType uint32) (interface{}, error) { 98 | if dt, ok := typeMap.TypeForOID(dataType); ok { 99 | return dt.Codec.DecodeValue(typeMap, dataType, pgtype.TextFormatCode, data) 100 | } 101 | return string(data), nil 102 | } 103 | -------------------------------------------------------------------------------- /benchmark/benchmark_initial/go-pq-cdc-kafka/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log/slog" 6 | "os" 7 | "strconv" 8 | "time" 9 | 10 | jsoniter "github.com/json-iterator/go" 11 | 12 | "github.com/Trendyol/go-pq-cdc/pq/publication" 13 | 14 | cdc "github.com/Trendyol/go-pq-cdc-kafka" 15 | "github.com/Trendyol/go-pq-cdc-kafka/config" 16 | cdcconfig "github.com/Trendyol/go-pq-cdc/config" 17 | "github.com/segmentio/kafka-go" 18 | gokafka "github.com/segmentio/kafka-go" 19 | ) 20 | 21 | // Use jsoniter for 2-3x faster JSON encoding 22 | var json = jsoniter.ConfigCompatibleWithStandardLibrary 23 | 24 | /* 25 | psql "postgres://cdc_user:cdc_pass@127.0.0.1/cdc_db?replication=database" 26 | 27 | INSERT INTO users (name) 28 | SELECT 29 | 'Oyleli' || i 30 | FROM generate_series(1, 10000000) AS i; 31 | */ 32 | 33 | type Message struct { 34 | Message kafka.Message 35 | Ack func() error 36 | } 37 | 38 | func main() { 39 | ctx := context.Background() 40 | 41 | cfg := config.Connector{ 42 | CDC: cdcconfig.Config{ 43 | Host: "postgres", 44 | Username: "cdc_user", 45 | Password: "cdc_pass", 46 | Database: "cdc_db", 47 | DebugMode: true, 48 | Snapshot: cdcconfig.SnapshotConfig{ 49 | Mode: cdcconfig.SnapshotModeSnapshotOnly, 50 | Tables: []publication.Table{ 51 | { 52 | Name: "users", 53 | Schema: "public", 54 | }, 55 | }, 56 | ChunkSize: 8000, 57 | Enabled: true, 58 | ClaimTimeout: 60 * time.Second, 59 | HeartbeatInterval: 10 * time.Second, 60 | }, 61 | Metric: cdcconfig.MetricConfig{ 62 | Port: 2112, 63 | }, 64 | Logger: cdcconfig.LoggerConfig{ 65 | LogLevel: slog.LevelInfo, 66 | }, 67 | }, 68 | Kafka: config.Kafka{ 69 | TableTopicMapping: map[string]string{ 70 | "public.users": "cdc.test.produce", 71 | }, 72 | Brokers: []string{"redpanda:9092"}, 73 | AllowAutoTopicCreation: true, 74 | ProducerBatchTickerDuration: time.Millisecond * 100, 75 | ProducerBatchSize: 10000, 76 | }, 77 | } 78 | 79 | connector, err := cdc.NewConnector(ctx, cfg, Handler) 80 | if err != nil { 81 | slog.Error("new connector", "error", err) 82 | os.Exit(1) 83 | } 84 | 85 | defer connector.Close() 86 | connector.Start(ctx) 87 | } 88 | 89 | func Handler(msg *cdc.Message) []gokafka.Message { 90 | // slog.Info("change captured", "message", msg) 91 | 92 | if msg.Type.IsSnapshot() { 93 | msg.NewData["operation"] = msg.Type 94 | newData, _ := json.Marshal(msg.NewData) 95 | 96 | return []gokafka.Message{ 97 | { 98 | Headers: nil, 99 | Key: []byte(strconv.Itoa(int(msg.NewData["id"].(int32)))), 100 | Value: newData, 101 | }, 102 | } 103 | } 104 | 105 | return []gokafka.Message{} 106 | } 107 | -------------------------------------------------------------------------------- /example/snapshot-initial-mode/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | cdc "github.com/Trendyol/go-pq-cdc" 6 | "github.com/Trendyol/go-pq-cdc/config" 7 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 8 | "github.com/Trendyol/go-pq-cdc/pq/publication" 9 | "github.com/Trendyol/go-pq-cdc/pq/replication" 10 | "github.com/Trendyol/go-pq-cdc/pq/slot" 11 | "log" 12 | "log/slog" 13 | "os" 14 | "time" 15 | ) 16 | 17 | func main() { 18 | ctx := context.Background() 19 | cfg := config.Config{ 20 | Host: "127.0.0.1", 21 | Port: 5433, 22 | Username: "cdc_user", 23 | Password: "cdc_pass", 24 | Database: "cdc_db", 25 | DebugMode: false, 26 | Publication: publication.Config{ 27 | CreateIfNotExists: true, 28 | Name: "cdc_publication", 29 | Operations: publication.Operations{ 30 | publication.OperationInsert, 31 | publication.OperationDelete, 32 | publication.OperationTruncate, 33 | publication.OperationUpdate, 34 | }, 35 | Tables: publication.Tables{ 36 | publication.Table{ 37 | Name: "users", 38 | ReplicaIdentity: publication.ReplicaIdentityDefault, 39 | Schema: "public", 40 | }, 41 | }, 42 | }, 43 | Slot: slot.Config{ 44 | CreateIfNotExists: true, 45 | Name: "cdc_slot", 46 | SlotActivityCheckerInterval: 3000, 47 | }, 48 | Snapshot: config.SnapshotConfig{ 49 | Enabled: true, 50 | Mode: config.SnapshotModeInitial, 51 | ChunkSize: 100, 52 | ClaimTimeout: 30 * time.Second, 53 | HeartbeatInterval: 5 * time.Second, 54 | }, 55 | Metric: config.MetricConfig{ 56 | Port: 8081, 57 | }, 58 | Logger: config.LoggerConfig{ 59 | LogLevel: slog.LevelInfo, 60 | }, 61 | } 62 | 63 | connector, err := cdc.NewConnector(ctx, cfg, Handler) 64 | if err != nil { 65 | slog.Error("new connector", "error", err) 66 | os.Exit(1) 67 | } 68 | 69 | defer connector.Close() 70 | connector.Start(ctx) 71 | } 72 | 73 | func Handler(ctx *replication.ListenerContext) { 74 | switch msg := ctx.Message.(type) { 75 | case *format.Insert: 76 | slog.Info("insert message received", "new", msg.Decoded) 77 | case *format.Delete: 78 | slog.Info("delete message received", "old", msg.OldDecoded) 79 | case *format.Update: 80 | slog.Info("update message received", "new", msg.NewDecoded, "old", msg.OldDecoded) 81 | case *format.Snapshot: 82 | handleSnapshot(msg) 83 | } 84 | 85 | if err := ctx.Ack(); err != nil { 86 | slog.Error("ack", "error", err) 87 | } 88 | } 89 | 90 | func handleSnapshot(s *format.Snapshot) { 91 | switch s.EventType { 92 | case format.SnapshotEventTypeBegin: 93 | log.Printf("📸 SNAPSHOT BEGIN | LSN: %s | Time: %s", 94 | s.LSN.String(), 95 | s.ServerTime.Format("15:04:05")) 96 | 97 | case format.SnapshotEventTypeData: 98 | slog.Info("snapshot message received", "data", s.Data) 99 | 100 | case format.SnapshotEventTypeEnd: 101 | log.Printf("📸 SNAPSHOT END | LSN: %s | Time: %s", 102 | s.LSN.String(), 103 | s.ServerTime.Format("15:04:05")) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /pq/message/format/update.go: -------------------------------------------------------------------------------- 1 | package format 2 | 3 | import ( 4 | "encoding/binary" 5 | "time" 6 | 7 | "github.com/Trendyol/go-pq-cdc/pq/message/tuple" 8 | "github.com/go-playground/errors" 9 | ) 10 | 11 | const ( 12 | UpdateTupleTypeKey = 'K' 13 | UpdateTupleTypeOld = 'O' 14 | UpdateTupleTypeNew = 'N' 15 | ) 16 | 17 | type Update struct { 18 | MessageTime time.Time 19 | NewTupleData *tuple.Data 20 | NewDecoded map[string]any 21 | OldTupleData *tuple.Data 22 | OldDecoded map[string]any 23 | TableNamespace string 24 | TableName string 25 | OID uint32 26 | XID uint32 27 | OldTupleType uint8 28 | } 29 | 30 | func NewUpdate(data []byte, streamedTransaction bool, relation map[uint32]*Relation, serverTime time.Time) (*Update, error) { 31 | msg := &Update{ 32 | MessageTime: serverTime, 33 | } 34 | if err := msg.decode(data, streamedTransaction); err != nil { 35 | return nil, err 36 | } 37 | 38 | rel, ok := relation[msg.OID] 39 | if !ok { 40 | return nil, errors.New("relation not found") 41 | } 42 | 43 | msg.TableNamespace = rel.Namespace 44 | msg.TableName = rel.Name 45 | 46 | var err error 47 | 48 | if msg.OldTupleData != nil { 49 | msg.OldDecoded, err = msg.OldTupleData.DecodeWithColumn(rel.Columns) 50 | if err != nil { 51 | return nil, err 52 | } 53 | } 54 | 55 | msg.NewDecoded, err = msg.NewTupleData.DecodeWithColumn(rel.Columns) 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | return msg, nil 61 | } 62 | 63 | func (m *Update) decode(data []byte, streamedTransaction bool) error { 64 | skipByte := 1 65 | 66 | if streamedTransaction { 67 | if len(data) < 11 { 68 | return errors.Newf("streamed transaction update message length must be at least 11 byte, but got %d", len(data)) 69 | } 70 | 71 | m.XID = binary.BigEndian.Uint32(data[skipByte:]) 72 | skipByte += 4 73 | } 74 | 75 | if len(data) < 7 { 76 | return errors.Newf("update message length must be at least 7 byte, but got %d", len(data)) 77 | } 78 | 79 | m.OID = binary.BigEndian.Uint32(data[skipByte:]) 80 | skipByte += 4 81 | 82 | m.OldTupleType = data[skipByte] 83 | 84 | var err error 85 | 86 | switch m.OldTupleType { 87 | case UpdateTupleTypeKey, UpdateTupleTypeOld: 88 | m.OldTupleData, err = tuple.NewData(data, m.OldTupleType, skipByte) 89 | if err != nil { 90 | return errors.Wrap(err, "update message old tuple data") 91 | } 92 | skipByte = m.OldTupleData.SkipByte 93 | fallthrough 94 | case UpdateTupleTypeNew: 95 | m.NewTupleData, err = tuple.NewData(data, UpdateTupleTypeNew, skipByte) 96 | if err != nil { 97 | return errors.Wrap(err, "update message new tuple data") 98 | } 99 | 100 | if m.OldTupleData != nil { 101 | for i, col := range m.NewTupleData.Columns { 102 | // because toasted columns not sent until the toasted column updated 103 | if col.DataType == tuple.DataTypeToast { 104 | m.NewTupleData.Columns[i] = m.OldTupleData.Columns[i] 105 | } 106 | } 107 | } 108 | default: 109 | return errors.New("update message undefined tuple type") 110 | } 111 | 112 | return nil 113 | } 114 | -------------------------------------------------------------------------------- /integration_test/basic_functionality_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | cdc "github.com/Trendyol/go-pq-cdc" 7 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 8 | "github.com/Trendyol/go-pq-cdc/pq/replication" 9 | "github.com/stretchr/testify/assert" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | // go test -race -p=1 -v -run=TestBasicFunctionalitySanityTest 15 | func TestBasicFunctionalitySanityTest(t *testing.T) { 16 | fmt.Println("sanity test") 17 | } 18 | 19 | func TestBasicFunctionality(t *testing.T) { 20 | ctx := context.Background() 21 | 22 | cdcCfg := Config 23 | cdcCfg.Slot.Name = "slot_test_basic_functionality" 24 | 25 | postgresConn, err := newPostgresConn() 26 | if !assert.NoError(t, err) { 27 | t.FailNow() 28 | } 29 | 30 | if !assert.NoError(t, SetupTestDB(ctx, postgresConn, cdcCfg)) { 31 | t.FailNow() 32 | } 33 | 34 | messageCh := make(chan any, 500) 35 | handlerFunc := func(ctx *replication.ListenerContext) { 36 | switch msg := ctx.Message.(type) { 37 | case *format.Insert, *format.Delete, *format.Update: 38 | messageCh <- msg 39 | } 40 | _ = ctx.Ack() 41 | } 42 | 43 | connector, err := cdc.NewConnector(ctx, cdcCfg, handlerFunc) 44 | if !assert.NoError(t, err) { 45 | t.FailNow() 46 | } 47 | 48 | t.Cleanup(func() { 49 | connector.Close() 50 | assert.NoError(t, RestoreDB(ctx)) 51 | assert.NoError(t, postgresConn.Close(ctx)) 52 | }) 53 | 54 | go connector.Start(ctx) 55 | 56 | waitCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 57 | if !assert.NoError(t, connector.WaitUntilReady(waitCtx)) { 58 | t.FailNow() 59 | } 60 | cancel() 61 | 62 | t.Run("Insert 10 book to table. Then check messages and metric", func(t *testing.T) { 63 | books := CreateBooks(10) 64 | for _, b := range books { 65 | err = pgExec(ctx, postgresConn, fmt.Sprintf("INSERT INTO books(id, name) VALUES(%d, '%s')", b.ID, b.Name)) 66 | assert.NoError(t, err) 67 | } 68 | 69 | for i := range 10 { 70 | m := <-messageCh 71 | assert.Equal(t, books[i].Map(), m.(*format.Insert).Decoded) 72 | } 73 | 74 | metric, _ := fetchInsertOpMetric() 75 | assert.True(t, metric == 10) 76 | }) 77 | 78 | t.Run("Update 5 book on table. Then check messages and metric", func(t *testing.T) { 79 | books := CreateBooks(5) 80 | for i, b := range books { 81 | b.ID = i + 1 82 | books[i] = b 83 | err = pgExec(ctx, postgresConn, fmt.Sprintf("UPDATE books SET name = '%s' WHERE id = %d", b.Name, b.ID)) 84 | assert.NoError(t, err) 85 | } 86 | 87 | for i := range 5 { 88 | m := <-messageCh 89 | assert.Equal(t, books[i].Map(), m.(*format.Update).NewDecoded) 90 | } 91 | 92 | metric, _ := fetchUpdateOpMetric() 93 | assert.True(t, metric == 5) 94 | }) 95 | 96 | t.Run("Delete 5 book from table. Then check messages and metric", func(t *testing.T) { 97 | for i := range 5 { 98 | err = pgExec(ctx, postgresConn, fmt.Sprintf("DELETE FROM books WHERE id = %d", i+1)) 99 | assert.NoError(t, err) 100 | } 101 | 102 | for i := range 5 { 103 | m := <-messageCh 104 | assert.Equal(t, int32(i+1), m.(*format.Delete).OldDecoded["id"]) 105 | } 106 | 107 | metric, _ := fetchDeleteOpMetric() 108 | assert.True(t, metric == 5) 109 | }) 110 | } 111 | -------------------------------------------------------------------------------- /benchmark/benchmark_initial/README.md: -------------------------------------------------------------------------------- 1 | ## 10 M Insert Test 2 | 3 | ### Hardware 4 | 5 | ```txt 6 | PC: Macbook Apple M1 Pro (2021) 7 | Memory: 32 GB 8 | 9 | go-pq-cdc: 10 | resources: 11 | limits: 12 | cpus: 1 13 | memory: 512M 14 | reservations: 15 | cpus: '0.25' 16 | memory: 128M 17 | 18 | Debezium: 19 | resources: 20 | limits: 21 | cpus: 2 22 | memory: 1024M 23 | reservations: 24 | cpus: '0.25' 25 | memory: 128M 26 | ``` 27 | 28 | ### Results 29 | 30 | In this snapshot benchmark, go-pq-cdc delivers roughly 2x higher throughput in the 1x run (~1 min vs 2 min) compared to Debezium, with lower CPU usage (~70% vs ~170%) and dramatically less memory (~45 MB vs ~2.5 GB), and in the 3x run it reduces the duration to ~20 seconds while maintaining a similarly efficient CPU/memory profile.” 31 | 32 | #### 1x Test 33 | 34 | | | go-pq-cdc | Debezium | 35 | | -------------------- | ---------- | ---------- | 36 | | Row Count | 10 m | 10 m | 37 | | Elapsed Time | 1 min | 2 min | 38 | | Cpu Usage Max | 69.9% | 167% | 39 | | Memory Usage Max | 44.6 MB | 2.56 GB | 40 | | Received Traffic Max | 6.30 MiB/s | 4.85 MiB/s | 41 | | Sent Traffic Max | 15.4 MiB/s | 43.7 MiB/s | 42 | 43 | > Note: snapshot progress and snapshot chunks metrics is not able to full completed. Because we are benchmarking on snapshot only mode. It means after snapshot finish, our container exit so prometheus cannot able to collect. No worries :) 44 | 45 | > You can also see postgresql grafana dashboard. We cannot see any big difference. 46 | 47 | ![10m_1x](./10m_1x.png) 48 | 49 | #### 3x Test 50 | 51 | | | go-pq-cdc | Debezium | 52 | | -------------------- | ---------- | ---------- | 53 | | Row Count | 10 m | 10 m | 54 | | Elapsed Time | 20 sec | 2 min | 55 | | Cpu Usage Max | 71% | 187% | 56 | | Memory Usage Max | 47.6 MB | 2.43 GB | 57 | | Received Traffic Max | 6.30 MiB/s | 4.85 MiB/s | 58 | | Sent Traffic Max | 15.4 MiB/s | 46.1 MiB/s | 59 | 60 | ![10m_3x](./10m_3x.png) 61 | 62 | ## Requirements 63 | 64 | - [Docker](https://docs.docker.com/compose/install/) 65 | - [psql](https://www.postgresql.org/download/) 66 | 67 | ## Instructions 68 | 69 | - Start the containers 70 | ```sh 71 | docker compose up -d 72 | ``` 73 | - Connect to Postgres database: 74 | ```sh 75 | psql postgres://cdc_user:cdc_pass@127.0.0.1:5432/cdc_db 76 | ``` 77 | - Insert data to users table: 78 | ```sql 79 | INSERT INTO users (name) 80 | SELECT 81 | 'Oyleli' || i 82 | FROM generate_series(1, 1000000) AS i; 83 | ``` 84 | - Go to grafana dashboard: http://localhost:3000/d/edl1ybvsmc64gb/benchmark?orgId=1 85 | > **Grafana Credentials** 86 | > Username: `go-pq-cdc-user` Password: `go-pq-cdc-pass` 87 | 88 | ## Ports 89 | 90 | - RedPanda Console: `8085` http://localhost:8085 91 | - RedPanda: `19092` http://localhost:19092 92 | - Grafana: `3000` http://localhost:3000 93 | - Prometheus: `9090` http://localhost:9090 94 | - cAdvisor: `8080` http://localhost:8080 95 | - PostgreSQL:`5432` http://localhost:5432 96 | - PostgreSQL Metric Exporter: `9187` http://localhost:9187 97 | - Debezium: `9093` http://localhost:9093 98 | - go-pq-cdc Metric: `2112` http://localhost:2112 99 | -------------------------------------------------------------------------------- /benchmark/benchmark_cdc/go-pq-cdc-kafka/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "log/slog" 7 | "os" 8 | "strconv" 9 | "time" 10 | 11 | cdc "github.com/Trendyol/go-pq-cdc-kafka" 12 | "github.com/Trendyol/go-pq-cdc-kafka/config" 13 | cdcconfig "github.com/Trendyol/go-pq-cdc/config" 14 | "github.com/Trendyol/go-pq-cdc/pq/publication" 15 | "github.com/Trendyol/go-pq-cdc/pq/slot" 16 | "github.com/segmentio/kafka-go" 17 | gokafka "github.com/segmentio/kafka-go" 18 | ) 19 | 20 | /* 21 | psql "postgres://cdc_user:cdc_pass@127.0.0.1/cdc_db?replication=database" 22 | 23 | INSERT INTO users (name) 24 | SELECT 25 | 'Oyleli' || i 26 | FROM generate_series(1, 10000000) AS i; 27 | */ 28 | 29 | type Message struct { 30 | Message kafka.Message 31 | Ack func() error 32 | } 33 | 34 | func main() { 35 | ctx := context.Background() 36 | 37 | cfg := config.Connector{ 38 | CDC: cdcconfig.Config{ 39 | Host: "postgres", 40 | Port: 5432, 41 | Username: "cdc_user", 42 | Password: "cdc_pass", 43 | Database: "cdc_db", 44 | DebugMode: false, 45 | Publication: publication.Config{ 46 | CreateIfNotExists: true, 47 | Name: "cdc_publication", 48 | Operations: publication.Operations{ 49 | publication.OperationInsert, 50 | publication.OperationDelete, 51 | publication.OperationTruncate, 52 | publication.OperationUpdate, 53 | }, 54 | Tables: publication.Tables{publication.Table{ 55 | Name: "users", 56 | ReplicaIdentity: publication.ReplicaIdentityFull, 57 | }}, 58 | }, 59 | Slot: slot.Config{ 60 | CreateIfNotExists: true, 61 | Name: "cdc_slot", 62 | SlotActivityCheckerInterval: 3000, 63 | }, 64 | Metric: cdcconfig.MetricConfig{ 65 | Port: 2112, 66 | }, 67 | Logger: cdcconfig.LoggerConfig{ 68 | LogLevel: slog.LevelInfo, 69 | }, 70 | }, 71 | Kafka: config.Kafka{ 72 | TableTopicMapping: map[string]string{ 73 | "public.users": "cdc.test.produce", 74 | }, 75 | Brokers: []string{"redpanda:9092"}, 76 | AllowAutoTopicCreation: true, 77 | ProducerBatchTickerDuration: time.Millisecond * 100, 78 | ProducerBatchSize: 10000, 79 | }, 80 | } 81 | 82 | connector, err := cdc.NewConnector(ctx, cfg, Handler) 83 | if err != nil { 84 | slog.Error("new connector", "error", err) 85 | os.Exit(1) 86 | } 87 | 88 | defer connector.Close() 89 | connector.Start(ctx) 90 | } 91 | 92 | func Handler(msg *cdc.Message) []gokafka.Message { 93 | slog.Info("change captured", "message", msg) 94 | if msg.Type.IsUpdate() || msg.Type.IsInsert() { 95 | msg.NewData["operation"] = msg.Type 96 | newData, _ := json.Marshal(msg.NewData) 97 | 98 | return []gokafka.Message{ 99 | { 100 | Headers: nil, 101 | Key: []byte(strconv.Itoa(int(msg.NewData["id"].(int32)))), 102 | Value: newData, 103 | }, 104 | } 105 | } 106 | 107 | if msg.Type.IsDelete() { 108 | msg.OldData["operation"] = msg.Type 109 | oldData, _ := json.Marshal(msg.OldData) 110 | 111 | return []gokafka.Message{ 112 | { 113 | Headers: nil, 114 | Key: []byte(strconv.Itoa(int(msg.OldData["id"].(int32)))), 115 | Value: oldData, 116 | }, 117 | } 118 | } 119 | 120 | return []gokafka.Message{} 121 | } 122 | -------------------------------------------------------------------------------- /integration_test/copy_protocol_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "context" 5 | cdc "github.com/Trendyol/go-pq-cdc" 6 | "github.com/Trendyol/go-pq-cdc/config" 7 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 8 | "github.com/Trendyol/go-pq-cdc/pq/replication" 9 | "github.com/jackc/pgx/v5" 10 | "github.com/jackc/pgx/v5/pgxpool" 11 | "github.com/stretchr/testify/assert" 12 | "sync/atomic" 13 | "testing" 14 | "time" 15 | ) 16 | 17 | func TestCopyProtocol(t *testing.T) { 18 | ctx := context.Background() 19 | 20 | cdcCfg := Config 21 | cdcCfg.Slot.Name = "slot_test_copy_protocol" 22 | 23 | postgresConn, err := newPostgresConn() 24 | if !assert.NoError(t, err) { 25 | t.FailNow() 26 | } 27 | 28 | if !assert.NoError(t, SetupTestDB(ctx, postgresConn, cdcCfg)) { 29 | t.FailNow() 30 | } 31 | 32 | messageCh := make(chan *replication.ListenerContext) 33 | totalCounter := atomic.Int64{} 34 | handlerFunc := func(ctx *replication.ListenerContext) { 35 | switch ctx.Message.(type) { 36 | case *format.Insert, *format.Delete, *format.Update: 37 | totalCounter.Add(1) 38 | messageCh <- ctx 39 | } 40 | } 41 | 42 | cdc2Cfg := cdcCfg 43 | cdc2Cfg.Metric.Port = 8085 44 | connector, err := cdc.NewConnector(ctx, cdcCfg, handlerFunc) 45 | if !assert.NoError(t, err) { 46 | t.FailNow() 47 | } 48 | 49 | connector2, err := cdc.NewConnector(ctx, cdcCfg, handlerFunc) 50 | if !assert.NoError(t, err) { 51 | t.FailNow() 52 | } 53 | 54 | cfg := config.Config{Host: Config.Host, Port: Config.Port, Username: "postgres", Password: "postgres", Database: Config.Database} 55 | pool, err := pgxpool.New(ctx, cfg.DSNWithoutSSL()) 56 | if !assert.NoError(t, err) { 57 | t.FailNow() 58 | } 59 | 60 | t.Cleanup(func() { 61 | pool.Close() 62 | connector2.Close() 63 | assert.NoError(t, RestoreDB(ctx)) 64 | }) 65 | 66 | go connector.Start(ctx) 67 | 68 | waitCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 69 | if !assert.NoError(t, connector.WaitUntilReady(waitCtx)) { 70 | t.FailNow() 71 | } 72 | cancel() 73 | 74 | go connector2.Start(ctx) 75 | 76 | t.Run("Insert 30 book to table with Copy protocol. Then stop the consumer after 16th message processed", func(t *testing.T) { 77 | entries := make([][]any, 30) 78 | books := CreateBooks(30) 79 | 80 | for i, user := range books { 81 | entries[i] = []any{user.ID, user.Name} 82 | } 83 | 84 | _, err = pool.CopyFrom( 85 | ctx, 86 | pgx.Identifier{"books"}, 87 | []string{"id", "name"}, 88 | pgx.CopyFromRows(entries), 89 | ) 90 | if err != nil { 91 | t.Errorf("error copying into %s table: %v", "books", err) 92 | } 93 | 94 | for { 95 | m := <-messageCh 96 | if v, ok := m.Message.(*format.Insert); ok { 97 | if v.Decoded["id"].(int32) == 16 { 98 | connector.Close() 99 | break 100 | } 101 | } 102 | 103 | assert.NoError(t, m.Ack()) 104 | } 105 | }) 106 | 107 | t.Run("Run CDC again. Then check message count after all messages consumed", func(t *testing.T) { 108 | waitCtx, cancel = context.WithTimeout(context.Background(), 3*time.Second) 109 | if !assert.NoError(t, connector2.WaitUntilReady(waitCtx)) { 110 | t.FailNow() 111 | } 112 | cancel() 113 | 114 | for { 115 | m := <-messageCh 116 | if v, ok := m.Message.(*format.Insert); ok { 117 | if v.Decoded["id"].(int32) == 30 { 118 | break 119 | } 120 | } 121 | } 122 | }) 123 | } 124 | -------------------------------------------------------------------------------- /pq/timescaledb/hypertable.go: -------------------------------------------------------------------------------- 1 | package timescaledb 2 | 3 | import ( 4 | "context" 5 | goerrors "errors" 6 | "fmt" 7 | "sync" 8 | "time" 9 | 10 | "github.com/Trendyol/go-pq-cdc/logger" 11 | "github.com/Trendyol/go-pq-cdc/pq" 12 | "github.com/go-playground/errors" 13 | "github.com/jackc/pgx/v5/pgconn" 14 | "github.com/jackc/pgx/v5/pgtype" 15 | ) 16 | 17 | var typeMap = pgtype.NewMap() 18 | 19 | var HyperTables = sync.Map{} 20 | 21 | type TimescaleDB struct { 22 | conn pq.Connection 23 | ticker *time.Ticker 24 | } 25 | 26 | func NewTimescaleDB(ctx context.Context, dsn string) (*TimescaleDB, error) { 27 | conn, err := pq.NewConnection(ctx, dsn) 28 | if err != nil { 29 | return nil, errors.Wrap(err, "new postgresql connection") 30 | } 31 | 32 | return &TimescaleDB{conn: conn, ticker: time.NewTicker(time.Second)}, nil 33 | } 34 | 35 | func (tdb *TimescaleDB) SyncHyperTables(ctx context.Context) { 36 | for range tdb.ticker.C { 37 | hyperTables, err := tdb.FindHyperTables(ctx) 38 | if err != nil { 39 | logger.Error("timescale tables", "error", err) 40 | continue 41 | } 42 | 43 | logger.Debug("timescale tables", "tables", hyperTables) 44 | } 45 | } 46 | 47 | func (tdb *TimescaleDB) FindHyperTables(ctx context.Context) (map[string]string, error) { 48 | query := "SELECT h.hypertable_schema, h.hypertable_name, c.chunk_schema, c.chunk_name FROM timescaledb_information.chunks c JOIN timescaledb_information.hypertables h ON c.hypertable_schema = h.hypertable_schema AND c.hypertable_name = h.hypertable_name;" 49 | resultReader := tdb.conn.Exec(ctx, query) 50 | results, err := resultReader.ReadAll() 51 | if err != nil { 52 | var pgErr *pgconn.PgError 53 | if goerrors.As(err, &pgErr) { 54 | if pgErr.Code == "42P01" { 55 | tdb.ticker.Stop() 56 | logger.Debug("timescale db hypertable relation not found", "error", err) 57 | return nil, nil 58 | } 59 | } 60 | 61 | return nil, errors.Wrap(err, "hyper tables result") 62 | } 63 | 64 | if len(results) == 0 || results[0].CommandTag.String() == "SELECT 0" { 65 | return nil, nil 66 | } 67 | 68 | if err = resultReader.Close(); err != nil { 69 | return nil, errors.Wrap(err, "hyper tables result reader close") 70 | } 71 | 72 | ht, err := decodeHyperTablesResult(results) 73 | if err != nil { 74 | return nil, errors.Wrap(err, "hyper tables result decode") 75 | } 76 | 77 | for k, v := range ht { 78 | HyperTables.Store(k, v) 79 | } 80 | 81 | return ht, nil 82 | } 83 | 84 | func decodeHyperTablesResult(results []*pgconn.Result) (map[string]string, error) { 85 | res := make(map[string]string) 86 | 87 | for _, result := range results { 88 | for i := range len(result.Rows) { 89 | var hyperName, hyperSchema, chunkSchema, chunkName string 90 | for j, fd := range result.FieldDescriptions { 91 | v, err := decodeTextColumnData(result.Rows[i][j], fd.DataTypeOID) 92 | if err != nil { 93 | return nil, err 94 | } 95 | 96 | if v == nil { 97 | continue 98 | } 99 | 100 | switch fd.Name { 101 | case "hypertable_schema": 102 | hyperSchema = v.(string) 103 | case "hypertable_name": 104 | hyperName = v.(string) 105 | case "chunk_schema": 106 | chunkSchema = v.(string) 107 | case "chunk_name": 108 | chunkName = v.(string) 109 | } 110 | } 111 | res[fmt.Sprintf("%s.%s", chunkSchema, chunkName)] = fmt.Sprintf("%s.%s", hyperSchema, hyperName) 112 | } 113 | } 114 | 115 | return res, nil 116 | } 117 | 118 | func decodeTextColumnData(data []byte, dataType uint32) (interface{}, error) { 119 | if dt, ok := typeMap.TypeForOID(dataType); ok { 120 | return dt.Codec.DecodeValue(typeMap, dataType, pgtype.TextFormatCode, data) 121 | } 122 | return string(data), nil 123 | } 124 | -------------------------------------------------------------------------------- /integration_test/transactional_process_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "context" 5 | cdc "github.com/Trendyol/go-pq-cdc" 6 | "github.com/Trendyol/go-pq-cdc/config" 7 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 8 | "github.com/Trendyol/go-pq-cdc/pq/replication" 9 | "github.com/jackc/pgx/v5/pgxpool" 10 | "github.com/stretchr/testify/assert" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | func TestTransactionalProcess(t *testing.T) { 16 | ctx := context.Background() 17 | 18 | cdcCfg := Config 19 | cdcCfg.Slot.Name = "slot_test_transactional_process" 20 | 21 | postgresConn, err := newPostgresConn() 22 | if !assert.NoError(t, err) { 23 | t.FailNow() 24 | } 25 | 26 | if !assert.NoError(t, SetupTestDB(ctx, postgresConn, cdcCfg)) { 27 | t.FailNow() 28 | } 29 | 30 | messageCh := make(chan any, 500) 31 | handlerFunc := func(ctx *replication.ListenerContext) { 32 | switch msg := ctx.Message.(type) { 33 | case *format.Insert, *format.Delete, *format.Update: 34 | messageCh <- msg 35 | } 36 | _ = ctx.Ack() 37 | } 38 | 39 | connector, err := cdc.NewConnector(ctx, cdcCfg, handlerFunc) 40 | if !assert.NoError(t, err) { 41 | t.FailNow() 42 | } 43 | 44 | cfg := config.Config{Host: Config.Host, Port: Config.Port, Username: "postgres", Password: "postgres", Database: Config.Database} 45 | pool, err := pgxpool.New(ctx, cfg.DSNWithoutSSL()) 46 | if !assert.NoError(t, err) { 47 | t.FailNow() 48 | } 49 | 50 | t.Cleanup(func() { 51 | connector.Close() 52 | err = RestoreDB(ctx) 53 | assert.NoError(t, err) 54 | 55 | pool.Close() 56 | }) 57 | 58 | go connector.Start(ctx) 59 | 60 | waitCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 61 | if !assert.NoError(t, connector.WaitUntilReady(waitCtx)) { 62 | t.FailNow() 63 | } 64 | cancel() 65 | 66 | t.Run("Start transactional operation and commit. Then check the messages and metrics", func(t *testing.T) { 67 | tx, err := pool.Begin(ctx) 68 | assert.NoError(t, err) 69 | 70 | _, err = tx.Exec(ctx, "INSERT INTO books (id, name) VALUES (12, 'j*va is best')") 71 | assert.NoError(t, err) 72 | 73 | _, err = tx.Exec(ctx, "UPDATE books SET name = 'go is best' WHERE id = 12") 74 | assert.NoError(t, err) 75 | 76 | err = tx.Commit(ctx) 77 | assert.NoError(t, err) 78 | 79 | insertMessage := <-messageCh 80 | assert.Equal(t, map[string]any{"id": int32(12), "name": "j*va is best"}, insertMessage.(*format.Insert).Decoded) 81 | updateMessage := <-messageCh 82 | assert.Equal(t, map[string]any{"id": int32(12), "name": "go is best"}, updateMessage.(*format.Update).NewDecoded) 83 | 84 | updateMetric, _ := fetchUpdateOpMetric() 85 | insertMetric, _ := fetchInsertOpMetric() 86 | deleteMetric, _ := fetchDeleteOpMetric() 87 | assert.True(t, updateMetric == 1) 88 | assert.True(t, insertMetric == 1) 89 | assert.True(t, deleteMetric == 0) 90 | }) 91 | 92 | t.Run("Start transactional operation and rollback. Then Delete book which id is 12. Then check the messages and metrics", func(t *testing.T) { 93 | tx, err := pool.Begin(ctx) 94 | assert.NoError(t, err) 95 | 96 | _, err = tx.Exec(ctx, "INSERT INTO books (id, name) VALUES (13, 'j*va is best')") 97 | assert.NoError(t, err) 98 | 99 | _, err = tx.Exec(ctx, "UPDATE books SET name = 'go is best' WHERE id = 13") 100 | assert.NoError(t, err) 101 | 102 | err = tx.Rollback(ctx) 103 | assert.NoError(t, err) 104 | 105 | _, err = pool.Exec(ctx, "DELETE FROM books WHERE id = 12") 106 | assert.NoError(t, err) 107 | 108 | deleteMessage := <-messageCh 109 | assert.Equal(t, int32(12), deleteMessage.(*format.Delete).OldDecoded["id"]) 110 | 111 | updateMetric, _ := fetchUpdateOpMetric() 112 | insertMetric, _ := fetchInsertOpMetric() 113 | deleteMetric, _ := fetchDeleteOpMetric() 114 | assert.True(t, updateMetric == 1) 115 | assert.True(t, insertMetric == 1) 116 | assert.True(t, deleteMetric == 1) 117 | }) 118 | } 119 | -------------------------------------------------------------------------------- /pq/publication/replica_identity.go: -------------------------------------------------------------------------------- 1 | package publication 2 | 3 | import ( 4 | "context" 5 | goerrors "errors" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/Trendyol/go-pq-cdc/logger" 10 | "github.com/go-playground/errors" 11 | "github.com/jackc/pgx/v5/pgconn" 12 | ) 13 | 14 | const ( 15 | ReplicaIdentityFull = "FULL" 16 | ReplicaIdentityDefault = "DEFAULT" 17 | ) 18 | 19 | var ( 20 | ErrorTablesNotExists = goerrors.New("table does not exists") 21 | ReplicaIdentityOptions = []string{ReplicaIdentityDefault, ReplicaIdentityFull} 22 | ReplicaIdentityMap = map[string]string{ 23 | "d": ReplicaIdentityDefault, // primary key on old value 24 | "f": ReplicaIdentityFull, // full row on old value 25 | } 26 | ) 27 | 28 | func (c *Publication) SetReplicaIdentities(ctx context.Context) error { 29 | if !c.cfg.CreateIfNotExists { 30 | return nil 31 | } 32 | 33 | tables, err := c.GetReplicaIdentities(ctx) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | diff := c.cfg.Tables.Diff(tables) 39 | 40 | for _, d := range diff { 41 | if err = c.AlterTableReplicaIdentity(ctx, d); err != nil { 42 | return err 43 | } 44 | } 45 | 46 | return nil 47 | } 48 | 49 | func (c *Publication) AlterTableReplicaIdentity(ctx context.Context, t Table) error { 50 | resultReader := c.conn.Exec(ctx, fmt.Sprintf("ALTER TABLE %s.%s REPLICA IDENTITY %s;", t.Schema, t.Name, t.ReplicaIdentity)) 51 | _, err := resultReader.ReadAll() 52 | if err != nil { 53 | return errors.Wrap(err, "table replica identity update result") 54 | } 55 | 56 | if err = resultReader.Close(); err != nil { 57 | return errors.Wrap(err, "table replica identity update result reader close") 58 | } 59 | 60 | logger.Info("table replica identity updated", "name", t.Name, "replica_identity", t.ReplicaIdentity) 61 | 62 | return nil 63 | } 64 | 65 | func (c *Publication) GetReplicaIdentities(ctx context.Context) ([]Table, error) { 66 | tableNames := make([]string, len(c.cfg.Tables)) 67 | 68 | for i, t := range c.cfg.Tables { 69 | if t.Schema == "" && !strings.Contains(t.Name, ".") { 70 | tableNames[i] = "'" + t.Name + "'" 71 | } else { 72 | tableNames[i] = "'" + t.Schema + "." + t.Name + "'" 73 | } 74 | } 75 | 76 | query := fmt.Sprintf("SELECT relname AS table_name, n.nspname AS schema_name, relreplident AS replica_identity FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid WHERE concat(n.nspname, '.', c.relname) IN (%s)", strings.Join(tableNames, ", ")) 77 | 78 | logger.Debug("executing query: ", query) 79 | 80 | resultReader := c.conn.Exec(ctx, query) 81 | results, err := resultReader.ReadAll() 82 | if err != nil { 83 | return nil, errors.Wrap(err, "replica identities result") 84 | } 85 | 86 | if len(results) == 0 || results[0].CommandTag.String() == "SELECT 0" { 87 | return nil, ErrorTablesNotExists 88 | } 89 | 90 | if err = resultReader.Close(); err != nil { 91 | return nil, errors.Wrap(err, "replica identities result reader close") 92 | } 93 | 94 | replicaIdentities, err := decodeReplicaIdentitiesResult(results) 95 | if err != nil { 96 | return nil, errors.Wrap(err, "replica identities result decode") 97 | } 98 | 99 | return replicaIdentities, nil 100 | } 101 | 102 | func decodeReplicaIdentitiesResult(results []*pgconn.Result) ([]Table, error) { 103 | var res []Table 104 | 105 | for _, result := range results { 106 | for i := range len(result.Rows) { 107 | var t Table 108 | for j, fd := range result.FieldDescriptions { 109 | v, err := decodeTextColumnData(result.Rows[i][j], fd.DataTypeOID) 110 | if err != nil { 111 | return nil, err 112 | } 113 | 114 | if v == nil { 115 | continue 116 | } 117 | 118 | switch fd.Name { 119 | case "table_name": 120 | t.Name = v.(string) 121 | case "schema_name": 122 | t.Schema = v.(string) 123 | case "replica_identity": 124 | t.ReplicaIdentity = ReplicaIdentityMap[string(v.(int32))] 125 | } 126 | } 127 | res = append(res, t) 128 | } 129 | } 130 | 131 | return res, nil 132 | } 133 | -------------------------------------------------------------------------------- /integration_test/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Trendyol/go-pq-cdc/integration 2 | 3 | go 1.22.4 4 | 5 | replace github.com/Trendyol/go-pq-cdc => ../ 6 | 7 | require ( 8 | github.com/Trendyol/go-pq-cdc v0.0.0-00010101000000-000000000000 9 | github.com/go-playground/errors v3.3.0+incompatible 10 | github.com/golang-migrate/migrate/v4 v4.17.1 11 | github.com/jackc/pgx/v5 v5.6.0 12 | github.com/stretchr/testify v1.9.0 13 | github.com/testcontainers/testcontainers-go v0.31.0 14 | ) 15 | 16 | require ( 17 | dario.cat/mergo v1.0.0 // indirect 18 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect 19 | github.com/Microsoft/go-winio v0.6.2 // indirect 20 | github.com/Microsoft/hcsshim v0.11.5 // indirect 21 | github.com/avast/retry-go/v4 v4.6.0 // indirect 22 | github.com/beorn7/perks v1.0.1 // indirect 23 | github.com/cenkalti/backoff/v4 v4.2.1 // indirect 24 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 25 | github.com/containerd/containerd v1.7.18 // indirect 26 | github.com/containerd/errdefs v0.1.0 // indirect 27 | github.com/containerd/log v0.1.0 // indirect 28 | github.com/cpuguy83/dockercfg v0.3.1 // indirect 29 | github.com/davecgh/go-spew v1.1.1 // indirect 30 | github.com/distribution/reference v0.5.0 // indirect 31 | github.com/docker/docker v25.0.6+incompatible // indirect 32 | github.com/docker/go-connections v0.5.0 // indirect 33 | github.com/docker/go-units v0.5.0 // indirect 34 | github.com/felixge/httpsnoop v1.0.4 // indirect 35 | github.com/go-logr/logr v1.4.1 // indirect 36 | github.com/go-logr/stdr v1.2.2 // indirect 37 | github.com/go-ole/go-ole v1.2.6 // indirect 38 | github.com/gogo/protobuf v1.3.2 // indirect 39 | github.com/golang/protobuf v1.5.4 // indirect 40 | github.com/google/uuid v1.6.0 // indirect 41 | github.com/hashicorp/errwrap v1.1.0 // indirect 42 | github.com/hashicorp/go-multierror v1.1.1 // indirect 43 | github.com/jackc/pgpassfile v1.0.0 // indirect 44 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect 45 | github.com/jackc/puddle/v2 v2.2.1 // indirect 46 | github.com/klauspost/compress v1.16.0 // indirect 47 | github.com/lib/pq v1.10.9 // indirect 48 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect 49 | github.com/magiconair/properties v1.8.7 // indirect 50 | github.com/moby/patternmatcher v0.6.0 // indirect 51 | github.com/moby/sys/sequential v0.5.0 // indirect 52 | github.com/moby/sys/user v0.1.0 // indirect 53 | github.com/moby/term v0.5.0 // indirect 54 | github.com/morikuni/aec v1.0.0 // indirect 55 | github.com/opencontainers/go-digest v1.0.0 // indirect 56 | github.com/opencontainers/image-spec v1.1.0 // indirect 57 | github.com/pkg/errors v0.9.1 // indirect 58 | github.com/pmezard/go-difflib v1.0.0 // indirect 59 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect 60 | github.com/prometheus/client_golang v1.19.1 // indirect 61 | github.com/prometheus/client_model v0.5.0 // indirect 62 | github.com/prometheus/common v0.48.0 // indirect 63 | github.com/prometheus/procfs v0.12.0 // indirect 64 | github.com/shirou/gopsutil/v3 v3.23.12 // indirect 65 | github.com/shoenig/go-m1cpu v0.1.6 // indirect 66 | github.com/sirupsen/logrus v1.9.3 // indirect 67 | github.com/tklauser/go-sysconf v0.3.12 // indirect 68 | github.com/tklauser/numcpus v0.6.1 // indirect 69 | github.com/yusufpapurcu/wmi v1.2.3 // indirect 70 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect 71 | go.opentelemetry.io/otel v1.24.0 // indirect 72 | go.opentelemetry.io/otel/metric v1.24.0 // indirect 73 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 74 | go.uber.org/atomic v1.7.0 // indirect 75 | golang.org/x/crypto v0.22.0 // indirect 76 | golang.org/x/sync v0.6.0 // indirect 77 | golang.org/x/sys v0.19.0 // indirect 78 | golang.org/x/text v0.14.0 // indirect 79 | google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 // indirect 80 | google.golang.org/grpc v1.59.0 // indirect 81 | google.golang.org/protobuf v1.34.1 // indirect 82 | gopkg.in/yaml.v2 v2.4.0 // indirect 83 | gopkg.in/yaml.v3 v3.0.1 // indirect 84 | ) 85 | -------------------------------------------------------------------------------- /pq/publication/publication.go: -------------------------------------------------------------------------------- 1 | package publication 2 | 3 | import ( 4 | "context" 5 | goerrors "errors" 6 | "strings" 7 | 8 | "github.com/Trendyol/go-pq-cdc/logger" 9 | "github.com/Trendyol/go-pq-cdc/pq" 10 | "github.com/go-playground/errors" 11 | "github.com/jackc/pgx/v5/pgconn" 12 | "github.com/jackc/pgx/v5/pgtype" 13 | ) 14 | 15 | var ( 16 | ErrorPublicationIsNotExists = goerrors.New("publication is not exists") 17 | ) 18 | 19 | var typeMap = pgtype.NewMap() 20 | 21 | type Publication struct { 22 | conn pq.Connection 23 | cfg Config 24 | } 25 | 26 | func New(cfg Config, conn pq.Connection) *Publication { 27 | return &Publication{cfg: cfg, conn: conn} 28 | } 29 | 30 | func (c *Publication) Create(ctx context.Context) (*Config, error) { 31 | info, err := c.Info(ctx) 32 | if err != nil { 33 | if !goerrors.Is(err, ErrorPublicationIsNotExists) || !c.cfg.CreateIfNotExists { 34 | return nil, errors.Wrap(err, "publication info") 35 | } 36 | } else { 37 | logger.Warn("publication already exists") 38 | return info, nil 39 | } 40 | 41 | resultReader := c.conn.Exec(ctx, c.cfg.createQuery()) 42 | _, err = resultReader.ReadAll() 43 | if err != nil { 44 | return nil, errors.Wrap(err, "publication create result") 45 | } 46 | 47 | if err = resultReader.Close(); err != nil { 48 | return nil, errors.Wrap(err, "publication create result reader close") 49 | } 50 | 51 | logger.Info("publication created", "name", c.cfg.Name) 52 | 53 | return &c.cfg, nil 54 | } 55 | 56 | func (c *Publication) Info(ctx context.Context) (*Config, error) { 57 | resultReader := c.conn.Exec(ctx, c.cfg.infoQuery()) 58 | results, err := resultReader.ReadAll() 59 | if err != nil { 60 | var v *pgconn.PgError 61 | if goerrors.As(err, &v) && v.Code == "42703" { 62 | return nil, ErrorPublicationIsNotExists 63 | } 64 | return nil, errors.Wrap(err, "publication info result") 65 | } 66 | 67 | if len(results) == 0 || results[0].CommandTag.String() == "SELECT 0" { 68 | return nil, ErrorPublicationIsNotExists 69 | } 70 | 71 | if err = resultReader.Close(); err != nil { 72 | return nil, errors.Wrap(err, "publication info result reader close") 73 | } 74 | 75 | publicationInfo, err := decodePublicationInfoResult(results[0]) 76 | if err != nil { 77 | return nil, errors.Wrap(err, "publication info result decode") 78 | } 79 | 80 | return publicationInfo, nil 81 | } 82 | 83 | func decodePublicationInfoResult(result *pgconn.Result) (*Config, error) { 84 | var publicationConfig Config 85 | var tables []string 86 | 87 | for i, fd := range result.FieldDescriptions { 88 | v, err := decodeTextColumnData(result.Rows[0][i], fd.DataTypeOID) 89 | if err != nil { 90 | return nil, err 91 | } 92 | 93 | if v == nil { 94 | continue 95 | } 96 | 97 | switch fd.Name { 98 | case "pubname": 99 | publicationConfig.Name = v.(string) 100 | case "pubinsert": 101 | if v.(bool) { 102 | publicationConfig.Operations = append(publicationConfig.Operations, "INSERT") 103 | } 104 | case "pubupdate": 105 | if v.(bool) { 106 | publicationConfig.Operations = append(publicationConfig.Operations, "UPDATE") 107 | } 108 | case "pubdelete": 109 | if v.(bool) { 110 | publicationConfig.Operations = append(publicationConfig.Operations, "DELETE") 111 | } 112 | case "pubtruncate": 113 | if v.(bool) { 114 | publicationConfig.Operations = append(publicationConfig.Operations, "TRUNCATE") 115 | } 116 | case "pubtables": 117 | for _, val := range v.([]any) { 118 | tables = append(tables, val.(string)) 119 | } 120 | } 121 | } 122 | 123 | for _, tableName := range tables { 124 | st := strings.Split(tableName, ".") 125 | publicationConfig.Tables = append(publicationConfig.Tables, Table{ 126 | Name: st[1], 127 | Schema: st[0], 128 | }) 129 | } 130 | 131 | return &publicationConfig, nil 132 | } 133 | 134 | func decodeTextColumnData(data []byte, dataType uint32) (interface{}, error) { 135 | if dt, ok := typeMap.TypeForOID(dataType); ok { 136 | return dt.Codec.DecodeValue(typeMap, dataType, pgtype.TextFormatCode, data) 137 | } 138 | return string(data), nil 139 | } 140 | -------------------------------------------------------------------------------- /integration_test/snapshot_helpers_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "testing" 8 | 9 | "github.com/Trendyol/go-pq-cdc/pq" 10 | "github.com/jackc/pgx/v5/pgconn" 11 | ) 12 | 13 | // Helper functions for snapshot tests 14 | 15 | func createTestTable(ctx context.Context, conn pq.Connection, tableName string) error { 16 | query := fmt.Sprintf(` 17 | DROP TABLE IF EXISTS %s; 18 | CREATE TABLE %s ( 19 | id INT PRIMARY KEY, 20 | name TEXT NOT NULL, 21 | age INT NOT NULL 22 | ); 23 | `, tableName, tableName) 24 | return pgExec(ctx, conn, query) 25 | } 26 | 27 | func createTextPrimaryKeyTable(ctx context.Context, conn pq.Connection, tableName string) error { 28 | query := fmt.Sprintf(` 29 | DROP TABLE IF EXISTS %s; 30 | CREATE TABLE %s ( 31 | id TEXT PRIMARY KEY, 32 | payload TEXT NOT NULL 33 | ); 34 | `, tableName, tableName) 35 | return pgExec(ctx, conn, query) 36 | } 37 | 38 | func execQuery(ctx context.Context, conn pq.Connection, query string) ([]*pgconn.Result, error) { 39 | resultReader := conn.Exec(ctx, query) 40 | results, err := resultReader.ReadAll() 41 | if err != nil { 42 | return nil, err 43 | } 44 | if err = resultReader.Close(); err != nil { 45 | return nil, err 46 | } 47 | return results, nil 48 | } 49 | 50 | func cleanupSnapshotTest(t *testing.T, ctx context.Context, tableName string, slotName string, publicationName string) { 51 | // Open a fresh connection for cleanup 52 | conn, err := newPostgresConn() 53 | if err != nil { 54 | t.Logf("Warning: Failed to create cleanup connection: %v", err) 55 | return 56 | } 57 | defer conn.Close(ctx) 58 | 59 | // Drop test table 60 | query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName) 61 | if err := pgExec(ctx, conn, query); err != nil { 62 | t.Logf("Warning: Failed to drop table: %v", err) 63 | } 64 | 65 | // Clean metadata tables (if they exist) 66 | _ = pgExec(ctx, conn, fmt.Sprintf("DELETE FROM cdc_snapshot_chunks WHERE slot_name = '%s'", slotName)) 67 | _ = pgExec(ctx, conn, fmt.Sprintf("DELETE FROM cdc_snapshot_job WHERE slot_name = '%s'", slotName)) 68 | 69 | // Drop publication and slot 70 | _ = pgExec(ctx, conn, fmt.Sprintf("DROP PUBLICATION IF EXISTS %s", publicationName)) 71 | _ = pgExec(ctx, conn, fmt.Sprintf("SELECT pg_drop_replication_slot('%s') WHERE EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = '%s')", slotName, slotName)) 72 | 73 | t.Log("✅ Cleanup completed") 74 | } 75 | 76 | // Metric fetching helpers 77 | 78 | func fetchSnapshotInProgressMetric() (int, error) { 79 | m, err := fetchMetrics("go_pq_cdc_snapshot_in_progress") 80 | if err != nil { 81 | return 0, err 82 | } 83 | var val int 84 | fmt.Sscanf(m, "%d", &val) 85 | return val, nil 86 | } 87 | 88 | func fetchSnapshotTotalRowsMetric() (int, error) { 89 | m, err := fetchMetrics("go_pq_cdc_snapshot_total_rows") 90 | if err != nil { 91 | return 0, err 92 | } 93 | var val int 94 | fmt.Sscanf(m, "%d", &val) 95 | return val, nil 96 | } 97 | 98 | func fetchSnapshotTotalChunksMetric() (int, error) { 99 | m, err := fetchMetrics("go_pq_cdc_snapshot_total_chunks") 100 | if err != nil { 101 | return 0, err 102 | } 103 | var val int 104 | fmt.Sscanf(m, "%d", &val) 105 | return val, nil 106 | } 107 | 108 | func fetchSnapshotCompletedChunksMetric() (int, error) { 109 | m, err := fetchMetrics("go_pq_cdc_snapshot_completed_chunks") 110 | if err != nil { 111 | return 0, err 112 | } 113 | var val int 114 | fmt.Sscanf(m, "%d", &val) 115 | return val, nil 116 | } 117 | 118 | func fetchSnapshotTotalTablesMetric() (int, error) { 119 | m, err := fetchMetrics("go_pq_cdc_snapshot_total_tables") 120 | if err != nil { 121 | return 0, err 122 | } 123 | var val int 124 | fmt.Sscanf(m, "%d", &val) 125 | return val, nil 126 | } 127 | 128 | func countWalsendersForSlot(ctx context.Context, conn pq.Connection, slotName string) (int, error) { 129 | query := fmt.Sprintf( 130 | "SELECT COUNT(*) FROM pg_stat_activity WHERE backend_type = 'walsender' AND query LIKE '%%%s%%'", 131 | slotName, 132 | ) 133 | resultReader := conn.Exec(ctx, query) 134 | results, err := resultReader.ReadAll() 135 | if err != nil { 136 | return 0, err 137 | } 138 | if err = resultReader.Close(); err != nil { 139 | return 0, err 140 | } 141 | if len(results) == 0 || len(results[0].Rows) == 0 { 142 | return 0, nil 143 | } 144 | count, _ := strconv.Atoi(string(results[0].Rows[0][0])) 145 | return count, nil 146 | } 147 | -------------------------------------------------------------------------------- /integration_test/system_identity_full_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | cdc "github.com/Trendyol/go-pq-cdc" 7 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 8 | "github.com/Trendyol/go-pq-cdc/pq/publication" 9 | "github.com/Trendyol/go-pq-cdc/pq/replication" 10 | "github.com/stretchr/testify/assert" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | func TestReplicaIdentityDefault(t *testing.T) { 16 | ctx := context.Background() 17 | 18 | cdcCfg := Config 19 | cdcCfg.Slot.Name = "slot_test_replica_identity_default" 20 | cdcCfg.Publication.Tables[0].ReplicaIdentity = publication.ReplicaIdentityDefault 21 | 22 | postgresConn, err := newPostgresConn() 23 | if !assert.NoError(t, err) { 24 | t.FailNow() 25 | } 26 | 27 | if !assert.NoError(t, SetupTestDB(ctx, postgresConn, cdcCfg)) { 28 | t.FailNow() 29 | } 30 | 31 | messageCh := make(chan any, 500) 32 | handlerFunc := func(ctx *replication.ListenerContext) { 33 | switch msg := ctx.Message.(type) { 34 | case *format.Insert, *format.Delete, *format.Update: 35 | messageCh <- msg 36 | } 37 | _ = ctx.Ack() 38 | } 39 | 40 | connector, err := cdc.NewConnector(ctx, cdcCfg, handlerFunc) 41 | if !assert.NoError(t, err) { 42 | t.FailNow() 43 | } 44 | 45 | defer func() { 46 | connector.Close() 47 | assert.NoError(t, RestoreDB(ctx)) 48 | assert.NoError(t, postgresConn.Close(ctx)) 49 | }() 50 | 51 | go connector.Start(ctx) 52 | 53 | waitCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 54 | if !assert.NoError(t, connector.WaitUntilReady(waitCtx)) { 55 | t.FailNow() 56 | } 57 | cancel() 58 | 59 | t.Run("should return old value is nil when update message received", func(t *testing.T) { 60 | books := CreateBooks(10) 61 | for _, b := range books { 62 | err = pgExec(ctx, postgresConn, fmt.Sprintf("INSERT INTO books(id, name) VALUES(%d, '%s')", b.ID, b.Name)) 63 | assert.NoError(t, err) 64 | } 65 | 66 | for range 10 { 67 | <-messageCh 68 | } 69 | 70 | booksNew := CreateBooks(5) 71 | for i, b := range booksNew { 72 | b.ID = i + 1 73 | booksNew[i] = b 74 | err = pgExec(ctx, postgresConn, fmt.Sprintf("UPDATE books SET name = '%s' WHERE id = %d", b.Name, b.ID)) 75 | assert.NoError(t, err) 76 | } 77 | 78 | for i := range 5 { 79 | m := <-messageCh 80 | assert.Equal(t, booksNew[i].Map(), m.(*format.Update).NewDecoded) 81 | assert.Nil(t, m.(*format.Update).OldDecoded["id"]) 82 | } 83 | }) 84 | } 85 | 86 | func TestReplicaIdentityFull(t *testing.T) { 87 | ctx := context.Background() 88 | 89 | cdcCfg := Config 90 | cdcCfg.Slot.Name = "slot_test_replica_identity_full" 91 | cdcCfg.Publication.Tables[0].ReplicaIdentity = publication.ReplicaIdentityFull 92 | 93 | postgresConn, err := newPostgresConn() 94 | if !assert.NoError(t, err) { 95 | t.FailNow() 96 | } 97 | 98 | if !assert.NoError(t, SetupTestDB(ctx, postgresConn, cdcCfg)) { 99 | t.FailNow() 100 | } 101 | 102 | messageCh := make(chan any, 500) 103 | handlerFunc := func(ctx *replication.ListenerContext) { 104 | switch msg := ctx.Message.(type) { 105 | case *format.Insert, *format.Delete, *format.Update: 106 | messageCh <- msg 107 | } 108 | _ = ctx.Ack() 109 | } 110 | 111 | connector, err := cdc.NewConnector(ctx, cdcCfg, handlerFunc) 112 | if !assert.NoError(t, err) { 113 | t.FailNow() 114 | } 115 | 116 | t.Cleanup(func() { 117 | connector.Close() 118 | assert.NoError(t, RestoreDB(ctx)) 119 | assert.NoError(t, postgresConn.Close(ctx)) 120 | }) 121 | 122 | go connector.Start(ctx) 123 | 124 | waitCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 125 | if !assert.NoError(t, connector.WaitUntilReady(waitCtx)) { 126 | t.FailNow() 127 | } 128 | cancel() 129 | 130 | t.Run("should return new value and old value when update message received", func(t *testing.T) { 131 | books := CreateBooks(10) 132 | for _, b := range books { 133 | err = pgExec(ctx, postgresConn, fmt.Sprintf("INSERT INTO books(id, name) VALUES(%d, '%s')", b.ID, b.Name)) 134 | assert.NoError(t, err) 135 | } 136 | 137 | for range 10 { 138 | <-messageCh 139 | } 140 | 141 | booksNew := CreateBooks(5) 142 | for i, b := range booksNew { 143 | b.ID = i + 1 144 | booksNew[i] = b 145 | err = pgExec(ctx, postgresConn, fmt.Sprintf("UPDATE books SET name = '%s' WHERE id = %d", b.Name, b.ID)) 146 | assert.NoError(t, err) 147 | } 148 | 149 | for i := range 5 { 150 | m := <-messageCh 151 | assert.Equal(t, booksNew[i].Map(), m.(*format.Update).NewDecoded) 152 | assert.Equal(t, books[i].Map(), m.(*format.Update).OldDecoded) 153 | } 154 | }) 155 | } 156 | -------------------------------------------------------------------------------- /example/simple-with-heartbeat/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log/slog" 6 | "os" 7 | "time" 8 | 9 | cdc "github.com/Trendyol/go-pq-cdc" 10 | "github.com/Trendyol/go-pq-cdc/config" 11 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 12 | "github.com/Trendyol/go-pq-cdc/pq/publication" 13 | "github.com/Trendyol/go-pq-cdc/pq/replication" 14 | "github.com/Trendyol/go-pq-cdc/pq/slot" 15 | ) 16 | 17 | /* 18 | Simulation guide (low-traffic DB + high-traffic other DB) 19 | ========================================================= 20 | 21 | This example spins up a Postgres instance with two databases: 22 | - cdc_db : the database used by the CDC connector (low traffic) 23 | - high_db : a second database used only to generate WAL (high traffic) 24 | 25 | Step 1: Start Postgres 26 | ---------------------- 27 | 28 | cd example/simple-with-heartbeat 29 | docker compose up -d 30 | 31 | Step 2: Run the connector (first WITHOUT heartbeat) 32 | --------------------------------------------------- 33 | 34 | # In this file, Heartbeat.Enabled is initially set to false. 35 | go run . 36 | 37 | Step 3: Generate WAL in high_db (different database) 38 | ---------------------------------------------------- 39 | 40 | # New terminal: 41 | psql "postgres://cdc_user:cdc_pass@127.0.0.1:5433/high_db" 42 | 43 | DO $$ 44 | BEGIN 45 | FOR i IN 1..50000 LOOP 46 | INSERT INTO public.hightraffic(value) VALUES (md5(random()::text)); 47 | END LOOP; 48 | END; 49 | $$; 50 | 51 | Step 4: Observe slot vs global WAL on cdc_db 52 | -------------------------------------------- 53 | 54 | psql "postgres://cdc_user:cdc_pass@127.0.0.1:5433/cdc_db" 55 | 56 | SELECT slot_name, restart_lsn, confirmed_flush_lsn 57 | FROM pg_replication_slots 58 | WHERE slot_name = 'cdc_slot'; 59 | 60 | SELECT pg_current_wal_lsn(); 61 | 62 | Because cdc_db itself is almost idle and heartbeat is disabled: 63 | - pg_current_wal_lsn() will move forward due to high_db traffic 64 | - restart_lsn / confirmed_flush_lsn for cdc_slot may lag behind 65 | 66 | Step 5: Enable heartbeat and rerun 67 | ---------------------------------- 68 | 69 | - Stop the Go process. 70 | - Set Heartbeat.Enabled = true below. 71 | - go run . 72 | 73 | Now, even if cdc_db has low application traffic, the heartbeat will insert 74 | into public.test_heartbeat_table inside cdc_db at a fixed interval, producing 75 | commits that advance confirmed_flush_lsn and restart_lsn. 76 | */ 77 | func main() { 78 | ctx := context.Background() 79 | cfg := config.Config{ 80 | Host: "127.0.0.1", 81 | Port: 5433, 82 | Username: "cdc_user", 83 | Password: "cdc_pass", 84 | Database: "cdc_db", 85 | DebugMode: false, 86 | Publication: publication.Config{ 87 | CreateIfNotExists: true, 88 | Name: "cdc_publication", 89 | Operations: publication.Operations{ 90 | publication.OperationInsert, 91 | publication.OperationDelete, 92 | publication.OperationTruncate, 93 | publication.OperationUpdate, 94 | }, 95 | Tables: publication.Tables{ 96 | { 97 | Name: "users", 98 | ReplicaIdentity: publication.ReplicaIdentityDefault, 99 | Schema: "public", 100 | }, 101 | { 102 | Name: "test_heartbeat_table", 103 | ReplicaIdentity: publication.ReplicaIdentityDefault, 104 | Schema: "public", 105 | }, 106 | }, 107 | }, 108 | Slot: slot.Config{ 109 | CreateIfNotExists: true, 110 | Name: "cdc_slot", 111 | SlotActivityCheckerInterval: 3000, 112 | }, 113 | Heartbeat: config.HeartbeatConfig{ 114 | // For the first run of the simulation, leave this as false. 115 | // Then set to true and rerun to see the effect of heartbeat. 116 | Enabled: false, 117 | Query: `INSERT INTO public.test_heartbeat_table(txt) VALUES ('hb')`, 118 | Interval: 5 * time.Second, 119 | }, 120 | Metric: config.MetricConfig{ 121 | Port: 8081, 122 | }, 123 | Logger: config.LoggerConfig{ 124 | LogLevel: slog.LevelInfo, 125 | }, 126 | } 127 | 128 | connector, err := cdc.NewConnector(ctx, cfg, Handler) 129 | if err != nil { 130 | slog.Error("new connector", "error", err) 131 | os.Exit(1) 132 | } 133 | 134 | defer connector.Close() 135 | connector.Start(ctx) 136 | } 137 | 138 | func Handler(ctx *replication.ListenerContext) { 139 | switch msg := ctx.Message.(type) { 140 | case *format.Insert: 141 | slog.Info("insert message received", "new", msg.Decoded) 142 | case *format.Delete: 143 | slog.Info("delete message received", "old", msg.OldDecoded) 144 | case *format.Update: 145 | slog.Info("update message received", "new", msg.NewDecoded, "old", msg.OldDecoded) 146 | } 147 | 148 | if err := ctx.Ack(); err != nil { 149 | slog.Error("ack", "error", err) 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /integration_test/heartbeat_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | cdc "github.com/Trendyol/go-pq-cdc" 10 | "github.com/Trendyol/go-pq-cdc/config" 11 | "github.com/Trendyol/go-pq-cdc/pq" 12 | "github.com/Trendyol/go-pq-cdc/pq/publication" 13 | "github.com/Trendyol/go-pq-cdc/pq/replication" 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | func TestHeartbeatAdvancesLSN(t *testing.T) { 18 | t.Helper() 19 | 20 | ctx := context.Background() 21 | 22 | // Use base config but customize slot / publication / heartbeat for this test 23 | cdcCfg := Config 24 | cdcCfg.Slot.Name = "slot_test_heartbeat_lsn" 25 | 26 | postgresConn, err := newPostgresConn() 27 | if !assert.NoError(t, err) { 28 | t.FailNow() 29 | } 30 | 31 | // Ensure base DB objects (books table, drop old publication) 32 | if !assert.NoError(t, SetupTestDB(ctx, postgresConn, cdcCfg)) { 33 | t.FailNow() 34 | } 35 | 36 | // Create heartbeat table in cdc database 37 | createHeartbeatTableSQL := ` 38 | CREATE TABLE IF NOT EXISTS public.heartbeat_events ( 39 | id bigserial PRIMARY KEY, 40 | txt text, 41 | ts timestamptz DEFAULT now() 42 | );` 43 | if !assert.NoError(t, pgExec(ctx, postgresConn, createHeartbeatTableSQL)) { 44 | t.FailNow() 45 | } 46 | 47 | // Extend publication with heartbeat table so that heartbeat changes are part of CDC stream 48 | cdcCfg.Publication.Tables = append(cdcCfg.Publication.Tables, 49 | publication.Table{ 50 | Name: "heartbeat_events", 51 | Schema: "public", 52 | ReplicaIdentity: publication.ReplicaIdentityFull, 53 | }, 54 | ) 55 | 56 | // Enable heartbeat to periodically insert into heartbeat_events 57 | cdcCfg.Heartbeat = config.HeartbeatConfig{ 58 | Enabled: true, 59 | Query: `INSERT INTO public.heartbeat_events(txt) VALUES ('hb')`, 60 | Interval: 2 * time.Second, 61 | } 62 | 63 | messageCh := make(chan any, 10) 64 | handlerFunc := func(ctx *replication.ListenerContext) { 65 | // We don't assert on specific heartbeat events here; just make sure ACKs flow. 66 | messageCh <- ctx.Message 67 | _ = ctx.Ack() 68 | } 69 | 70 | connector, err := cdc.NewConnector(ctx, cdcCfg, handlerFunc) 71 | if !assert.NoError(t, err) { 72 | t.FailNow() 73 | } 74 | 75 | t.Cleanup(func() { 76 | connector.Close() 77 | assert.NoError(t, RestoreDB(ctx)) 78 | assert.NoError(t, postgresConn.Close(ctx)) 79 | }) 80 | 81 | go connector.Start(ctx) 82 | 83 | waitCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 84 | if !assert.NoError(t, connector.WaitUntilReady(waitCtx)) { 85 | cancel() 86 | t.FailNow() 87 | } 88 | cancel() 89 | 90 | // Capture initial LSNs for the test slot 91 | initialRestart, initialConfirmed, err := readSlotLSNs(ctx, postgresConn, cdcCfg.Slot.Name) 92 | if !assert.NoError(t, err) { 93 | t.FailNow() 94 | } 95 | 96 | // Wait long enough for a few heartbeat cycles 97 | time.Sleep(7 * time.Second) 98 | 99 | finalRestart, finalConfirmed, err := readSlotLSNs(ctx, postgresConn, cdcCfg.Slot.Name) 100 | if !assert.NoError(t, err) { 101 | t.FailNow() 102 | } 103 | 104 | // Heartbeat should cause at least confirmed_flush_lsn to move forward. 105 | assert.NotEmpty(t, initialConfirmed) 106 | assert.NotEmpty(t, finalConfirmed) 107 | assert.NotEqualf(t, initialConfirmed, finalConfirmed, 108 | "expected confirmed_flush_lsn to advance due to heartbeat, got initial=%s final=%s", 109 | initialConfirmed, finalConfirmed, 110 | ) 111 | 112 | // restart_lsn may move less frequently, but for practical purposes 113 | // it should also advance when only heartbeat is producing changes. 114 | assert.NotEmpty(t, initialRestart) 115 | assert.NotEmpty(t, finalRestart) 116 | } 117 | 118 | // readSlotLSNs fetches restart_lsn and confirmed_flush_lsn for a given slot 119 | // as textual LSN representations. 120 | func readSlotLSNs(ctx context.Context, conn pq.Connection, slotName string) (restartLSN string, confirmedLSN string, err error) { 121 | sql := fmt.Sprintf( 122 | "SELECT restart_lsn, confirmed_flush_lsn FROM pg_replication_slots WHERE slot_name = '%s';", 123 | slotName, 124 | ) 125 | 126 | rr := conn.Exec(ctx, sql) 127 | results, err := rr.ReadAll() 128 | if err != nil { 129 | return "", "", err 130 | } 131 | if err = rr.Close(); err != nil { 132 | return "", "", err 133 | } 134 | 135 | if len(results) == 0 || len(results[0].Rows) == 0 { 136 | return "", "", fmt.Errorf("slot %s not found", slotName) 137 | } 138 | 139 | row := results[0].Rows[0] 140 | if len(row) < 2 { 141 | return "", "", fmt.Errorf("unexpected column count for slot %s", slotName) 142 | } 143 | 144 | // Values are textual LSNs (e.g. "0/123ABC"), NULL becomes empty string. 145 | restartLSN = string(row[0]) 146 | confirmedLSN = string(row[1]) 147 | return restartLSN, confirmedLSN, nil 148 | } 149 | -------------------------------------------------------------------------------- /pq/snapshot/job.go: -------------------------------------------------------------------------------- 1 | package snapshot 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/Trendyol/go-pq-cdc/pq" 9 | "github.com/go-playground/errors" 10 | ) 11 | 12 | // ChunkStatus represents the status of a chunk 13 | type ChunkStatus string 14 | 15 | const ( 16 | ChunkStatusPending ChunkStatus = "pending" 17 | ChunkStatusInProgress ChunkStatus = "in_progress" 18 | ChunkStatusCompleted ChunkStatus = "completed" 19 | ) 20 | 21 | // Chunk represents a unit of work for snapshot processing 22 | type Chunk struct { 23 | ClaimedAt *time.Time 24 | HeartbeatAt *time.Time 25 | CompletedAt *time.Time 26 | RangeEnd *int64 27 | RangeStart *int64 28 | Status ChunkStatus 29 | TableName string 30 | ClaimedBy string 31 | TableSchema string 32 | SlotName string 33 | ID int64 34 | ChunkIndex int 35 | ChunkStart int64 36 | ChunkSize int64 37 | } 38 | 39 | func (c *Chunk) hasRangeBounds() bool { 40 | return c.RangeStart != nil && c.RangeEnd != nil 41 | } 42 | 43 | // Job represents the overall snapshot job metadata 44 | type Job struct { 45 | StartedAt time.Time 46 | SlotName string 47 | SnapshotID string 48 | SnapshotLSN pq.LSN 49 | TotalChunks int 50 | CompletedChunks int 51 | Completed bool 52 | } 53 | 54 | const ( 55 | jobTableName = "cdc_snapshot_job" 56 | chunksTableName = "cdc_snapshot_chunks" 57 | ) 58 | 59 | // loadJob loads the job metadata 60 | func (s *Snapshotter) loadJob(ctx context.Context, slotName string) (*Job, error) { 61 | var job *Job 62 | 63 | err := s.retryDBOperation(ctx, func() error { 64 | query := fmt.Sprintf(` 65 | SELECT slot_name, snapshot_id, snapshot_lsn, started_at, 66 | completed, total_chunks, completed_chunks 67 | FROM %s WHERE slot_name = '%s' 68 | `, jobTableName, slotName) 69 | 70 | results, err := s.execQuery(ctx, s.metadataConn, query) 71 | if err != nil { 72 | return errors.Wrap(err, "load job") 73 | } 74 | 75 | if len(results) == 0 || len(results[0].Rows) == 0 { 76 | job = nil 77 | return nil // Not found (not an error) 78 | } 79 | 80 | row := results[0].Rows[0] 81 | if len(row) < 7 { 82 | return errors.New("invalid job row") 83 | } 84 | 85 | job = &Job{ 86 | SlotName: string(row[0]), 87 | SnapshotID: string(row[1]), 88 | } 89 | 90 | // Parse LSN 91 | job.SnapshotLSN, err = pq.ParseLSN(string(row[2])) 92 | if err != nil { 93 | return errors.Wrap(err, "parse snapshot LSN") 94 | } 95 | 96 | // Parse timestamp 97 | job.StartedAt, err = parseTimestamp(string(row[3])) 98 | if err != nil { 99 | return errors.Wrap(err, "parse started_at timestamp") 100 | } 101 | 102 | job.Completed = string(row[4]) == "t" || string(row[4]) == "true" 103 | if _, err := fmt.Sscanf(string(row[5]), "%d", &job.TotalChunks); err != nil { 104 | return errors.Wrap(err, "parse total chunks") 105 | } 106 | if _, err := fmt.Sscanf(string(row[6]), "%d", &job.CompletedChunks); err != nil { 107 | return errors.Wrap(err, "parse completed chunks") 108 | } 109 | 110 | return nil 111 | }) 112 | 113 | return job, err 114 | } 115 | 116 | // LoadJob is the public API for connector 117 | func (s *Snapshotter) LoadJob(ctx context.Context, slotName string) (*Job, error) { 118 | return s.loadJob(ctx, slotName) 119 | } 120 | 121 | // checkJobCompleted checks if all chunks are completed 122 | func (s *Snapshotter) checkJobCompleted(ctx context.Context, slotName string) (bool, error) { 123 | var isCompleted bool 124 | 125 | err := s.retryDBOperation(ctx, func() error { 126 | query := fmt.Sprintf(` 127 | SELECT 128 | COUNT(*) as total, 129 | COUNT(*) FILTER (WHERE status = 'completed') as completed 130 | FROM %s 131 | WHERE slot_name = '%s' 132 | `, chunksTableName, slotName) 133 | 134 | results, err := s.execQuery(ctx, s.metadataConn, query) 135 | if err != nil { 136 | return errors.Wrap(err, "check job completed") 137 | } 138 | 139 | if len(results) == 0 || len(results[0].Rows) == 0 { 140 | isCompleted = false 141 | return nil 142 | } 143 | 144 | row := results[0].Rows[0] 145 | var total, completed int 146 | if _, err := fmt.Sscanf(string(row[0]), "%d", &total); err != nil { 147 | return errors.Wrap(err, "parse total count") 148 | } 149 | if _, err := fmt.Sscanf(string(row[1]), "%d", &completed); err != nil { 150 | return errors.Wrap(err, "parse completed count") 151 | } 152 | 153 | isCompleted = total > 0 && total == completed 154 | return nil 155 | }) 156 | 157 | return isCompleted, err 158 | } 159 | 160 | // Helper functions 161 | 162 | func parseTimestamp(s string) (time.Time, error) { 163 | formats := []string{ 164 | postgresTimestampFormatMicros, 165 | postgresTimestampFormat, 166 | } 167 | 168 | for _, format := range formats { 169 | if t, err := time.Parse(format, s); err == nil { 170 | return t, nil 171 | } 172 | } 173 | 174 | return time.Time{}, fmt.Errorf("unable to parse timestamp: %s", s) 175 | } 176 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contribution Guidelines 2 | 3 | Thank you for your interest in go-pq-cdc! 4 | 5 | This project welcomes contributions and suggestions. Most contributions require you to signoff on your commits. Please 6 | follow the instructions provided below. 7 | 8 | Contributions come in many forms: submitting issues, writing code, participating in discussions and community calls. 9 | 10 | This document provides the guidelines for how to contribute to the project. 11 | 12 | ### Issues 13 | 14 | This section describes the guidelines for submitting issues 15 | 16 | **Issue Types** 17 | 18 | There are 2 types of issues: 19 | 20 | - Issue/Bug: You've found a bug with the code, and want to report it, or create an issue to track the bug. 21 | - Issue/Feature: You have something on your mind, which requires input form others in a discussion, before it eventually 22 | manifests as a proposal. 23 | 24 | ### Before You File 25 | 26 | Before you file an issue, make sure you've checked the following: 27 | 28 | 1. Check for existing issues 29 | Before you create a new issue, please do a search in open issues to see if the issue or feature request has already 30 | been filed. 31 | 32 | If you find your issue already exists, make relevant comments and add your reaction. Use a reaction: 33 | 👍 up-vote 34 | 👎 down-vote 35 | 36 | 2. For bugs 37 | Check it's not an environment issue. For example, if your configurations correct or network connections is alive. 38 | 39 | ### Contributing to go-pq-cdc 40 | 41 | Pull Requests 42 | All contributions come through pull requests. To submit a proposed change, we recommend following this workflow: 43 | 44 | - Make sure there's an issue (bug or feature) raised, which sets the expectations for the contribution you are about to 45 | make. 46 | - Fork the relevant repo and create a new branch 47 | - Create your change 48 | - Code changes require tests 49 | - Update relevant documentation for the change 50 | - Commit sign-off and open a PR 51 | - Wait for the CI process to finish and make sure all checks are green 52 | - A maintainer of the project will be assigned, and you can expect a review within a few days 53 | 54 | ### Use work-in-progress PRs for early feedback 55 | 56 | A good way to communicate before investing too much time is to create a "Work-in-progress" PR and share it with your 57 | reviewers. The standard way of doing this is to add a "[WIP]" prefix in your PR's title and assign the do-not-merge 58 | label. This will let people looking at your PR know that it is not well baked yet. 59 | 60 | ### Developer Certificate of Origin: Signing your work 61 | 62 | **Every commit needs to be signed** 63 | 64 | The Developer Certificate of Origin (DCO) is a lightweight way for contributors to certify that they wrote or otherwise 65 | have the right to submit the code they are contributing to the project. Here is the full text of the DCO, reformatted 66 | for readability: 67 | 68 | By making a contribution to this project, I certify that: 69 | 70 | (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or 71 | 72 | (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or 73 | 74 | (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. 75 | 76 | (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. 77 | 78 | Contributors sign-off that they adhere to these requirements by adding a Signed-off-by line to commit messages. 79 | 80 | This is my commit message 81 | 82 | Signed-off-by: Random X Developer 83 | 84 | Git even has a -s command line option to append this automatically to your commit message: 85 | 86 | $ git commit -s -m 'This is my commit message' 87 | 88 | Each Pull Request is checked whether or not commits in a Pull Request do contain a valid Signed-off-by line. 89 | 90 | I didn't sign my commit, now what?! 91 | No worries - You can easily replay your changes, sign them and force push them! 92 | 93 | git checkout 94 | git commit --amend --no-edit --signoff 95 | git push --force-with-lease 96 | 97 | ### Use of Third-party code 98 | 99 | - Third-party code must include licenses. 100 | 101 | **Thank You!** - Your contributions to open source, large or small, make projects like this possible. Thank you for 102 | taking the time to contribute. 103 | 104 | ### Code of Conduct 105 | 106 | This project has adopted the Contributor Covenant Code of Conduct -------------------------------------------------------------------------------- /example/postgresql/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | cdc "github.com/Trendyol/go-pq-cdc" 7 | "github.com/Trendyol/go-pq-cdc/config" 8 | "github.com/Trendyol/go-pq-cdc/pq/message/format" 9 | "github.com/Trendyol/go-pq-cdc/pq/publication" 10 | "github.com/Trendyol/go-pq-cdc/pq/replication" 11 | "github.com/Trendyol/go-pq-cdc/pq/slot" 12 | "github.com/jackc/pgx/v5" 13 | "github.com/jackc/pgx/v5/pgxpool" 14 | "log/slog" 15 | "os" 16 | "time" 17 | ) 18 | 19 | /* 20 | psql "postgres://cdc_user:cdc_pass@127.0.0.1:5433/cdc_db" 21 | 22 | CREATE TABLE users ( 23 | user_id integer PRIMARY KEY, 24 | name text NOT NULL 25 | ); 26 | */ 27 | 28 | /* 29 | psql "postgres://cdc_user:cdc_pass@127.0.0.1/cdc_db?replication=database" 30 | 31 | CREATE TABLE users ( 32 | id serial PRIMARY KEY, 33 | name text NOT NULL, 34 | created_on timestamptz 35 | ); 36 | 37 | INSERT INTO users (name) 38 | SELECT 39 | 'Oyleli' || i 40 | FROM generate_series(1, 1000000) AS i; 41 | */ 42 | 43 | type Message struct { 44 | Query string 45 | Args []any 46 | Ack func() error 47 | } 48 | 49 | var ( 50 | UpsertQuery = "INSERT INTO users (user_id, name) VALUES ($1, $2) ON CONFLICT (user_id) DO UPDATE SET name = excluded.name;" 51 | DeleteQuery = "DELETE FROM users WHERE user_id = $1;" 52 | ) 53 | 54 | func main() { 55 | ctx := context.Background() 56 | pool, err := pgxpool.New(ctx, "postgres://cdc_user:cdc_pass@127.0.0.1:5433/cdc_db") 57 | if err != nil { 58 | slog.Error("new pool", "error", err) 59 | os.Exit(1) 60 | } 61 | 62 | messages := make(chan Message, 10000) 63 | go Produce(ctx, pool, messages) 64 | 65 | cfg := config.Config{ 66 | Host: "127.0.0.1", 67 | Username: "cdc_user", 68 | Password: "cdc_pass", 69 | Database: "cdc_db", 70 | Publication: publication.Config{ 71 | CreateIfNotExists: true, 72 | Name: "cdc_publication", 73 | Operations: publication.Operations{ 74 | publication.OperationInsert, 75 | publication.OperationDelete, 76 | publication.OperationTruncate, 77 | publication.OperationUpdate, 78 | }, 79 | Tables: publication.Tables{publication.Table{ 80 | Name: "users", 81 | ReplicaIdentity: publication.ReplicaIdentityDefault, 82 | Schema: "public", 83 | }}, 84 | }, 85 | Slot: slot.Config{ 86 | CreateIfNotExists: true, 87 | Name: "cdc_slot", 88 | SlotActivityCheckerInterval: 3000, 89 | }, 90 | Metric: config.MetricConfig{ 91 | Port: 8081, 92 | }, 93 | Logger: config.LoggerConfig{ 94 | LogLevel: slog.LevelInfo, 95 | }, 96 | } 97 | 98 | connector, err := cdc.NewConnector(ctx, cfg, FilteredMapper(messages)) 99 | if err != nil { 100 | slog.Error("new connector", "error", err) 101 | os.Exit(1) 102 | } 103 | 104 | connector.Start(ctx) 105 | } 106 | 107 | func FilteredMapper(messages chan Message) replication.ListenerFunc { 108 | return func(ctx *replication.ListenerContext) { 109 | switch msg := ctx.Message.(type) { 110 | case *format.Insert: 111 | messages <- Message{ 112 | Query: UpsertQuery, 113 | Args: []any{msg.Decoded["id"].(int32), msg.Decoded["name"].(string)}, 114 | Ack: ctx.Ack, 115 | } 116 | case *format.Delete: 117 | messages <- Message{ 118 | Query: DeleteQuery, 119 | Args: []any{msg.OldDecoded["id"].(int32)}, 120 | Ack: ctx.Ack, 121 | } 122 | case *format.Update: 123 | messages <- Message{ 124 | Query: UpsertQuery, 125 | Args: []any{msg.NewDecoded["id"].(int32), msg.NewDecoded["name"].(string)}, 126 | Ack: ctx.Ack, 127 | } 128 | } 129 | } 130 | } 131 | 132 | func Produce(ctx context.Context, w *pgxpool.Pool, messages <-chan Message) { 133 | var lastAck func() error 134 | counter := 0 135 | bulkSize := 10000 136 | 137 | queue := make([]*pgx.QueuedQuery, bulkSize) 138 | 139 | for { 140 | select { 141 | case event := <-messages: 142 | lastAck = event.Ack 143 | 144 | queue[counter] = &pgx.QueuedQuery{SQL: event.Query, Arguments: event.Args} 145 | counter++ 146 | if counter == bulkSize { 147 | batchResults := w.SendBatch(ctx, &pgx.Batch{QueuedQueries: queue}) 148 | err := Exec(batchResults, counter) 149 | if err != nil { 150 | slog.Error("batch results", "error", err) 151 | continue 152 | } 153 | slog.Info("postgresql write", "count", counter) 154 | counter = 0 155 | if err = event.Ack(); err != nil { 156 | slog.Error("ack", "error", err) 157 | } 158 | } 159 | 160 | case <-time.After(time.Millisecond): 161 | if counter > 0 { 162 | batchResults := w.SendBatch(ctx, &pgx.Batch{QueuedQueries: queue[:counter]}) 163 | err := Exec(batchResults, counter) 164 | if err != nil { 165 | slog.Error("batch results", "error", err) 166 | continue 167 | } 168 | slog.Info("postgresql write", "count", counter) 169 | counter = 0 170 | if err = lastAck(); err != nil { 171 | slog.Error("ack", "error", err) 172 | } 173 | } 174 | } 175 | } 176 | } 177 | 178 | func Exec(br pgx.BatchResults, sqlCount int) error { 179 | defer br.Close() 180 | var batchErr error 181 | for t := 0; t < sqlCount; t++ { 182 | _, err := br.Exec() 183 | if err != nil { 184 | batchErr = errors.Join(batchErr, err) 185 | } 186 | } 187 | return batchErr 188 | } 189 | -------------------------------------------------------------------------------- /example/postgresql/go.sum: -------------------------------------------------------------------------------- 1 | github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= 2 | github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= 3 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 4 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 5 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 6 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/go-playground/errors v3.3.0+incompatible h1:w7qP6bdFXNmI86aV8VEfhXrGxoQWYHc/OX4Muw4FgW0= 11 | github.com/go-playground/errors v3.3.0+incompatible/go.mod h1:n+RcthKmtLxDczVHKkhqiUSOGtTjvRl+HB4Gga0vWSI= 12 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 13 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 14 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 15 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 16 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= 17 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 18 | github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= 19 | github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= 20 | github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= 21 | github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= 22 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 23 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 24 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 25 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 26 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 27 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 28 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 29 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 30 | github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= 31 | github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= 32 | github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= 33 | github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= 34 | github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= 35 | github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= 36 | github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= 37 | github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= 38 | github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= 39 | github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 40 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 41 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 42 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 43 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 44 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 45 | golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= 46 | golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= 47 | golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= 48 | golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 49 | golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= 50 | golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 51 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 52 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 53 | google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= 54 | google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 55 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 56 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 57 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 58 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 59 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 60 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 61 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 62 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 63 | -------------------------------------------------------------------------------- /example/simple-file-config/go.sum: -------------------------------------------------------------------------------- 1 | github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= 2 | github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= 3 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 4 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 5 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 6 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/go-playground/errors v3.3.0+incompatible h1:w7qP6bdFXNmI86aV8VEfhXrGxoQWYHc/OX4Muw4FgW0= 11 | github.com/go-playground/errors v3.3.0+incompatible/go.mod h1:n+RcthKmtLxDczVHKkhqiUSOGtTjvRl+HB4Gga0vWSI= 12 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 13 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 14 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 15 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 16 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= 17 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 18 | github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= 19 | github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= 20 | github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= 21 | github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= 22 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 23 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 24 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 25 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 26 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 27 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 28 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 29 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 30 | github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= 31 | github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= 32 | github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= 33 | github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= 34 | github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= 35 | github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= 36 | github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= 37 | github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= 38 | github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= 39 | github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 40 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 41 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 42 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 43 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 44 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 45 | golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= 46 | golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= 47 | golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= 48 | golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 49 | golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= 50 | golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 51 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 52 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 53 | google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= 54 | google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 55 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 56 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 57 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 58 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 59 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 60 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 61 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 62 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 63 | -------------------------------------------------------------------------------- /example/snapshot-with-scaling/go.sum: -------------------------------------------------------------------------------- 1 | github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= 2 | github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= 3 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 4 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 5 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 6 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/go-playground/errors v3.3.0+incompatible h1:w7qP6bdFXNmI86aV8VEfhXrGxoQWYHc/OX4Muw4FgW0= 11 | github.com/go-playground/errors v3.3.0+incompatible/go.mod h1:n+RcthKmtLxDczVHKkhqiUSOGtTjvRl+HB4Gga0vWSI= 12 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 13 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 14 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 15 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 16 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= 17 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 18 | github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= 19 | github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= 20 | github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= 21 | github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= 22 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 23 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 24 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 25 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 26 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 27 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 28 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 29 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 30 | github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= 31 | github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= 32 | github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= 33 | github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= 34 | github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= 35 | github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= 36 | github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= 37 | github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= 38 | github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= 39 | github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 40 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 41 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 42 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 43 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 44 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 45 | golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= 46 | golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= 47 | golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= 48 | golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 49 | golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= 50 | golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 51 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 52 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 53 | google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= 54 | google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 55 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 56 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 57 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 58 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 59 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 60 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 61 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 62 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 63 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= 2 | github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= 3 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 4 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 5 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 6 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 7 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/go-playground/errors v3.3.0+incompatible h1:w7qP6bdFXNmI86aV8VEfhXrGxoQWYHc/OX4Muw4FgW0= 12 | github.com/go-playground/errors v3.3.0+incompatible/go.mod h1:n+RcthKmtLxDczVHKkhqiUSOGtTjvRl+HB4Gga0vWSI= 13 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 14 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 15 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 16 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 17 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= 18 | github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 19 | github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= 20 | github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= 21 | github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= 22 | github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= 23 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 24 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 25 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 26 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 27 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 28 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 29 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 30 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 31 | github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= 32 | github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= 33 | github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= 34 | github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= 35 | github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= 36 | github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= 37 | github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= 38 | github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= 39 | github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= 40 | github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 41 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 42 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 43 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 44 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 45 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 46 | golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= 47 | golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= 48 | golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= 49 | golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 50 | golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= 51 | golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 52 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 53 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 54 | google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= 55 | google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 56 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 57 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 58 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 59 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 60 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 61 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 62 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 63 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 64 | -------------------------------------------------------------------------------- /benchmark/benchmark_initial/SCALING_GUIDE.md: -------------------------------------------------------------------------------- 1 | # Go-PQ-CDC-Kafka Scaling Guide 2 | 3 | This guide explains the necessary steps to run multiple instances of the `go-pq-cdc-kafka` service. 4 | 5 | ## Changes Made 6 | 7 | ### 1. Docker Compose Changes 8 | - ✅ `container_name` removed (Docker Compose will automatically name each instance) 9 | - ✅ Port mapping replaced with `expose` (to prevent port conflicts) 10 | - ✅ Metrics are only exposed on the internal network 11 | 12 | ### 2. Prometheus Configuration 13 | - ✅ DNS service discovery added 14 | - ✅ All scaled instances will be automatically scraped 15 | 16 | ## Usage 17 | 18 | ### Starting Multiple Instances 19 | 20 | **To start with 3 instances:** 21 | ```bash 22 | cd benchmark/benchmark_initial 23 | docker-compose up --scale go-pq-cdc-kafka=3 -d 24 | ``` 25 | 26 | **To start with 5 instances:** 27 | ```bash 28 | docker-compose up --scale go-pq-cdc-kafka=5 -d 29 | ``` 30 | 31 | ### Viewing Running Instances 32 | 33 | ```bash 34 | docker-compose ps go-pq-cdc-kafka 35 | ``` 36 | 37 | Or: 38 | ```bash 39 | docker ps | grep go-pq-cdc-kafka 40 | ``` 41 | 42 | ### Changing the Number of Instances 43 | 44 | **Scaling while running (e.g., from 3 to 5):** 45 | ```bash 46 | docker-compose up --scale go-pq-cdc-kafka=5 -d 47 | ``` 48 | 49 | **Scale down (e.g., from 5 to 2):** 50 | ```bash 51 | docker-compose up --scale go-pq-cdc-kafka=2 -d 52 | ``` 53 | 54 | ### Monitoring Logs of a Specific Instance 55 | 56 | ```bash 57 | # Logs of all instances 58 | docker-compose logs -f go-pq-cdc-kafka 59 | 60 | # Specific container 61 | docker logs -f benchmark_initial_go-pq-cdc-kafka_1 62 | docker logs -f benchmark_initial_go-pq-cdc-kafka_2 63 | ``` 64 | 65 | ### Checking Instances in Prometheus 66 | 67 | In Prometheus UI (http://localhost:9090): 68 | 1. Go to Status → Targets 69 | 2. Find the `go_pq_cdc_exporter` job 70 | 3. You will see all scaled instances listed 71 | 72 | ## Important Notes 73 | 74 | ### ⚠️ Things to Be Careful About 75 | 76 | 1. **Snapshot Mode**: Currently, each instance is running in `SnapshotModeSnapshotOnly` mode. This means each instance will try to take a snapshot from the database. Be careful with coordination. 77 | 78 | 2. **Replication Slot**: Each instance cannot use the same PostgreSQL replication slot. If you're using replication, different slot names are required for each instance. 79 | 80 | 3. **Kafka Partitions**: If you're using multiple instances, it's important for performance that your Kafka topic has multiple partitions. 81 | 82 | 4. **Resource Limits**: CPU and memory limits for each instance: 83 | - CPU Limit: 1 core 84 | - Memory Limit: 512MB 85 | - 3 instances = total 3 cores, 1.5GB RAM 86 | 87 | ### 📊 Monitoring 88 | 89 | In Grafana (http://localhost:3000) you can see metrics for all instances: 90 | - CPU usage 91 | - Memory usage 92 | - Kafka produce rate 93 | - CDC lag 94 | 95 | ### 🔧 Troubleshooting 96 | 97 | **Problem: Instances are not starting** 98 | ```bash 99 | # Check logs 100 | docker-compose logs go-pq-cdc-kafka 101 | 102 | # Check health status 103 | docker-compose ps 104 | ``` 105 | 106 | **Problem: Prometheus is not seeing instances** 107 | ```bash 108 | # Test DNS resolution 109 | docker-compose exec prometheus nslookup tasks.go-pq-cdc-kafka 110 | ``` 111 | 112 | **Problem: Insufficient resources** 113 | ```bash 114 | # Check resource usage 115 | docker stats 116 | ``` 117 | 118 | ## Alternative: Manual Instance Definition 119 | 120 | If you want different configuration for each instance, you can define them manually in docker-compose.yml: 121 | 122 | ```yaml 123 | go-pq-cdc-kafka-1: 124 | build: 125 | context: ../../ 126 | dockerfile: ./benchmark/benchmark_initial/go-pq-cdc-kafka/Dockerfile 127 | # ... other settings 128 | 129 | go-pq-cdc-kafka-2: 130 | build: 131 | context: ../../ 132 | dockerfile: ./benchmark/benchmark_initial/go-pq-cdc-kafka/Dockerfile 133 | # ... other settings 134 | 135 | go-pq-cdc-kafka-3: 136 | build: 137 | context: ../../ 138 | dockerfile: ./benchmark/benchmark_initial/go-pq-cdc-kafka/Dockerfile 139 | # ... other settings 140 | ``` 141 | 142 | ## Performance Tips 143 | 144 | 1. **Batch Size**: There's a `ProducerBatchSize: 10000` setting in `main.go`. Optimize it with the number of instances. 145 | 146 | 2. **Chunk Size**: There's a `ChunkSize: 8000` setting for snapshots. Adjust according to database load. 147 | 148 | 3. **Network**: All services must be on the same Docker network. 149 | 150 | 4. **PostgreSQL**: WAL (Write-Ahead Log) settings should be sufficient: 151 | ``` 152 | wal_level=logical 153 | max_wal_senders=10 154 | max_replication_slots=10 155 | ``` 156 | 157 | ## Example Scenarios 158 | 159 | ### Scenario 1: High Throughput Test 160 | ```bash 161 | # Start with 5 instances 162 | docker-compose up --scale go-pq-cdc-kafka=5 -d 163 | 164 | # Add test data 165 | docker-compose exec postgres psql -U cdc_user -d cdc_db -c \ 166 | "INSERT INTO users (name) SELECT 'User' || i FROM generate_series(1, 1000000) AS i;" 167 | 168 | # Monitor performance 169 | docker stats 170 | ``` 171 | 172 | ### Scenario 2: Graceful Scale Down 173 | ```bash 174 | # First show existing instances 175 | docker-compose ps go-pq-cdc-kafka 176 | 177 | # Gradually scale down 178 | docker-compose up --scale go-pq-cdc-kafka=3 -d 179 | sleep 30 180 | docker-compose up --scale go-pq-cdc-kafka=1 -d 181 | ``` 182 | 183 | ### Scenario 3: Load Testing 184 | ```bash 185 | # Test with different instance counts 186 | for i in 1 2 3 5 10; do 187 | echo "Testing with $i instances..." 188 | docker-compose up --scale go-pq-cdc-kafka=$i -d 189 | sleep 60 190 | # Record metrics 191 | done 192 | ``` 193 | 194 | ## Resources 195 | 196 | - Docker Compose Scale: https://docs.docker.com/compose/reference/up/ 197 | - Prometheus DNS SD: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config 198 | - Kafka Partitioning: https://kafka.apache.org/documentation/#intro_concepts_and_terms 199 | 200 | -------------------------------------------------------------------------------- /pq/snapshot/helpers.go: -------------------------------------------------------------------------------- 1 | package snapshot 2 | 3 | import ( 4 | "context" 5 | goerrors "errors" 6 | "net" 7 | "strconv" 8 | "strings" 9 | "syscall" 10 | "time" 11 | 12 | "github.com/Trendyol/go-pq-cdc/logger" 13 | "github.com/Trendyol/go-pq-cdc/pq" 14 | "github.com/go-playground/errors" 15 | "github.com/jackc/pgx/v5/pgconn" 16 | ) 17 | 18 | // Time format constants for PostgreSQL timestamp formatting 19 | const ( 20 | postgresTimestampFormat = "2006-01-02 15:04:05" 21 | postgresTimestampFormatMicros = "2006-01-02 15:04:05.999999" 22 | ) 23 | 24 | // execSQL executes a SQL statement without returning results (DDL, DML) 25 | func (s *Snapshotter) execSQL(ctx context.Context, conn pq.Connection, sql string) error { 26 | resultReader := conn.Exec(ctx, sql) 27 | _, err := resultReader.ReadAll() 28 | if err != nil { 29 | return err 30 | } 31 | return resultReader.Close() 32 | } 33 | 34 | // execQuery executes a SQL query and returns results 35 | // Query should be pre-formatted (use fmt.Sprintf before calling) 36 | func (s *Snapshotter) execQuery(ctx context.Context, conn pq.Connection, query string) ([]*pgconn.Result, error) { 37 | resultReader := conn.Exec(ctx, query) 38 | results, err := resultReader.ReadAll() 39 | if err != nil { 40 | return nil, err 41 | } 42 | if err = resultReader.Close(); err != nil { 43 | return nil, err 44 | } 45 | return results, nil 46 | } 47 | 48 | // retryDBOperation retries a database operation on transient errors 49 | func (s *Snapshotter) retryDBOperation(ctx context.Context, operation func() error) error { 50 | maxRetries := 3 51 | retryDelay := 1 * time.Second 52 | 53 | var lastErr error 54 | for attempt := 0; attempt < maxRetries; attempt++ { 55 | if attempt > 0 { 56 | // Exponential backoff 57 | delay := retryDelay * time.Duration(1<