├── db-commands.sql ├── create-connector.txt ├── docker-compose.yml └── README.md /db-commands.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE payment; 2 | \c payment 3 | CREATE TABLE transaction(id SERIAL PRIMARY KEY, amount int, customerId varchar(36)); 4 | 5 | 6 | --insert into transaction(id, amount,customerId) values(85, 87,'37b920fd-ecdd-7172-693a-d7be6db9792c'); 7 | --update transaction set amount=77 where id=85 8 | -------------------------------------------------------------------------------- /create-connector.txt: -------------------------------------------------------------------------------- 1 | curl -X POST -H "Accept:application/json" -H "Content-Type:application/json" localhost:8083/connectors/ -d ' 2 | { 3 | "name": "payment-connector", 4 | "config": { 5 | "connector.class": "io.debezium.connector.postgresql.PostgresConnector", 6 | "tasks.max": "1", 7 | "database.hostname": "postgres", 8 | "database.port": "5432", 9 | "database.user": "appuser", 10 | "database.password": "qwerty", 11 | "database.dbname" : "payment", 12 | "database.server.name": "dbserver1", 13 | "database.whitelist": "payment", 14 | "database.history.kafka.bootstrap.servers": "kafka:9092", 15 | "database.history.kafka.topic": "schema-changes.payment" 16 | } 17 | }' 18 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.1' 2 | services: 3 | postgres: 4 | image: debezium/postgres 5 | environment: 6 | POSTGRES_PASSWORD: qwerty 7 | POSTGRES_USER: appuser 8 | volumes: 9 | - ./postgres:/data/postgres 10 | ports: 11 | - 6532:6532 12 | zookeeper: 13 | image: confluentinc/cp-zookeeper 14 | ports: 15 | - "2181:2181" 16 | environment: 17 | ZOOKEEPER_CLIENT_PORT: 2181 18 | kafka: 19 | image: confluentinc/cp-kafka 20 | depends_on: 21 | - zookeeper 22 | - postgres 23 | ports: 24 | - "9092:9092" 25 | environment: 26 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 27 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 28 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 29 | KAFKA_LOG_CLEANER_DELETE_RETENTION_MS: 5000 30 | KAFKA_BROKER_ID: 1 31 | KAFKA_MIN_INSYNC_REPLICAS: 1 32 | connector: 33 | image: debezium/connect:latest 34 | ports: 35 | - "8083:8083" 36 | environment: 37 | GROUP_ID: 1 38 | CONFIG_STORAGE_TOPIC: my_connect_configs 39 | OFFSET_STORAGE_TOPIC: my_connect_offsets 40 | BOOTSTRAP_SERVERS: kafka:9092 41 | depends_on: 42 | - zookeeper 43 | - postgres 44 | - kafka 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # postgresql-change-data-capture-using-debezium 2 | PostgreSQL Change Data Capture (CDC) Using Debezium 3 | ![](http://www.canertosuner.com/image.axd?picture=/2020/dbzum_imgs_1.png) 4 | 5 | 1) Create containers 6 | ``` 7 | docker-compose up 8 | ``` 9 | 10 | 2) Get-into postgresql container to create database & table 11 | ``` 12 | CREATE DATABASE payment; 13 | \c payment 14 | CREATE TABLE transaction(id SERIAL PRIMARY KEY, amount int, customerId varchar(36)); 15 | ``` 16 | 17 | 3) Define a postgres-connector 18 | ``` 19 | curl -X POST -H "Accept:application/json" -H "Content-Type:application/json" localhost:8083/connectors/ -d ' 20 | { 21 | "name": "payment-connector", 22 | "config": { 23 | "connector.class": "io.debezium.connector.postgresql.PostgresConnector", 24 | "tasks.max": "1", 25 | "database.hostname": "postgres", 26 | "database.port": "5432", 27 | "database.user": "appuser", 28 | "database.password": "qwerty", 29 | "database.dbname" : "payment", 30 | "database.server.name": "dbserver1", 31 | "database.whitelist": "payment", 32 | "database.history.kafka.bootstrap.servers": "kafka:9092", 33 | "database.history.kafka.topic": "schema-changes.payment" 34 | } 35 | }' 36 | ``` 37 | 38 | 4) Get-into kafka container to list the topic that you have created. 39 | ``` 40 | kafka-topics --zookeeper zookeeper:2181 --list 41 | ``` 42 | 43 | 5) Create a consumer to consume db tracking messages 44 | ``` 45 | kafka-console-consumer --bootstrap-server kafka:9092 --from-beginning --topic dbserver1.public.transaction --property print.key=true --property key.separator="-" 46 | ``` 47 | 48 | 6) Insert into transaction than update it 49 | ``` 50 | insert into transaction(id, amount,customerId) values(85, 87,'37b920fd-ecdd-7172-693a-d7be6db9792c'); 51 | update transaction set amount=77 where id=85 52 | ``` 53 | 54 | 55 | 56 | **You will see the messages consumed by consumer as blow;** 57 | 58 | **Insert message-payload with "op" : "c"** 59 | ``` 60 | { 61 | "payload":{ 62 | "before":null, 63 | "after":{ 64 | "id":85, 65 | "amount":87, 66 | "customerid":"37b920fd-ecdd-7172-693a-d7be6db9792c" 67 | }, 68 | "source":{ 69 | "version":"1.0.2.Final", 70 | "connector":"postgresql", 71 | "name":"dbserver1", 72 | "ts_ms":1583931003883, 73 | "snapshot":"false", 74 | "db":"payment", 75 | "schema":"public", 76 | "table":"transaction", 77 | "txId":568, 78 | "lsn":23936360, 79 | "xmin":null 80 | }, 81 | "op":"c", 82 | "ts_ms":1583931003889 83 | } 84 | } 85 | ``` 86 | 87 | 88 | **Update message-payload with "op" : "u"** 89 | ``` 90 | { 91 | "payload":{ 92 | "before":null, 93 | "after":{ 94 | "id":85, 95 | "amount":77, 96 | "customerid":"37b920fd-ecdd-7172-693a-d7be6db9792c" 97 | }, 98 | "source":{ 99 | "version":"1.0.2.Final", 100 | "connector":"postgresql", 101 | "name":"dbserver1", 102 | "ts_ms":1583931065480, 103 | "snapshot":"false", 104 | "db":"payment", 105 | "schema":"public", 106 | "table":"transaction", 107 | "txId":569, 108 | "lsn":23936888, 109 | "xmin":null 110 | }, 111 | "op":"u", 112 | "ts_ms":1583931065486 113 | } 114 | } 115 | ``` 116 | --------------------------------------------------------------------------------