├── Pipfile ├── Pipfile.lock ├── README.md ├── avro_producer.py ├── docker-compose.yml ├── setup.cfg └── sink-postgres.json /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.python.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | confluent-kafka = {extras = ["avro"], version = "==0.11.4"} 8 | requests = "==2.18.4" 9 | "avro-python3" = "==1.8.2" 10 | "flake8" = "==3.5.0" 11 | 12 | [dev-packages] 13 | 14 | [requires] 15 | python_version = "3.6" 16 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "922231651b64d3b681a737bacc0f0f89823910935cdfebfb7c62aeed4ed608d6" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.6" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.python.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "avro-python3": { 20 | "hashes": [ 21 | "sha256:f82cf0d66189600b1e6b442f650ad5aca6c189576723dcbf6f9ce096eab81bd6" 22 | ], 23 | "index": "pypi", 24 | "version": "==1.8.2" 25 | }, 26 | "certifi": { 27 | "hashes": [ 28 | "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7", 29 | "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0" 30 | ], 31 | "version": "==2018.4.16" 32 | }, 33 | "chardet": { 34 | "hashes": [ 35 | "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", 36 | "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" 37 | ], 38 | "version": "==3.0.4" 39 | }, 40 | "confluent-kafka": { 41 | "hashes": [ 42 | "sha256:0791df1df35f307e465006c3a4c056e10596c62479d246f3fa6768650bb8070c", 43 | "sha256:0b25f219fb40a0f9c8aa4a07091b2e56711903bd701826bbed630a64667cf826", 44 | "sha256:0c2cf87ce5e32957b7c10476295536a13eaad4158559e24936d6d464bc3fe17d", 45 | "sha256:139ecd519eced72bf9425bc6226fc2a5eb3106932865fafc9e6d439ca0faa2b0", 46 | "sha256:14c4f9dc85c9fae3e4bfaf7ea48b694c11fcaae179bcc32e1e8cbf6d8df763fe", 47 | "sha256:211438f31c106814d4ad8b19353e263533a3e2e4b8544f40db5688bfe298c161", 48 | "sha256:55171e5b6b70f76a7671e8179ca362fd83a2fa8d8bb062c986c467607ff14753", 49 | "sha256:6a2362aed3d082bdacd48e9ec185974f5d501091f0526a630bbe272a2d3e1020", 50 | "sha256:774fc81908cbda66f415a5b0a0c6140a9c9d00186248c2b4c7bb497f856b2033", 51 | "sha256:7d867ff0fb79c962495248d34e4532dc6749b22c4f8504051a86eb8b64297afe", 52 | "sha256:7d889da5474a1a0a0c4f6c05edecd95c0c3e99c1e1958a6a2428e84c01f7dd0f", 53 | "sha256:8cf480199685127c9692b0bf1e15eac82e71ae34b7967a016ab31a318741abb1", 54 | "sha256:9e883f1dbcaedaa1453b525fa6a0105f56dd5b83d7299c5f88c0617084eb0adc", 55 | "sha256:a45ecf4530645100371dacfc528c8de89c885632c117269d956540f1c9c88735", 56 | "sha256:bca83baf6bab225cca310701f615f771378113781ca1ddc6f245437ec90a878f", 57 | "sha256:d85d06ee131145a57b8691196e7621a4c8d22f71e3c7492d1056700d1f436fe5", 58 | "sha256:d9b64d98143398979ea1b6ffa3e792448829c024e4abf9ab1b8f8a0bad0e317f" 59 | ], 60 | "index": "pypi", 61 | "version": "==0.11.4" 62 | }, 63 | "cython": { 64 | "hashes": [ 65 | "sha256:0344e9352b0915910e212c38403b63f902ce1cba75dde7a43a9112ff960eb2a5", 66 | "sha256:0a390c39e912fc5f82d5feae2d16ea061971407099e1efb0fecb255cb96fbeff", 67 | "sha256:0f2b2e09f94c498f555935e732b7321b5f62f00e7a789238f6c5ddd66987a54d", 68 | "sha256:15614592616b6dd5e919e158796350ebeba6cb6b5d2998cfff41b53f568c8355", 69 | "sha256:1aae6d6e9858888144cea147eb5e677830f45faaff3d305d77378c3cba55f526", 70 | "sha256:200583297f23e558744bc4688d8a2b2605ab6ad7d1494a9fd8c8094ad65ebf3c", 71 | "sha256:295facc211a6b55db9979455b856180f2839be22ab767ffdea55986bee83ca9f", 72 | "sha256:36c16bf39280fe857213d8da31c07a6179d3878c3dc2e435dce0974b9f8f0729", 73 | "sha256:3fef8dfa9cf86ab7814ca31369374ddd5b9524f54406aa83b53b5937965b8e88", 74 | "sha256:439d233d3214e3d69c033a9a93516758f2c8a03e83ea51ae14b6eed13687d224", 75 | "sha256:455ab39c6c0849a6c008fcdf2fae42475f18d0801a3be229e8f75367bbe3b325", 76 | "sha256:56821e3791209e6a11992e294afbf7e3dcda7d4fd54d06396dd521928d3d14fe", 77 | "sha256:62b594584889b33bbea7e71f9d7c5c6539091b341334ef7ca1ae7e30a9dd3e15", 78 | "sha256:70f81a75fb25c1c3c61843e3a6fe771a76c4ebf4d154455a7eff0740ad47dff4", 79 | "sha256:8011090beb09251cb4ece1e14263e574b38eda696b788552b369ad343373d0e9", 80 | "sha256:80d6a0369333a162fc32a22637f5870f3e87fb038c7b58860bbe00b05b58aa62", 81 | "sha256:85b04e32af58a3c008c0ba8169017770aaa342a5972b748f81d043d66363e437", 82 | "sha256:9ed273d82116fa148c92901b9639030e087979d455982bd7bf727fb486c0bd17", 83 | "sha256:a1af59e6c9b4acc07c429d8495fc016a35e0a1270f28c57317352f512df7e214", 84 | "sha256:b894ff4daf8dfaf657bf2d5e7190a4de11b2400b1e0fb0902974d35c23a26dea", 85 | "sha256:c2659981150b4de04397dcfd4bff64e384d3ba25af60d1b22820fdf108298cb2", 86 | "sha256:c981a750858f1727995acf861ab030b267d264ca6efda2f01104941187a3675f", 87 | "sha256:cc4152b19ec168391f7815d24b70c8911829ba281bd5fcd98cab9dc21abe62ff", 88 | "sha256:d0f5b1668e7f7f6fc9849f49a20c5db10562a0ab29cd66818894dfebbca7b304", 89 | "sha256:d7152006ed1a3adb8f978077b57d237ddafa188240af53cd72b5c79e4ed000e3", 90 | "sha256:e5f877472993474296125c22b84c334b550010815e513cccce73da854a132d64", 91 | "sha256:e7c2c87ff2f99ed4be1bb046d6eddfb388af627928037f9e0a420c05daaf14ed", 92 | "sha256:edd7d499685655031be5b4d33005096b6345f81eeb7ab9d2dd415db0c7bcf64e", 93 | "sha256:f99a777fda569a88deea863eac2722b5e88957c4d5f4413949740da791857ac9" 94 | ], 95 | "version": "==0.28.3" 96 | }, 97 | "fastavro": { 98 | "hashes": [ 99 | "sha256:72cd1cedaa7225b6b18a366f1eae89593c85d81818a875436fb4c7721b9e278f", 100 | "sha256:8359901eb05d09c97ec7e1894a9467f9762611671e896f1f5fcd365f5e430afa", 101 | "sha256:b06155801d8ba317c26d9fdc361256751fc74af01f02865851f91f826c206bb5", 102 | "sha256:b08810fc45017a67cca04c14da83495390ff30e27ac4aa23e8e31fca6e705613", 103 | "sha256:b10ba2257d7c20afb85f5d8cc432703d7eda3d4f129f453875061f13ad08504c", 104 | "sha256:bd21cfeef899d9d5442fcde7fef7e712f2bfa11ad32b0736f93a5d3e8d9dab36", 105 | "sha256:d6457b40b7c2c1f5f8b5e0922437c6cad49d10e16a6f96f87150b802e098d9e5", 106 | "sha256:f94701d72b497e03d7ac7c423934ea229d34bae582c64504a404a614537e92b9" 107 | ], 108 | "version": "==0.19.6" 109 | }, 110 | "flake8": { 111 | "hashes": [ 112 | "sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0", 113 | "sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37" 114 | ], 115 | "index": "pypi", 116 | "version": "==3.5.0" 117 | }, 118 | "idna": { 119 | "hashes": [ 120 | "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f", 121 | "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4" 122 | ], 123 | "version": "==2.6" 124 | }, 125 | "mccabe": { 126 | "hashes": [ 127 | "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", 128 | "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" 129 | ], 130 | "version": "==0.6.1" 131 | }, 132 | "psycopg2": { 133 | "hashes": [ 134 | "sha256:027ae518d0e3b8fff41990e598bc7774c3d08a3a20e9ecc0b59fb2aaaf152f7f", 135 | "sha256:092a80da1b052a181b6e6c765849c9b32d46c5dac3b81bf8c9b83e697f3cdbe8", 136 | "sha256:0b9851e798bae024ed1a2a6377a8dab4b8a128a56ed406f572f9f06194e4b275", 137 | "sha256:179c52eb870110a8c1b460c86d4f696d58510ea025602cd3f81453746fccb94f", 138 | "sha256:19983b77ec1fc2a210092aa0333ee48811fd9fb5f194c6cd5b927ed409aea5f8", 139 | "sha256:1d90379d01d0dc50ae9b40c863933d87ff82d51dd7d52cea5d1cb7019afd72cd", 140 | "sha256:27467fd5af1dcc0a82d72927113b8f92da8f44b2efbdb8906bd76face95b596d", 141 | "sha256:32702e3bd8bfe12b36226ba9846ed9e22336fc4bd710039d594b36bd432ae255", 142 | "sha256:33f9e1032095e1436fa9ec424abcbd4c170da934fb70e391c5d78275d0307c75", 143 | "sha256:36030ca7f4b4519ee4f52a74edc4ec73c75abfb6ea1d80ac7480953d1c0aa3c3", 144 | "sha256:363fbbf4189722fc46779be1fad2597e2c40b3f577dc618f353a46391cf5d235", 145 | "sha256:6f302c486132f8dd11f143e919e236ea4467d53bf18c451cac577e6988ecbd05", 146 | "sha256:733166464598c239323142c071fa4c9b91c14359176e5ae7e202db6bcc1d2eb5", 147 | "sha256:7cbc3b21ce2f681ca9ad2d8c0901090b23a30c955e980ebf1006d41f37068a95", 148 | "sha256:888bba7841116e529f407f15c6d28fe3ef0760df8c45257442ec2f14f161c871", 149 | "sha256:8966829cb0d21a08a3c5ac971a2eb67c3927ae27c247300a8476554cc0ce2ae8", 150 | "sha256:8bf51191d60f6987482ef0cfe8511bbf4877a5aa7f313d7b488b53189cf26209", 151 | "sha256:8eb94c0625c529215b53c08fb4e461546e2f3fc96a49c13d5474b5ad7aeab6cf", 152 | "sha256:8ebba5314c609a05c6955e5773c7e0e57b8dd817e4f751f30de729be58fa5e78", 153 | "sha256:932a4c101af007cb3132b1f8a9ffef23386acc53dad46536dc5ba43a3235ae02", 154 | "sha256:ad75fe10bea19ad2188c5cb5fc4cdf53ee808d9b44578c94a3cd1e9fc2beb656", 155 | "sha256:aeaba399254ca79c299d9fe6aa811d3c3eac61458dee10270de7f4e71c624998", 156 | "sha256:b178e0923c93393e16646155794521e063ec17b7cc9f943f15b7d4b39776ea2c", 157 | "sha256:b68e89bb086a9476fa85298caab43f92d0a6af135a5f433d1f6b6d82cafa7b55", 158 | "sha256:d74cf9234ba76426add5e123449be08993a9b13ff434c6efa3a07caa305a619f", 159 | "sha256:f3d3a88128f0c219bdc5b2d9ccd496517199660cea021c560a3252116df91cbd", 160 | "sha256:fe6a7f87356116f5ea840c65b032af17deef0e1a5c34013a2962dd6f99b860dd" 161 | ], 162 | "index": "pypi", 163 | "version": "==2.7.4" 164 | }, 165 | "pycodestyle": { 166 | "hashes": [ 167 | "sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766", 168 | "sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9" 169 | ], 170 | "version": "==2.3.1" 171 | }, 172 | "pyflakes": { 173 | "hashes": [ 174 | "sha256:08bd6a50edf8cffa9fa09a463063c425ecaaf10d1eb0335a7e8b1401aef89e6f", 175 | "sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805" 176 | ], 177 | "version": "==1.6.0" 178 | }, 179 | "requests": { 180 | "hashes": [ 181 | "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", 182 | "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" 183 | ], 184 | "index": "pypi", 185 | "version": "==2.18.4" 186 | }, 187 | "sqlalchemy": { 188 | "hashes": [ 189 | "sha256:2d5f08f714a886a1382c18be501e614bce50d362384dc089474019ce0768151c" 190 | ], 191 | "index": "pypi", 192 | "version": "==1.2.8" 193 | }, 194 | "urllib3": { 195 | "hashes": [ 196 | "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", 197 | "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" 198 | ], 199 | "version": "==1.22" 200 | } 201 | }, 202 | "develop": {} 203 | } 204 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [Streaming Data from Kafka to Postgres with Kafka Connect, AVRO, Schema Registry and Python](https://hellokoding.com/kafka-connect-sinks-data-to-postgres-example-with-avro-schema-registry-and-python/) 2 | -------------------------------------------------------------------------------- /avro_producer.py: -------------------------------------------------------------------------------- 1 | from confluent_kafka import avro 2 | from confluent_kafka.avro import AvroProducer 3 | import time 4 | 5 | 6 | value_schema_str = """ 7 | { 8 | "namespace": "hellokoding.kafka", 9 | "name": "value", 10 | "type": "record", 11 | "fields" : [ 12 | {"name" : "browser", "type" : "string"}, 13 | {"name" : "created_at", "type" : "int", "logicalType": "date"} 14 | ] 15 | } 16 | """ 17 | 18 | key_schema_str = """ 19 | { 20 | "namespace": "hellokoding.kafka", 21 | "name": "key", 22 | "type": "record", 23 | "fields" : [ 24 | {"name" : "url", "type" : "string"} 25 | ] 26 | } 27 | """ 28 | 29 | value_schema = avro.loads(value_schema_str) 30 | key_schema = avro.loads(key_schema_str) 31 | value = {"browser": "Chrome", "created_at": int(time.time())} 32 | key = {"url": "http://localhost:8081"} 33 | 34 | avroProducer = AvroProducer({ 35 | 'bootstrap.servers': 'localhost:9092', 36 | 'schema.registry.url': 'http://localhost:8081' 37 | }, default_key_schema=key_schema, default_value_schema=value_schema) 38 | 39 | avroProducer.produce(topic='page_1', value=value, key=key) 40 | avroProducer.flush() 41 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | postgres: 5 | image: postgres:10 6 | container_name: some-postgres 7 | environment: 8 | POSTGRES_PASSWORD: pass 9 | POSTGRES_USER: hellokoding 10 | POSTGRES_DB: kafka-sink 11 | ports: 12 | - 5432:5432 -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 -------------------------------------------------------------------------------- /sink-postgres.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sink-1", 3 | "config": { 4 | "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", 5 | 6 | "key.converter": "io.confluent.connect.avro.AvroConverter", 7 | "key.converter.schema.registry.url": "http://localhost:8081", 8 | "value.converter": "io.confluent.connect.avro.AvroConverter", 9 | "value.converter.schema.registry.url": "http://localhost:8081", 10 | 11 | "connection.url": "jdbc:postgresql://localhost:5432/kafka-sink", 12 | "connection.user": "hellokoding", 13 | "connection.password": "pass", 14 | "insert.mode": "upsert", 15 | "auto.create": true, 16 | "auto.evolve": true, 17 | "topics": "page_1", 18 | "pk.mode": "record_key" 19 | } 20 | } --------------------------------------------------------------------------------