├── Procfile ├── scripts └── heroku-exec.sh ├── Aptfile ├── Connectors ├── jdbc-postgres-example.json ├── jdbc-demo-connector.json └── make-connector ├── app.json ├── setup_certs ├── README.md └── start-distributed /Procfile: -------------------------------------------------------------------------------- 1 | web: ./start-distributed 2 | -------------------------------------------------------------------------------- /scripts/heroku-exec.sh: -------------------------------------------------------------------------------- 1 | [ -z "$SSH_CLIENT" ] && source <(curl --fail --retry 3 -sSL "$HEROKU_EXEC_URL") -------------------------------------------------------------------------------- /Aptfile: -------------------------------------------------------------------------------- 1 | :key:nginx:https://nginx.org/keys/nginx_signing.key 2 | :key:confluent:http://packages.confluent.io/deb/5.2/archive.key 3 | :repo:deb [arch=amd64] https://packages.confluent.io/deb/5.2 stable main 4 | confluent-kafka-2.11 5 | confluent-schema-registry 6 | confluent-kafka-connect-jdbc 7 | confluent-kafka-connect-s3 8 | nginx 9 | apache2-utils 10 | curl 11 | openssh-server 12 | openssl -------------------------------------------------------------------------------- /Connectors/jdbc-postgres-example.json: -------------------------------------------------------------------------------- 1 | { "name": "jdbc-postgresql-example", 2 | "config": { 3 | "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", 4 | "connection.url": "jdbc:postgresql://hostxxxx", 5 | "connection.user": "xxxx", 6 | "connection.password": "xxxx", 7 | "topic.prefix": "pg-", 8 | "mode":"bulk" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Connectors/jdbc-demo-connector.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "demo-connector", 3 | "config": { 4 | "connector.class": "io.debezium.connector.postgresql.PostgresConnector", 5 | "tasks.max": "1", 6 | "database.hostname": "postgres", 7 | "database.port": "", 8 | "database.user": "", 9 | "database.password": "", 10 | "database.dbname" : "postgres", 11 | "database.server.name": "postgresql-jchao-perpendicular-77247", 12 | "database.whitelist": "demo", 13 | "database.history.kafka.bootstrap.servers": "", 14 | "database.history.kafka.topic": "schema-changes.demo" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Kafka Connect JDBC Connector", 3 | "description": "Kafka Connect JDBC connector configured for Heroku Data", 4 | "keywords": [ 5 | "Kafka", 6 | "Postgres", 7 | "Connect", 8 | "CDC" 9 | ], 10 | "repository": "https://github.com/ahamidi/kafka-connect-jdbc", 11 | "env": { 12 | "TRUSTSTORE_PASSWORD": { 13 | "description": "Password used for the Java truststore", 14 | "generator": "secret" 15 | }, 16 | "KEYSTORE_PASSWORD": { 17 | "description": "Password used for the Java keystore", 18 | "generator": "secret" 19 | } 20 | }, 21 | "formation": { 22 | "web": { 23 | "quantity": 1, 24 | "size": "Standard-1X" 25 | } 26 | }, 27 | "buildpacks": [ 28 | { 29 | "url": "https://github.com/heroku/heroku-buildpack-apt" 30 | }, 31 | { 32 | "url": "heroku/jvm" 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /setup_certs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | addon="$1" 6 | 7 | [ -z $addon ] && { 8 | echo "addon is missing" >&2 9 | exit 1 10 | } 11 | 12 | client_key="$(echo $addon)_CLIENT_CERT_KEY" 13 | client_cert="$(echo $addon)_CLIENT_CERT" 14 | trusted_cert="$(echo $addon)_TRUSTED_CERT" 15 | 16 | [ -z $TRUSTSTORE_PASSWORD ] && { 17 | echo "TRUSTSTORE_PASSWORD is missing" >&2 18 | exit 1 19 | } 20 | 21 | [ -z $KEYSTORE_PASSWORD ] && { 22 | echo "KEYSTORE_PASSWORD is missing" >&2 23 | exit 1 24 | } 25 | 26 | rm -f .{keystore,truststore}.{pem,pkcs12,jks} 27 | rm -f .cacerts 28 | 29 | echo -n "${!client_key}" >> .keystore.pem 30 | echo -n "${!client_cert}" >> .keystore.pem 31 | echo -n "${!trusted_cert}" > .truststore.pem 32 | 33 | keytool -importcert -file .truststore.pem -keystore .truststore.jks -deststorepass $TRUSTSTORE_PASSWORD -noprompt 34 | 35 | openssl pkcs12 -export -in .keystore.pem -out .keystore.pkcs12 -password pass:$KEYSTORE_PASSWORD 36 | keytool -importkeystore -srcstoretype PKCS12 \ 37 | -destkeystore .keystore.jks -deststorepass $KEYSTORE_PASSWORD \ 38 | -srckeystore .keystore.pkcs12 -srcstorepass $KEYSTORE_PASSWORD 39 | 40 | rm -f .{keystore,truststore}.{pem,pkcs12} 41 | -------------------------------------------------------------------------------- /Connectors/make-connector: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | app_name="$1" 6 | table_name="$2" 7 | 8 | usage() { 9 | echo "Usage: $0 HEROKU_APP_NAME POSTGRES_TABLE_NAME" 10 | } 11 | 12 | [ -z $app_name ] && { 13 | echo "APP_NAME is missing" >&2 14 | usage 15 | exit 1 16 | } 17 | 18 | [ -z $table_name ] && { 19 | echo "TABLE_NAME is missing" >&2 20 | usage 21 | exit 1 22 | } 23 | 24 | eval $(heroku config:get -s DATABASE_URL KAFKA_URL -a "$app_name") 25 | 26 | url_re='^postgres://([[:alnum:]]+):([[:alnum:]]+)@([-\.[:alnum:]]+):([[:digit:]]+)/([[:alnum:]]+)' 27 | if [[ $DATABASE_URL =~ $url_re ]]; then 28 | user=${BASH_REMATCH[1]} 29 | pass=${BASH_REMATCH[2]} 30 | host=${BASH_REMATCH[3]} 31 | port=${BASH_REMATCH[4]} 32 | database=${BASH_REMATCH[5]} 33 | fi 34 | 35 | # cat <`. 14 | 15 | **Kafka Consumer Group** 16 | You will also need to create the Consumer Group `kafka-connect` 17 | 18 | ### Config Vars 19 | 20 | * `KAFKA_ADDON` 21 | * `POSTGRES_ADDON` 22 | * `TRUSTSTORE_PASSWORD` 23 | * `KEYSTORE_PASSWORD` 24 | 25 | ### Buildpacks 26 | 27 | * Apt: `https://github.com/heroku/heroku-buildpack-apt` < TODO fails on keys https://github.com/heroku/heroku-buildpack-apt/pull/41 28 | * Apt: `heroku buildpacks:add --index=1 https://github.com/amiel/heroku-buildpack-apt#feature/support-adding-keys 29 | ` 30 | * JVM: `heroku/jvm` 31 | 32 | ### Setup 33 | ## Using the app.json 34 | 35 | heroku kafka:topics:create pg-[postgres table name] 36 | 37 | ## Manually 38 | heroku config:set TRUSTSTORE_PASSWORD="trustypassword9" 39 | heroku config:set KEYSTORE_PASSWORD="keytrustypassword9" 40 | heroku buildpacks:add --index 1 heroku-community/apt 41 | heroku buildpacks:add heroku/jvm 42 | 43 | ### Verify install 44 | heroku ps:exec 45 | https://devcenter.heroku.com/articles/exec#getting-started 46 | https://[appname].herokuapp.com/connectors 47 | 48 | 49 | Check the log for something like "INFO Added plugin 'io.confluent.connect.jdbc.JdbcSourceConnector'" 50 | 51 | ### JDBC connector info 52 | https://www.confluent.io/blog/kafka-connect-deep-dive-jdbc-source-connector 53 | https://docs.confluent.io/current/connect/managing/connectors.html#connect-bundled-connectors 54 | https://docs.confluent.io/current/connect/kafka-connect-jdbc/source-connector/index.html 55 | https://docs.confluent.io/current/connect/kafka-connect-jdbc/source-connector/source_config_options.html 56 | 57 | configure the connector 58 | 59 | curl -X POST -H "Content-Type: application/json" -d @jdbc-postgres-example.json "http://lyric-connect-jdbc.herokuapp.com/connectors" 60 | 61 | 62 | curl -X POST http://localhost:8083/connectors -H "Content-Type: application/json" -d '{ 63 | "name": "jdbc_source_postgres_01", 64 | "config": { 65 | "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", 66 | "connection.url": "jdbc:postgresql://postgres:5432/postgres", 67 | "connection.user": "connect_user", 68 | "connection.password": "password", 69 | "topic.prefix": "postgres-01-", 70 | "mode":"bulk", 71 | "poll.interval.ms" : 3600000 72 | } 73 | }' 74 | 75 | 76 | curl -X POST -H "Content-Type: application/json" -d @jdbc-source-with-schema.json "http://localhost:8083/connectors" 77 | ... 78 | {"name":"jdbc-postgres-source-with-schema-connector","config":{"connector.class":"io.confluent.connect.jdbc.JdbcSourceConnector","tasks.max":"1","mode":"bulk","connection.url":"jdbc:postgresql://localhost:5432/customers?user=postgres&password=newpass","table.whitelist":"addresses","topic.prefix":"jdbc-postgresql-with-schema-example-","name":"jdbc-postgres-source-with-schema-connector"},"tasks":[],"type":null} 79 | -------------------------------------------------------------------------------- /start-distributed: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | DYNO_CONNECT_DISTRIBUTED_PROPERTIES_FILE="dyno-connect-distributed.properties" 4 | 5 | kafka_addon_name=${KAFKA_ADDON:-KAFKA} 6 | prefix_env_var="$(echo $kafka_addon_name)_PREFIX" 7 | kafka_prefix=$(echo ${!prefix_env_var}) 8 | kafka_url_env_var="$(echo $kafka_addon_name)_URL" 9 | postgres_addon_name=${POSTGRES_ADDON:-DATABASE} 10 | 11 | 12 | ./setup_certs $kafka_addon_name 13 | 14 | cat > $DYNO_CONNECT_DISTRIBUTED_PROPERTIES_FILE <