├── .gitignore ├── README.md ├── azure ├── azure-blob-storage-sink.properties └── azure-blob-storage-source.properties ├── gcp ├── gcs-sink.properties └── gcs-source.properties ├── kafka-delete-all-topics.sh ├── mysql ├── README.md ├── mysql-bulk-sink.json ├── mysql-bulk-sink.properties ├── mysql-bulk-source.json └── mysql-bulk-source.properties ├── s3 ├── README.md ├── s3-sink.json ├── s3-sink.properties ├── s3-source.json └── s3-source.properties └── voluble ├── full-stack-json-connect.yml ├── joinable-examples.json ├── joinable.json ├── mdrogalis-voluble-0.3.1.zip ├── voluble-source.properties ├── voluble.json └── whizbang-java-0.3.1-jar-with-dependencies.jar /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Kafka Connect Examples 3 | 4 | Repo of various Kafka Connect examples. The accompanying tutorials and screencasts for [Kafka Connect](https://supergloo.com/kafka-connect/) and other Kafka tutorials available at https://www.supergloo.com/kafka-tutorial/ 5 | 6 | 7 | [Kafka Connect mySQL Examples](https://supergloo.com/kafka-connect/kafka-connect-mysql-example/) 8 | 9 | [Kafka Connect S3 Examples](https://supergloo.com/kafka-connect/kafka-connect-s3-examples/) 10 | 11 | [Kafka Test Data Generation](https://supergloo.com/kafka/kafka-test-data/) 12 | 13 | [GCP Kafka Examples with Google Cloud Storage (GCS)](https://supergloo.com/kafka-connect/gcp-kafka-connect-gcs-examples/) 14 | 15 | [Azure Kafka Examples with Azure Blob Storage](https://supergloo.com/kafka-connect/azure-kafka-blob-storage/) 16 | 17 | # Kafka Connect Running Modes 18 | 19 | Most of these tutorials above can be adjusted to run in either Standalone or Distributed mode. More these [Kafka Connect Modes here](https://supergloo.com/kafka-connect/running-kafka-connect-standalone-vs-distributed-mode-examples/) 20 | -------------------------------------------------------------------------------- /azure/azure-blob-storage-sink.properties: -------------------------------------------------------------------------------- 1 | # Azure Blog Storage Sink example for Kafka Connect 2 | # This example is meant to 3 | # be accompanied with blog post and screencast 4 | # at https://supergloo.com 5 | # 6 | # People are counting on you 7 | # 8 | 9 | name=azure-bs-sink 10 | connector.class=io.confluent.connect.azure.blob.AzureBlobStorageSinkConnector 11 | tasks.max=1 12 | flush.size=3 13 | 14 | azblob.account.name=tmcgrathstorageaccount 15 | # azblob.account.key= 16 | azblob.container.name=kafka-connect-example 17 | 18 | # Key converter same for both examples 19 | key.converter=org.apache.kafka.connect.storage.StringConverter 20 | 21 | # Avro example with orders test data 22 | # topics=orders 23 | #format.class=io.confluent.connect.azure.blob.format.avro.AvroFormat 24 | #value.converter=io.confluent.connect.avro.AvroConverter 25 | #value.converter.schema.registry.url=http://localhost:8081 26 | 27 | # JSON example with pageviews test data 28 | topics=pageviews 29 | format.class=io.confluent.connect.azure.blob.format.json.JsonFormat 30 | value.converter=org.apache.kafka.connect.json.JsonConverter 31 | value.converter.schemas.enable=false 32 | 33 | schema.compatibility=NONE 34 | 35 | partitioner.class=io.confluent.connect.storage.partitioner.DefaultPartitioner 36 | 37 | ############################# 38 | # CONFLUENT LICENSE SETTINGS 39 | ############################# 40 | 41 | confluent.license= 42 | confluent.topic.bootstrap.servers=localhost:9092 43 | confluent.topic.replication.factor=1 44 | -------------------------------------------------------------------------------- /azure/azure-blob-storage-source.properties: -------------------------------------------------------------------------------- 1 | # Azure Blog Storage Kafka Connect Source Example 2 | # Minimum changes needed to run in your environment 3 | # `azblob.account.name` and `azblob.account.key` 4 | # 5 | 6 | name=azure-bs-source 7 | connector.class=io.confluent.connect.azure.blob.storage.AzureBlobStorageSourceConnector 8 | tasks.max=1 9 | flush.size=3 10 | 11 | azblob.account.name=tmcgrathstorageaccount 12 | #azblob.account.key= 13 | azblob.container.name=kafka-connect-example 14 | 15 | confluent.topic.bootstrap.servers=localhost:9092 16 | confluent.topic.replication.factor=1 17 | confluent.license= 18 | 19 | format.class=io.confluent.connect.azure.blob.storage.format.avro.AvroFormat 20 | 21 | # If destination topic already exists, uncomment the following 22 | # and update as needed 23 | #transforms=AddPrefix 24 | #transforms.AddPrefix.type=org.apache.kafka.connect.transforms.RegexRouter 25 | #transforms.AddPrefix.regex=.* 26 | #transforms.AddPrefix.replacement=copy_of_$0 27 | -------------------------------------------------------------------------------- /gcp/gcs-sink.properties: -------------------------------------------------------------------------------- 1 | # GCS Sink example for Kafka Connect 2 | # Two examples are provided and this example is meant to 3 | # be accompanied with blog post and screencast 4 | # at https://supergloo.com 5 | # 6 | # You need to change `gcs.bucket.name` and `gcs.credentials` vars 7 | # and set either AVRO or JSON outputs depending on your source 8 | # data. Default is Avro using the orders example 9 | # 10 | # You may need to change other settings if you go outside of the 11 | # quickstart demos I provided. 12 | # Good luck, we're all counting on you 13 | # 14 | 15 | name=gcs-sink 16 | connector.class=io.confluent.connect.gcs.GcsSinkConnector 17 | tasks.max=1 18 | 19 | time.interval=HOURLY 20 | 21 | gcs.bucket.name=kafka-connect-example 22 | gcs.part.size=5242880 23 | flush.size=3 24 | 25 | gcs.credentials.path=//gcp-key.json 26 | storage.class=io.confluent.connect.gcs.storage.GcsStorage 27 | 28 | # Key converter same for both examples 29 | key.converter=org.apache.kafka.connect.storage.StringConverter 30 | 31 | # Avro example with orders test data 32 | topics=orders 33 | format.class=io.confluent.connect.gcs.format.avro.AvroFormat 34 | value.converter=io.confluent.connect.avro.AvroConverter 35 | value.converter.schema.registry.url=http://localhost:8081 36 | 37 | # JSON example with pageviews test data 38 | # topics=pageviews 39 | # format.class=io.confluent.connect.gcs.format.json.JsonFormat 40 | # value.converter=org.apache.kafka.connect.json.JsonConverter 41 | # value.converter.schemas.enable=false 42 | 43 | schema.compatibility=NONE 44 | 45 | partitioner.class=io.confluent.connect.storage.partitioner.DefaultPartitioner 46 | 47 | # Example values when using a time-based partitioner 48 | #partitioner.class=io.confluent.connect.storage.partitioner.TimeBasedPartitioner 49 | # For instance, set partition duration to 1 hour 50 | #partition.duration.ms=3600000 51 | #path.format='year'=YYYY_'month'=MM_'day'=dd_'hour'=HH 52 | #locale=en 53 | #timezone=America/Los_Angeles 54 | # Use a deterministic partitioner based on Kafka record timestamps 55 | #timestamp.extractor=Record 56 | # For example, rotate a file every 15 minutes 57 | #rotate.interval.ms=900000 58 | # Also, set size-based rotation to a high value 59 | #flush.size=1000000 60 | 61 | # Or use a field based partitioner 62 | #partitioner.class=io.confluent.connect.storage.partitioner.FieldPartitioner 63 | #partition.field.name= 64 | 65 | 66 | ############################# 67 | # CONFLUENT LICENSE SETTINGS 68 | ############################# 69 | 70 | # Confluent will issue a license key to each subscriber. The license key will be a short snippet 71 | # of text that you can copy and paste. Without the license key, you can use this connector for a 72 | # 30-day trial period. If you are a subscriber, please contact Confluent Support for more 73 | # information. 74 | #confluent.license= 75 | 76 | # Name of the Kafka topic used for Confluent Platform configuration, including licensing 77 | # information. 78 | #confluent.topic=_confluent-command 79 | 80 | # A list of host/port pairs to use for establishing the initial connection to the Kafka cluster 81 | # used for licensing. All servers in the cluster will be discovered from the initial connection. 82 | # This list should be in the form host1:port1,host2:port2,.... Since these servers 83 | # are just used for the initial connection to discover the full cluster membership (which may 84 | # change dynamically), this list need not contain the full set of servers (you may want more than 85 | # one, though, in case a server is down). 86 | # This is a required config property 87 | confluent.topic.bootstrap.servers=localhost:9092 88 | 89 | # The replication factor for the Kafka topic used for Confluent Platform configuration, including 90 | # licensing information. This is used only if the topic does not already exist, and the default 91 | # of 3 is appropriate for production use. If you are using a development environment with less 92 | # than 3 brokers, you must set this to the number of brokers (often 1). 93 | #confluent.topic.replication.factor=3 94 | confluent.topic.replication.factor=1 95 | 96 | # In secured clusters, add any security configs to the Kafka clients used on confluent.topic by 97 | # using the prefixes, such as confluent.topic. 98 | # or confluent.topic.producer. and confluent.topic.consumer. 99 | # if settings differ between the producer and the consumer that read the confluent.topic 100 | -------------------------------------------------------------------------------- /gcp/gcs-source.properties: -------------------------------------------------------------------------------- 1 | # Minimum changes needed to run in your environment 2 | # `gcs.credentials.path` and `gcs.bucket.name` if you don't have a bucket 3 | # named kafka-connect-example 4 | # 5 | 6 | name=GcsSourceConnector 7 | tasks.max=1 8 | confluent.topic.bootstrap.servers=localhost:9092 9 | confluent.topic.replication.factor=1 10 | connector.class=io.confluent.connect.gcs.GcsSourceConnector 11 | gcs.bucket.name=kafka-connect-example 12 | gcs.credentials.path=/changeme/gcp-key.json 13 | format.class=io.confluent.connect.gcs.format.avro.AvroFormat 14 | 15 | # transforms=AddPrefix 16 | # transforms.AddPrefix.type=org.apache.kafka.connect.transforms.RegexRouter 17 | # transforms.AddPrefix.regex=.* 18 | # transforms.AddPrefix.replacement=copy_of_$0 19 | -------------------------------------------------------------------------------- /kafka-delete-all-topics.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Set you KAFKA_HOME to use 3 | 4 | TOPICS=$($KAFKA_HOME/bin/kafka-topics.sh --zookeeper 0.0.0.0:2181 --list ) 5 | 6 | for T in $TOPICS 7 | do 8 | if [ "$T" != "__consumer_offsets" ]; then 9 | $KAFKA_HOME/bin/kafka-topics.sh --zookeeper 0.0.0.0:2181 --delete --topic $T 10 | fi 11 | done 12 | -------------------------------------------------------------------------------- /mysql/README.md: -------------------------------------------------------------------------------- 1 | # Kafka Connect mySQL (JDBC) Example 2 | 3 | These files are the configuration examples used in the tutorial and screencasts for [Kafka Connect with mySQL](https://supergloo.com/kafka-connect/kafka-connect-mysql-example/) 4 | 5 | Also, included in this repo, but not specifically mentioned in the tutorial are JSON config files which you may find helpful. Especially, if you are using Kafka Connect REST API in Distributed Mode. More info and examples on [Kafka Connect Modes found here](https://supergloo.com/kafka-connect/running-kafka-connect-standalone-vs-distributed-mode-examples) 6 | -------------------------------------------------------------------------------- /mysql/mysql-bulk-sink.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mysql-bulk-sink", 3 | "config": { 4 | "connector.class":"io.confluent.connect.jdbc.JdbcSinkConnector", 5 | "tasks.max":"3", 6 | "connection.url":"jdbc:mysql://localhost/employeesink", 7 | "connection.user":"ss", 8 | "connection.password":"ss", 9 | "auto.create":"true", 10 | "topics.regex":"^mysql-.*" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /mysql/mysql-bulk-sink.properties: -------------------------------------------------------------------------------- 1 | name=mysql-bulk-sink 2 | connector.class=io.confluent.connect.jdbc.JdbcSinkConnector 3 | tasks.max=3 4 | 5 | connection.url=jdbc:mysql://localhost/employeesink 6 | connection.user=ss 7 | connection.password=ss 8 | auto.create=true 9 | 10 | topics.regex=^mysql-.* 11 | # Question - how do we configure the destination table to remove "mysql-" ? 12 | -------------------------------------------------------------------------------- /mysql/mysql-bulk-source.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mysql-bulk-source", 3 | "config": { 4 | "connector.class":"io.confluent.connect.jdbc.JdbcSourceConnector", 5 | "tasks.max":"10", 6 | "connection.url":"jdbc:mysql://localhost/employees", 7 | "connection.user":"ss", 8 | "connection.password":"ss", 9 | "mode":"bulk", 10 | "topic.prefix":"mysql-" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /mysql/mysql-bulk-source.properties: -------------------------------------------------------------------------------- 1 | name=mysql-bulk-source 2 | connector.class=io.confluent.connect.jdbc.JdbcSourceConnector 3 | tasks.max=10 4 | 5 | connection.url=jdbc:mysql://localhost/employees 6 | connection.user=ss 7 | connection.password=ss 8 | 9 | mode=bulk 10 | topic.prefix=mysql- 11 | -------------------------------------------------------------------------------- /s3/README.md: -------------------------------------------------------------------------------- 1 | # Kafka Connect S3 Examples 2 | 3 | Source configuration for the [Kafka Connect with S3 tutorial](https://supergloo.com/kafka-connect/kafka-connect-s3-examples/) 4 | 5 | Also, included in this repo but not specifically mentioned in the tutorial are JSON config example files which you may find helpful when using the Kafka Connect REST API. More info and examples of REST API usage and [Kafka Connect Modes found here](https://supergloo.com/kafka-connect/running-kafka-connect-standalone-vs-distributed-mode-examples) 6 | -------------------------------------------------------------------------------- /s3/s3-sink.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "s3-sink", 3 | "config": { 4 | "connector.class":"io.confluent.connect.s3.S3SinkConnector", 5 | "tasks.max":"1", 6 | "topics":"mysql-employees", 7 | "s3.region":"us-east-1", 8 | "s3.bucket.name":"kafka-connect-example", 9 | "s3.part.size":"5242880", 10 | "flush.size":"3", 11 | "storage.class":"io.confluent.connect.s3.storage.S3Storage", 12 | "format.class":"io.confluent.connect.s3.format.avro.AvroFormat", 13 | "partitioner.class":"io.confluent.connect.storage.partitioner.DefaultPartitioner", 14 | "schema.compatibility":"NONE" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /s3/s3-sink.properties: -------------------------------------------------------------------------------- 1 | # Kafka S3 sink example 2 | # See comments for different examples such as multiple topics 3 | # and different output formats and partitioners 4 | 5 | name=s3-sink 6 | connector.class=io.confluent.connect.s3.S3SinkConnector 7 | tasks.max=1 8 | 9 | # One or more topics? 10 | topics=mysql-employees 11 | # topics.regex=mysql* 12 | 13 | s3.region=us-east-1 14 | s3.bucket.name=kafka-connect-example 15 | s3.part.size=5242880 16 | flush.size=3 17 | 18 | storage.class=io.confluent.connect.s3.storage.S3Storage 19 | 20 | #Output to Avro,Parquet or JSON? 21 | format.class=io.confluent.connect.s3.format.avro.AvroFormat 22 | #format.class=io.confluent.connect.s3.format.parquet.ParquetFormat 23 | #format.class=io.confluent.connect.s3.format.json.JsonFormat 24 | 25 | partitioner.class=io.confluent.connect.storage.partitioner.DefaultPartitioner 26 | #partitioner.class= io.confluent.connect.storage.partitioner.DailyPartitioner 27 | # https://docs.confluent.io/current/connect/kafka-connect-s3/index.html#partitioning-records-into-s3-objects for more on partitioning options 28 | 29 | schema.compatibility=NONE 30 | #partition.field.name= 31 | #partition.duration.ms= 32 | #path.format= 33 | #locale= 34 | #timezone= 35 | -------------------------------------------------------------------------------- /s3/s3-source.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "s3-source", 3 | "config": { 4 | "connector.class":"io.confluent.connect.s3.source.S3SourceConnector", 5 | "tasks.max":"1", 6 | "s3.region":"us-east-1", 7 | "s3.bucket.name":"kafka-connect-example", 8 | "format.class":"io.confluent.connect.s3.format.avro.AvroFormat", 9 | "partitioner.class":"io.confluent.connect.storage.partitioner.DefaultPartitioner", 10 | "schema.compatibility":"NONE", 11 | "transforms":"AddPrefix", 12 | "transforms.AddPrefix.type":"org.apache.kafka.connect.transforms.RegexRouter", 13 | "transforms.AddPrefix.regex":".*", 14 | "transforms.AddPrefix.replacement":"copy_of_$0", 15 | "confluent.license":"", 16 | "confluent.topic.bootstrap.servers":"localhost:9092", 17 | "confluent.topic.replication.factor":"1" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /s3/s3-source.properties: -------------------------------------------------------------------------------- 1 | # Kafka Connect S3 Source example 2 | 3 | name=s3-source 4 | connector.class=io.confluent.connect.s3.source.S3SourceConnector 5 | tasks.max=1 6 | 7 | s3.region=us-east-1 8 | s3.bucket.name=kafka-connect-example 9 | # s3.part.size=5242880 10 | # flush.size=3 11 | 12 | format.class=io.confluent.connect.s3.format.avro.AvroFormat 13 | partitioner.class=io.confluent.connect.storage.partitioner.DefaultPartitioner 14 | 15 | schema.compatibility=NONE 16 | 17 | transforms=AddPrefix 18 | transforms.AddPrefix.type=org.apache.kafka.connect.transforms.RegexRouter 19 | transforms.AddPrefix.regex=.* 20 | transforms.AddPrefix.replacement=copy_of_$0 21 | 22 | confluent.license= 23 | confluent.topic.bootstrap.servers=localhost:9092 24 | confluent.topic.replication.factor=1 25 | -------------------------------------------------------------------------------- /voluble/full-stack-json-connect.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | zoo1: 5 | image: zookeeper:3.4.9 6 | restart: unless-stopped 7 | hostname: zoo1 8 | ports: 9 | - "2181:2181" 10 | environment: 11 | ZOO_MY_ID: 1 12 | ZOO_PORT: 2181 13 | ZOO_SERVERS: server.1=zoo1:2888:3888 14 | volumes: 15 | - ./full-stack/zoo1/data:/data 16 | - ./full-stack/zoo1/datalog:/datalog 17 | 18 | 19 | kafka1: 20 | image: confluentinc/cp-kafka:5.5.1 21 | hostname: kafka1 22 | ports: 23 | - "9092:9092" 24 | environment: 25 | KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 26 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT 27 | KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL 28 | KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" 29 | KAFKA_BROKER_ID: 1 30 | KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" 31 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 32 | volumes: 33 | - ./full-stack/kafka1/data:/var/lib/kafka/data 34 | depends_on: 35 | - zoo1 36 | 37 | kafka-schema-registry: 38 | image: confluentinc/cp-schema-registry:5.5.1 39 | hostname: kafka-schema-registry 40 | ports: 41 | - "8081:8081" 42 | environment: 43 | SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092 44 | SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry 45 | SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 46 | depends_on: 47 | - zoo1 48 | - kafka1 49 | 50 | schema-registry-ui: 51 | image: landoop/schema-registry-ui:0.9.5 52 | hostname: kafka-schema-registry-ui 53 | ports: 54 | - "8001:8000" 55 | environment: 56 | SCHEMAREGISTRY_URL: http://kafka-schema-registry:8081/ 57 | PROXY: "true" 58 | depends_on: 59 | - kafka-schema-registry 60 | 61 | kafka-rest-proxy: 62 | image: confluentinc/cp-kafka-rest:5.5.1 63 | hostname: kafka-rest-proxy 64 | ports: 65 | - "8082:8082" 66 | environment: 67 | # KAFKA_REST_ZOOKEEPER_CONNECT: zoo1:2181 68 | KAFKA_REST_LISTENERS: http://0.0.0.0:8082/ 69 | KAFKA_REST_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081/ 70 | KAFKA_REST_HOST_NAME: kafka-rest-proxy 71 | KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092 72 | depends_on: 73 | - zoo1 74 | - kafka1 75 | - kafka-schema-registry 76 | 77 | kafka-topics-ui: 78 | image: landoop/kafka-topics-ui:0.9.4 79 | hostname: kafka-topics-ui 80 | ports: 81 | - "8000:8000" 82 | environment: 83 | KAFKA_REST_PROXY_URL: "http://kafka-rest-proxy:8082/" 84 | PROXY: "true" 85 | depends_on: 86 | - zoo1 87 | - kafka1 88 | - kafka-schema-registry 89 | - kafka-rest-proxy 90 | 91 | kafka-connect: 92 | image: confluentinc/cp-kafka-connect:5.5.1 93 | hostname: kafka-connect 94 | ports: 95 | - "8083:8083" 96 | environment: 97 | CONNECT_BOOTSTRAP_SERVERS: "kafka1:19092" 98 | CONNECT_REST_PORT: 8083 99 | CONNECT_GROUP_ID: compose-connect-group 100 | CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs 101 | CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets 102 | CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status 103 | CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter " 104 | CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" 105 | CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false" 106 | CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" 107 | CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" 108 | CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect" 109 | CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO" 110 | CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" 111 | CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1" 112 | CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1" 113 | CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1" 114 | CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars' 115 | volumes: 116 | - ./connectors:/etc/kafka-connect/jars/ 117 | depends_on: 118 | - zoo1 119 | - kafka1 120 | - kafka-schema-registry 121 | - kafka-rest-proxy 122 | 123 | kafka-connect-ui: 124 | image: landoop/kafka-connect-ui:0.9.7 125 | hostname: kafka-connect-ui 126 | ports: 127 | - "8003:8000" 128 | environment: 129 | CONNECT_URL: "http://kafka-connect:8083/" 130 | PROXY: "true" 131 | depends_on: 132 | - kafka-connect 133 | 134 | ksqldb-server: 135 | image: confluentinc/cp-ksqldb-server:5.5.1 136 | hostname: ksqldb-server 137 | ports: 138 | - "8088:8088" 139 | environment: 140 | KSQL_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092 141 | KSQL_LISTENERS: http://0.0.0.0:8088/ 142 | KSQL_KSQL_SERVICE_ID: ksqldb-server_ 143 | depends_on: 144 | - zoo1 145 | - kafka1 146 | 147 | zoonavigator: 148 | image: elkozmon/zoonavigator:0.8.0 149 | ports: 150 | - "8004:8000" 151 | environment: 152 | HTTP_PORT: 8000 153 | AUTO_CONNECT_CONNECTION_STRING: zoo1:2181 154 | -------------------------------------------------------------------------------- /voluble/joinable-examples.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "joinable", 3 | "config": { 4 | "connector.class": "io.mdrogalis.voluble.VolubleSourceConnector", 5 | 6 | "genkp.inventory.sometimes.with": "#{Code.asin}", 7 | "genkp.inventory.sometimes.matching": "inventory.key", 8 | "genv.inventory.amount_in_stock.with": "#{number.number_between '5','15'}", 9 | "genv.inventory.product_name.with": "#{Commerce.product_name}", 10 | "genv.inventory.last_updated.with": "#{date.past '10','SECONDS'}", 11 | 12 | "genkp.customer.with": "#{Code.isbn10}", 13 | "genv.customer.name.with": "#{Name.full_name}", 14 | "genv.customer.gender.with": "#{Demographic.sex}", 15 | "genv.customer.favorite_beer.with": "#{Beer.name}", 16 | "genv.customer.state.with": "#{Address.state}", 17 | 18 | "genkp.order.matching": "inventory.key", 19 | "genv.order.quantity.with": "#{number.number_between '1','5'}", 20 | "genv.order.customer_id.matching": "customer.key", 21 | 22 | "global.throttle.ms": "1000", 23 | "global.history.records.max": "10000" 24 | } 25 | } 26 | 27 | # Enriched Page Views Example 28 | "config": { 29 | "connector.class": "io.mdrogalis.voluble.VolubleSourceConnector", 30 | 31 | "genv.pageview.page_id.with": "#{number.number_between '1','1000'}", 32 | "genv.pageview.user_id.with": "user.key", 33 | 34 | "genkp.user.with": "#{Code.isbn10}", 35 | "genv.user.name.with": "#{Name.full_name}", 36 | "genv.user.gender.with": "#{Demographic.sex}", 37 | "genv.user.favorite_beer.with": "#{Beer.name}", 38 | "genv.user.state.with": "#{Address.state}", 39 | "global.throttle.ms": "1000", 40 | "global.history.records.max": "10000" 41 | } 42 | 43 | # Ad Tech 44 | "config": { 45 | "connector.class": "io.mdrogalis.voluble.VolubleSourceConnector", 46 | 47 | "genv.impression.ad_id.with": "#{Code.isbn10}", 48 | "genv.impression.impression_time.with": "#{date.past '10','SECONDS'}", 49 | 50 | "genv.click.ad_id.matching": "impression.ad_id", 51 | "genv.click.click_time.with": "#{date.past '10','MILISECONDS'}", 52 | 53 | "global.throttle.ms": "1000", 54 | "global.history.records.max": "10000" 55 | } 56 | -------------------------------------------------------------------------------- /voluble/joinable.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "joinable", 3 | "config": { 4 | "connector.class": "io.mdrogalis.voluble.VolubleSourceConnector", 5 | 6 | "genkp.inventory.sometimes.with": "#{Code.asin}", 7 | "genkp.inventory.sometimes.matching": "inventory.key", 8 | "genv.inventory.amount_in_stock.with": "#{number.number_between '5','15'}", 9 | "genv.inventory.product_name.with": "#{Commerce.product_name}", 10 | "genv.inventory.last_updated.with": "#{date.past '10','SECONDS'}", 11 | 12 | "genkp.customer.with": "#{Code.isbn10}", 13 | "genv.customer.name.with": "#{Name.full_name}", 14 | "genv.customer.gender.with": "#{Demographic.sex}", 15 | "genv.customer.favorite_beer.with": "#{Beer.name}", 16 | "genv.customer.state.with": "#{Address.state}", 17 | 18 | "genkp.order.matching": "inventory.key", 19 | "genv.order.quantity.with": "#{number.number_between '1','5'}", 20 | "genv.order.customer_id.matching": "customer.key", 21 | 22 | "global.throttle.ms": "1000", 23 | "global.history.records.max": "10000" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /voluble/mdrogalis-voluble-0.3.1.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tmcgrath/kafka-connect-examples/2aa7ff7a6d10cd3016faf7fe8fa7be63a263bf06/voluble/mdrogalis-voluble-0.3.1.zip -------------------------------------------------------------------------------- /voluble/voluble-source.properties: -------------------------------------------------------------------------------- 1 | name=voluble-source 2 | connector.class=io.mdrogalis.voluble.VolubleSourceConnector 3 | 4 | genkp.owners.with=#{Internet.uuid} 5 | genv.owners.name->full.with=#{Name.full_name} 6 | genv.owners.creditCardNumber.with=#{Finance.credit_card} 7 | 8 | genk.cats.name.with=#{FunnyName.name} 9 | genv.cats.owner.matching=owners.key 10 | 11 | genk.diets.catName.matching=cats.key.name 12 | genv.diets.dish.with=#{Food.vegetables} 13 | genv.diets.measurement.with=#{Food.measurements} 14 | genv.diets.size.with=#{Food.measurement_sizes} 15 | 16 | genk.adopters.name.sometimes.with=#{Name.full_name} 17 | genk.adopters.name.sometimes.matching=adopters.key.name 18 | genv.adopters.jobTitle.with=#{Job.title} 19 | attrk.adopters.name.matching.rate=0.05 20 | topic.adopters.tombstone.rate=0.10 21 | 22 | global.throttle.ms=500 23 | global.history.records.max=100000 24 | -------------------------------------------------------------------------------- /voluble/voluble.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "voluble", 3 | "config": { 4 | "connector.class": "io.mdrogalis.voluble.VolubleSourceConnector", 5 | "genkp.owners.with": "#{Internet.uuid}", 6 | "genv.owners.name->full.with": "#{Name.full_name}", 7 | "genv.owners.creditCardNumber.with": "#{Finance.credit_card}", 8 | 9 | "genk.cats.name.with": "#{FunnyName.name}", 10 | "genv.cats.owner.matching": "owners.key", 11 | 12 | "genk.diets.catName.matching": "cats.key.name", 13 | "genv.diets.dish.with": "#{Food.vegetables}", 14 | "genv.diets.measurement.with": "#{Food.measurements}", 15 | "genv.diets.size.with": "#{Food.measurement_sizes}", 16 | 17 | "genk.adopters.name.sometimes.with": "#{Name.full_name}", 18 | "genk.adopters.name.sometimes.matching": "adopters.key.name", 19 | "genv.adopters.jobTitle.with": "#{Job.title}", 20 | "attrk.adopters.name.matching.rate": "0.05", 21 | "topic.adopters.tombstone.rate": "0.10", 22 | 23 | "global.throttle.ms": "500", 24 | "global.history.records.max": "100000" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /voluble/whizbang-java-0.3.1-jar-with-dependencies.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tmcgrath/kafka-connect-examples/2aa7ff7a6d10cd3016faf7fe8fa7be63a263bf06/voluble/whizbang-java-0.3.1-jar-with-dependencies.jar --------------------------------------------------------------------------------