├── docker-compose.yaml ├── es-skink.properties └── mysql.properties /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | 5 | mysql: 6 | image: wesleywillians/mysql-kafka-connect:latest 7 | command: --innodb-use-native-aio=0 8 | tty: true 9 | restart: always 10 | ports: 11 | - "33600:3306" 12 | environment: 13 | - MYSQL_DATABASE=products 14 | - MYSQL_ROOT_PASSWORD=root 15 | - MYSQL_USER=root 16 | extra_hosts: 17 | - "host.docker.internal:172.17.0.1" 18 | 19 | zookeeper: 20 | image: confluentinc/cp-zookeeper:latest 21 | environment: 22 | ZOOKEEPER_CLIENT_PORT: 2181 23 | 24 | kafka: 25 | image: confluentinc/cp-kafka:latest 26 | depends_on: 27 | - zookeeper 28 | ports: 29 | - "9092:9092" 30 | - "9094:9094" 31 | environment: 32 | KAFKA_BROKER_ID: 1 33 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 34 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 35 | KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL 36 | KAFKA_LISTENERS: INTERNAL://:9092,OUTSIDE://:9094 37 | KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,OUTSIDE://host.docker.internal:9094 38 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT 39 | extra_hosts: 40 | - "host.docker.internal:172.17.0.1" 41 | 42 | control-center: 43 | image: confluentinc/cp-enterprise-control-center:6.0.1 44 | hostname: control-center 45 | depends_on: 46 | - kafka 47 | ports: 48 | - "9021:9021" 49 | environment: 50 | CONTROL_CENTER_BOOTSTRAP_SERVERS: 'kafka:9092' 51 | CONTROL_CENTER_REPLICATION_FACTOR: 1 52 | CONTROL_CENTER_CONNECT_CLUSTER: http://kafka-connect:8083 53 | PORT: 9021 54 | extra_hosts: 55 | - "host.docker.internal:172.17.0.1" 56 | 57 | kafka-connect: 58 | image: confluentinc/cp-kafka-connect-base:6.0.0 59 | container_name: kafka-connect 60 | depends_on: 61 | - zookeeper 62 | - kafka 63 | ports: 64 | - 8083:8083 65 | environment: 66 | CONNECT_BOOTSTRAP_SERVERS: "kafka:9092" 67 | CONNECT_REST_PORT: 8083 68 | CONNECT_GROUP_ID: kafka-connect 69 | CONNECT_CONFIG_STORAGE_TOPIC: _connect-configs 70 | CONNECT_OFFSET_STORAGE_TOPIC: _connect-offsets 71 | CONNECT_STATUS_STORAGE_TOPIC: _connect-status 72 | CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter 73 | CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter 74 | CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" 75 | CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" 76 | CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect" 77 | CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO" 78 | CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" 79 | CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n" 80 | CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1" 81 | CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1" 82 | CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1" 83 | # # Optional settings to include to support Confluent Control Center 84 | # CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" 85 | # CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" 86 | # --------------- 87 | CONNECT_PLUGIN_PATH: /usr/share/java,/usr/share/confluent-hub-components,/data/connect-jars 88 | # If you want to use the Confluent Hub installer to d/l component, but make them available 89 | # when running this offline, spin up the stack once and then run : 90 | # docker cp kafka-connect:/usr/share/confluent-hub-components ./data/connect-jars 91 | volumes: 92 | - $PWD/data:/data 93 | # In the command section, $ are replaced with $$ to avoid the error 'Invalid interpolation format for "command" option' 94 | command: 95 | - bash 96 | - -c 97 | - | 98 | echo "Installing Connector" 99 | confluent-hub install --no-prompt debezium/debezium-connector-mysql:1.2.2 100 | confluent-hub install --no-prompt confluentinc/kafka-connect-elasticsearch:10.0.1 101 | # 102 | echo "Launching Kafka Connect worker" 103 | /etc/confluent/docker/run & 104 | # 105 | sleep infinity 106 | extra_hosts: 107 | - "host.docker.internal:172.17.0.1" 108 | 109 | es01: 110 | image: docker.elastic.co/elasticsearch/elasticsearch:7.11.2 111 | container_name: es01 112 | environment: 113 | - node.name=es01 114 | - cluster.name=es-docker-cluster 115 | - cluster.initial_master_nodes=es01 116 | - bootstrap.memory_lock=true 117 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 118 | ulimits: 119 | memlock: 120 | soft: -1 121 | hard: -1 122 | volumes: 123 | - ./es01:/usr/share/elasticsearch/data 124 | ports: 125 | - 9200:9200 126 | extra_hosts: 127 | - "host.docker.internal:172.17.0.1" 128 | 129 | kib01: 130 | image: docker.elastic.co/kibana/kibana:7.11.2 131 | container_name: kib01 132 | ports: 133 | - 5601:5601 134 | environment: 135 | ELASTICSEARCH_URL: http://es01:9200 136 | ELASTICSEARCH_HOSTS: '["http://es01:9200"]' 137 | extra_hosts: 138 | - "host.docker.internal:172.17.0.1" -------------------------------------------------------------------------------- /es-skink.properties: -------------------------------------------------------------------------------- 1 | name=elastisearch-sink 2 | topics=mysql-server.products.products 3 | connector.class=io.confluent.connect.elasticsearch.ElasticsearchSinkConnector 4 | connection.url=http://es01:9200 5 | type.name=_doc 6 | value.converter=org.apache.kafka.connect.json.JsonConverter 7 | value.converter.schemas.enable=false 8 | schema.ignore=true 9 | key.ignore=true -------------------------------------------------------------------------------- /mysql.properties: -------------------------------------------------------------------------------- 1 | name=mysql-connector 2 | connector.class=io.debezium.connector.mysql.MySqlConnector 3 | tasks.max=1 4 | database.user=root 5 | database.server.name=mysql-server 6 | database.hostname=mysql 7 | database.password=root 8 | database.history.kafka.bootstrap.servers=kafka:9092 9 | database.history.kafka.topic=history 10 | database.port=3306 --------------------------------------------------------------------------------