├── testOffsetMm2 ├── .gitignore ├── pom.xml └── src │ └── main │ └── java │ └── TestOffsetMm2.java ├── README.md ├── mm2_config └── mm2.properties └── docker-compose.yml /testOffsetMm2/.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /target 3 | *.iml -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kafka-docker-mm2 2 | Example of how to setup Kafka Docker multi cluster environment with MirrorMaker2 3 | 4 | Read the [Medium article](https://medium.com/larus-team/how-to-setup-mirrormaker-2-0-on-apache-kafka-multi-cluster-environment-87712d7997a4) and simply follow the provided step by step guide on how to use this repo 5 | -------------------------------------------------------------------------------- /testOffsetMm2/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | it.kafka.mm2 8 | TestOffsetMm2 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 11 13 | 11 14 | 2.8.0 15 | 16 | 17 | 18 | 19 | org.apache.kafka 20 | kafka-clients 21 | ${kafka.version} 22 | 23 | 24 | 25 | commons-io 26 | commons-io 27 | 2.6 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /mm2_config/mm2.properties: -------------------------------------------------------------------------------- 1 | # Kafka datacenters 2 | clusters=clusterA, clusterB 3 | clusterA.bootstrap.servers=broker1A:29092,broker2A:39092,broker3A:49092 4 | clusterB.bootstrap.servers=broker1B:29093,broker2B:29094,broker3B:29095 5 | 6 | #clusterA and clusterB configurations. Default value for the following settings is 3. 7 | #If you want more details about those internal configurations, please see https://docs.confluent.io/home/connect/userguide.html#kconnect-internal-topics 8 | #and https://docs.confluent.io/platform/current/connect/references/allconfigs.html#distributed-worker-configuration 9 | clusterA.config.storage.replication.factor=3 10 | clusterB.config.storage.replication.factor=3 11 | 12 | clusterA.offset.storage.replication.factor=3 13 | clusterB.offset.storage.replication.factor=3 14 | 15 | clusterA.status.storage.replication.factor=3 16 | clusterB.status.storage.replication.factor=3 17 | 18 | clusterA->clusterB.enabled=true 19 | clusterB->clusterA.enabled=true 20 | 21 | # MirrorMaker configuration. Default value for the following settings is 3 22 | offset-syncs.topic.replication.factor=3 23 | heartbeats.topic.replication.factor=3 24 | checkpoints.topic.replication.factor=3 25 | 26 | topics=.* 27 | groups=.* 28 | 29 | tasks.max=2 30 | replication.factor=3 31 | refresh.topics.enabled=true 32 | sync.topic.configs.enabled=true 33 | refresh.topics.interval.seconds=30 34 | 35 | topics.blacklist=.*[\-\.]internal, .*\.replica, __consumer_offsets 36 | groups.blacklist=console-consumer-.*, connect-.*, __.* 37 | 38 | # Enable heartbeats and checkpoints 39 | clusterA->clusterB.emit.heartbeats.enabled=true 40 | clusterA->clusterB.emit.checkpoints.enabled=true 41 | clusterB->clusterA.emit.heartbeats.enabled=true 42 | clusterB->clusterA.emit.checkpoints.enabled=true -------------------------------------------------------------------------------- /testOffsetMm2/src/main/java/TestOffsetMm2.java: -------------------------------------------------------------------------------- 1 | import org.apache.kafka.clients.consumer.Consumer; 2 | import org.apache.kafka.clients.consumer.ConsumerConfig; 3 | import org.apache.kafka.clients.consumer.ConsumerRecords; 4 | import org.apache.kafka.clients.consumer.KafkaConsumer; 5 | import org.apache.kafka.common.TopicPartition; 6 | import org.apache.kafka.common.serialization.StringDeserializer; 7 | 8 | import java.time.Duration; 9 | import java.util.Collections; 10 | import java.util.List; 11 | import java.util.Properties; 12 | 13 | public class TestOffsetMm2 { 14 | public static void main(String[] args) { 15 | 16 | final Properties props = new Properties(); 17 | 18 | /* 19 | Use one of the following urls depending on the the broker you want to connect to: 20 | * broker1A-> localhost:9092 21 | * broker2A-> localhost:9093 22 | * broker3A-> localhost:9094 23 | * clusterA-> localhost:9092,localhost:9093,localhost:9094 24 | * broker1B-> localhost:8092 25 | * broker2B-> localhost:8093 26 | * broker3B-> localhost:8094 27 | * clusterB-> localhost:8092,localhost:8093,localhost:8094 28 | */ 29 | props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); 30 | props.put(ConsumerConfig.GROUP_ID_CONFIG, "mm2_group"); 31 | props.put(ConsumerConfig.CLIENT_ID_CONFIG, "mm2_consumer_topic1"); 32 | props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 33 | props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 34 | props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); 35 | props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 36 | 37 | final Consumer consumer = new KafkaConsumer(props); 38 | 39 | /* 40 | Change topic name and partition accordingly to the topic you want to consume from 41 | */ 42 | TopicPartition partitionZero = new TopicPartition("topic1", 0); 43 | List partitions = Collections.singletonList(partitionZero); 44 | consumer.assign(partitions); 45 | consumer.seekToBeginning(partitions); 46 | 47 | 48 | final int giveUp = 100; 49 | int noRecordsCount = 0; 50 | 51 | while (true) { 52 | final ConsumerRecords consumerRecords = consumer.poll(Duration.ofMillis(1000)); 53 | 54 | if (consumerRecords.count() == 0) { 55 | noRecordsCount++; 56 | if (noRecordsCount > giveUp) 57 | break; 58 | else 59 | continue; 60 | } 61 | 62 | consumerRecords.forEach(record -> { 63 | 64 | System.out.println("Record Key " + record.key()); 65 | System.out.println("Record value " + record.value()); 66 | System.out.println("Record partition " + record.partition()); 67 | System.out.println("Record offset " + record.offset()); 68 | }); 69 | consumer.commitAsync(); 70 | } 71 | consumer.close(); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | 5 | zookeeperA: 6 | image: confluentinc/cp-zookeeper 7 | hostname: zookeeperA 8 | container_name: zookeeperA 9 | ports: 10 | - 22181:22181 11 | - 22888:22888 12 | - 23888:23888 13 | volumes: 14 | - ./zookeeperA/data:/data 15 | environment: 16 | ZOOKEEPER_SERVER_ID: 1 17 | ZOOKEEPER_CLIENT_PORT: 22181 18 | ZOOKEEPER_TICK_TIME: 2000 19 | ZOOKEEPER_INIT_LIMIT: 5 20 | ZOOKEEPER_SYNC_LIMIT: 2 21 | ZOOKEEPER_SERVERS: zookeeperA:22888:23888 22 | 23 | broker1A: 24 | image: confluentinc/cp-kafka 25 | hostname: broker1A 26 | container_name: broker1A 27 | ports: 28 | - 9092:9092 29 | - 29092:29092 30 | depends_on: 31 | - zookeeperA 32 | environment: 33 | KAFKA_BROKER_ID: 1 34 | KAFKA_ZOOKEEPER_CONNECT: zookeeperA:22181 35 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 36 | KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29092,PLAINTEXT_HOST://0.0.0.0:9092 37 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker1A:29092,PLAINTEXT_HOST://localhost:9092 38 | ALLOW_PLAINTEXT_LISTENER: 'yes' 39 | KAFKA_AUTO_OFFSET_RESET: "latest" 40 | 41 | broker2A: 42 | image: confluentinc/cp-kafka 43 | hostname: broker2A 44 | container_name: broker2A 45 | ports: 46 | - 9093:9093 47 | - 39092:39092 48 | depends_on: 49 | - zookeeperA 50 | environment: 51 | KAFKA_BROKER_ID: 2 52 | KAFKA_ZOOKEEPER_CONNECT: zookeeperA:22181 53 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 54 | KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:39092,PLAINTEXT_HOST://0.0.0.0:9093 55 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker2A:39092,PLAINTEXT_HOST://localhost:9093 56 | ALLOW_PLAINTEXT_LISTENER: 'yes' 57 | KAFKA_AUTO_OFFSET_RESET: "latest" 58 | 59 | broker3A: 60 | image: confluentinc/cp-kafka 61 | hostname: broker3A 62 | container_name: broker3A 63 | ports: 64 | - 9094:9094 65 | - 49092:49092 66 | depends_on: 67 | - zookeeperA 68 | environment: 69 | KAFKA_BROKER_ID: 3 70 | KAFKA_ZOOKEEPER_CONNECT: zookeeperA:22181 71 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 72 | KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:49092,PLAINTEXT_HOST://0.0.0.0:9094 73 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker3A:49092,PLAINTEXT_HOST://localhost:9094 74 | ALLOW_PLAINTEXT_LISTENER: 'yes' 75 | KAFKA_AUTO_OFFSET_RESET: "latest" 76 | 77 | zookeeperB: 78 | image: confluentinc/cp-zookeeper 79 | hostname: zookeeperB 80 | container_name: zookeeperB 81 | ports: 82 | - 32181:32181 83 | - 32888:32888 84 | - 33888:33888 85 | volumes: 86 | - ./zookeeperB/data:/data 87 | environment: 88 | ZOOKEEPER_SERVER_ID: 1 89 | ZOOKEEPER_CLIENT_PORT: 32181 90 | ZOOKEEPER_TICK_TIME: 2000 91 | ZOOKEEPER_INIT_LIMIT: 5 92 | ZOOKEEPER_SYNC_LIMIT: 2 93 | ZOOKEEPER_SERVERS: zookeeperB:32888:33888 94 | 95 | broker1B: 96 | image: confluentinc/cp-kafka 97 | hostname: broker1B 98 | container_name: broker1B 99 | ports: 100 | - 8092:8092 101 | - 29093:29093 102 | depends_on: 103 | - zookeeperB 104 | environment: 105 | KAFKA_BROKER_ID: 1 106 | KAFKA_ZOOKEEPER_CONNECT: zookeeperB:32181 107 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 108 | KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29093,PLAINTEXT_HOST://0.0.0.0:8092 109 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker1B:29093,PLAINTEXT_HOST://localhost:8092 110 | ALLOW_PLAINTEXT_LISTENER: 'yes' 111 | KAFKA_AUTO_OFFSET_RESET: "latest" 112 | 113 | broker2B: 114 | image: confluentinc/cp-kafka 115 | hostname: broker2B 116 | container_name: broker2B 117 | ports: 118 | - 8093:8093 119 | - 29094:29094 120 | depends_on: 121 | - zookeeperB 122 | environment: 123 | KAFKA_BROKER_ID: 2 124 | KAFKA_ZOOKEEPER_CONNECT: zookeeperB:32181 125 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 126 | KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29094,PLAINTEXT_HOST://0.0.0.0:8093 127 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker2B:29094,PLAINTEXT_HOST://localhost:8093 128 | ALLOW_PLAINTEXT_LISTENER: 'yes' 129 | KAFKA_AUTO_OFFSET_RESET: "latest" 130 | 131 | broker3B: 132 | image: confluentinc/cp-kafka 133 | hostname: broker3B 134 | container_name: broker3B 135 | ports: 136 | - 8094:8094 137 | - 29095:29095 138 | depends_on: 139 | - zookeeperB 140 | environment: 141 | KAFKA_BROKER_ID: 3 142 | KAFKA_ZOOKEEPER_CONNECT: zookeeperB:32181 143 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 144 | KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29095,PLAINTEXT_HOST://0.0.0.0:8094 145 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker3B:29095,PLAINTEXT_HOST://localhost:8094 146 | ALLOW_PLAINTEXT_LISTENER: 'yes' 147 | KAFKA_AUTO_OFFSET_RESET: "latest" 148 | 149 | mirror-maker: 150 | image: confluentinc/cp-kafka 151 | hostname: mirror-maker 152 | container_name: mirror-maker 153 | volumes: 154 | - ./mm2_config:/tmp/kafka/config 155 | ports: 156 | - 9091:9091 157 | - 29096:29096 158 | depends_on: 159 | - zookeeperA 160 | - zookeeperB 161 | - broker1A 162 | - broker2A 163 | - broker3A 164 | - broker1B 165 | - broker2B 166 | - broker3B 167 | environment: 168 | KAFKA_BROKER_ID: 4 169 | KAFKA_ZOOKEEPER_CONNECT: zookeeperA:22181,zookeeperB:32181 170 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 171 | KAFKA_LISTENERS: PLAINTEXT://:29096,PLAINTEXT_HOST://:9091 172 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://mirror-maker:29096,PLAINTEXT_HOST://localhost:9091 173 | ALLOW_PLAINTEXT_LISTENER: 'yes' 174 | KAFKA_AUTO_OFFSET_RESET: "latest" --------------------------------------------------------------------------------