>> pool;
96 |
97 | /**
98 | * Read/write lock for the pool's thread safety.
99 | */
100 | private final ReadWriteLock lock;
101 |
102 | /**
103 | * Read lock for the pool.
104 | */
105 | private final Lock readLock;
106 |
107 | /**
108 | * Write lock for the pool.
109 | */
110 | private final Lock writeLock;
111 |
112 | /**
113 | * Round-robin retrieval counter. For the sake of simplicity, a single rotator value is used although there may actually be
114 | * multiple producer arrays managed by the pool.
115 | */
116 | private final AtomicInteger producerRotation;
117 |
118 | /**
119 | * Pool shutdown indicator.
120 | */
121 | private boolean shutdown;
122 |
123 | /**
124 | * Creates a new producer pool.
125 | */
126 | public KafkaProducerPool() {
127 | this.pool = new HashMap<>();
128 | this.lock = new ReentrantReadWriteLock(true);
129 | this.readLock = lock.readLock();
130 | this.writeLock = lock.writeLock();
131 | this.producerRotation = new AtomicInteger(0);
132 | this.shutdown = false;
133 | }
134 |
135 | /**
136 | * Retrieves a {@link Producer} instance corresponding to the given {@link Properties}.
137 | *
138 | *
139 | * The following properties are set by default, and can be overridden by setting a different value in the passed {@code
140 | * properties}:
141 | *
142 | *
143 | * key | value
144 | * ================================================================
145 | * compression.type | lz4
146 | * acks | -1
147 | * retry.backoff.ms | 1000 ms
148 | * retries | 5
149 | * batch.size | 10485760 bytes
150 | * linger.ms | 50 ms
151 | * max.request.size | 104857600 bytes
152 | *
153 | *
154 | *
155 | * The {@link #KAFKA_PRODUCER_CONCURRENCY} property can be set to control the number of producers created
156 | * for each unique configuration.
157 | *
158 | *
159 | * NOTE: the returned producer must not be {@link Producer#close() closed}. Use the {@link #close()} method instead to
160 | * close all producers in the pool.
161 | *
162 | * @param properties
163 | * the properties for the producer
164 | * @return a Kafka message producer
165 | * @throws org.apache.kafka.common.KafkaException
166 | * if an error occurs creating the producer
167 | * @throws IllegalArgumentException
168 | * if properties is {@code null}
169 | * @throws IllegalStateException
170 | * if the pool has already been {@link #close() closed}
171 | * @throws NumberFormatException
172 | * if the value of the {@link #KAFKA_PRODUCER_CONCURRENCY} property cannot be parsed as an
173 | * integer.
174 | */
175 | public Producer getProducer(Properties properties) {
176 | if (properties == null) {
177 | throw new IllegalArgumentException("properties cannot be null");
178 | }
179 |
180 | // Include all default producer properties first, then add in the passed properties
181 | Properties producerProperties = new Properties();
182 | producerProperties.putAll(DEFAULT_PRODUCER_PROPERTIES);
183 | producerProperties.putAll(properties);
184 |
185 | List> producers = null;
186 | readLock.lock();
187 | try {
188 | if (shutdown) {
189 | throw new IllegalStateException("pool has already been shutdown");
190 | }
191 | producers = pool.get(producerProperties);
192 | } finally {
193 | readLock.unlock();
194 | }
195 |
196 | if (producers == null) {
197 | writeLock.lock();
198 | try {
199 | // Check shutdown status again, in case someone else has already shutdown the pool.
200 | if (shutdown) {
201 | throw new IllegalStateException("pool has already been shutdown");
202 | }
203 |
204 | // Check for existence again, in case someone else beat us here.
205 | if (pool.containsKey(producerProperties)) {
206 | producers = pool.get(producerProperties);
207 | } else {
208 | int producerConcurrency = getProducerConcurrency(producerProperties);
209 |
210 | // clone the properties in case creating the Producer mutates them (like happens
211 | // when using opentelemtry-agent) which breaks the cache key
212 | Properties originalProperties = (Properties) producerProperties.clone();
213 |
214 | // Create a new group of producers.
215 | producers = new ArrayList<>(producerConcurrency);
216 | for (int i = 0; i < producerConcurrency; ++i) {
217 | producers.add(createProducer(producerProperties));
218 | }
219 |
220 | pool.put(originalProperties, producers);
221 | }
222 | } finally {
223 | writeLock.unlock();
224 | }
225 | }
226 |
227 | // Return the next producer in the rotation. Make sure we correctly handle max int overflow
228 | // if invoked that many times in a long running process.
229 | return producers.get(Math.abs(getProducerRotation()) % producers.size());
230 | }
231 |
232 | // Visible for testing
233 | Producer createProducer(Properties properties) {
234 | return new KafkaProducer<>(properties);
235 | }
236 |
237 | // Visible for testing
238 | int getProducerRotation() {
239 | return producerRotation.getAndIncrement();
240 | }
241 |
242 | /**
243 | * Retrieves the {@link #KAFKA_PRODUCER_CONCURRENCY} from the given {@link Properties}
244 | *
245 | * @param props
246 | * the {@link Properties} used to configure the producer
247 | * @return the {@link #KAFKA_PRODUCER_CONCURRENCY} from the given {@link Properties}
248 | */
249 | private static int getProducerConcurrency(Properties props) {
250 | String producerConcurrencyProperty = props.getProperty(KAFKA_PRODUCER_CONCURRENCY,
251 | DEFAULT_KAFKA_PRODUCER_CONCURRENCY);
252 |
253 | int producerConcurrency = DEFAULT_KAFKA_PRODUCER_CONCURRENCY_INT;
254 |
255 | try {
256 | producerConcurrency = Integer.parseInt(producerConcurrencyProperty);
257 | } catch (NumberFormatException e) {
258 | if (LOGGER.isWarnEnabled())
259 | LOGGER.warn("Unable to parse [{}] config from value [{}]. Using default [{}] instead.",
260 | KAFKA_PRODUCER_CONCURRENCY, producerConcurrencyProperty,
261 | DEFAULT_KAFKA_PRODUCER_CONCURRENCY, e);
262 | }
263 |
264 | // Verify producer concurrency is valid
265 | if (producerConcurrency <= 0) {
266 | if (LOGGER.isWarnEnabled())
267 | LOGGER.warn("The value for config [{}] was <= 0 [{}]. Using default [{}] instead.",
268 | KAFKA_PRODUCER_CONCURRENCY, producerConcurrency,
269 | DEFAULT_KAFKA_PRODUCER_CONCURRENCY);
270 |
271 | producerConcurrency = DEFAULT_KAFKA_PRODUCER_CONCURRENCY_INT;
272 | }
273 |
274 | return producerConcurrency;
275 | }
276 |
277 | /**
278 | * Closes all {@link Producer producers} that have been produced by this pool.
279 | *
280 | * Any subsequent invocation of this method is ignored.
281 | *
282 | *
283 | * @throws IOException
284 | * if an error occurs during producer closure.
285 | *
286 | * {@inheritDoc}
287 | */
288 | @Override
289 | public void close() throws IOException {
290 | writeLock.lock();
291 | try {
292 | if (!shutdown) {
293 | shutdown = true;
294 | final Stream exceptions = pool.values().stream()
295 | .flatMap(Collection::stream)
296 | .flatMap(producer -> {
297 | try {
298 | producer.close();
299 | } catch (Exception e) {
300 | LOGGER.error("Could not close producer", e);
301 | return Stream.of(e);
302 | }
303 | return Stream.empty();
304 | });
305 |
306 | // Throw exception if any of the producers in the pool could not be closed.
307 | final Optional exception = exceptions.findFirst();
308 | if (exception.isPresent()) {
309 | throw new IOException(exception.get());
310 | }
311 | }
312 | } finally {
313 | writeLock.unlock();
314 | }
315 | }
316 | }
317 |
--------------------------------------------------------------------------------
/common-kafka/src/main/java/com/cerner/common/kafka/producer/KafkaProducerWrapper.java:
--------------------------------------------------------------------------------
1 | package com.cerner.common.kafka.producer;
2 |
3 | import com.cerner.common.kafka.KafkaExecutionException;
4 | import com.yammer.metrics.Metrics;
5 | import com.yammer.metrics.core.Histogram;
6 | import com.yammer.metrics.core.Timer;
7 | import com.yammer.metrics.core.TimerContext;
8 | import org.apache.kafka.clients.producer.Producer;
9 | import org.apache.kafka.clients.producer.ProducerRecord;
10 | import org.apache.kafka.clients.producer.RecordMetadata;
11 | import org.slf4j.Logger;
12 | import org.slf4j.LoggerFactory;
13 |
14 | import java.io.Closeable;
15 | import java.io.Flushable;
16 | import java.io.IOException;
17 | import java.util.Collections;
18 | import java.util.LinkedList;
19 | import java.util.List;
20 | import java.util.concurrent.ExecutionException;
21 | import java.util.concurrent.Future;
22 | import java.util.stream.Collectors;
23 |
24 | /**
25 | * Wraps around an instance of a {@link Producer producer}. {@link #close() Closing} the wrapper will
26 | * not affect the wrapped producer instance. The creators of the wrapped producer instances are still
27 | * responsible for closing the producer when appropriate.
28 | *
29 | * Supports synchronously sending {@link ProducerRecord records}.
30 | *
31 | * Example usage:
32 | *
33 | * Properties producerProps = new Properties();
34 | * // required properties to construct a KafkaProducer instance
35 | * // constants for these config keys exist in org.apache.kafka.clients.producer.ProducerConfig
36 | * producerProps.put("bootstrap.servers", "localhost:4242");
37 | * producerProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
38 | * producerProps.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
39 | *
40 | * // properties that affect batching
41 | *
42 | * // linger.ms is used to tell the producer to wait this amount of time
43 | * // for more records to fill the buffer before sending. The default value of
44 | * // 0 will tell the producer to start sending immediately and to not wait.
45 | * // {@link #sendSynchronously(List)} will attempt to send the entire batch at once
46 | * // but could be affected by this value if it is not set, or set too low. This could
47 | * // result in more I/O operations taking place than what is desired. It is recommended
48 | * // to set this value appropriately high as to not affect sending in batches.
49 | * producerProps.put("linger.ms", 10000);
50 | *
51 | * // batch.size will tell the producer how many records to hold in memory, in bytes,
52 | * // before it should send its buffer. This value will include both the key and the payload.
53 | * // It is recommended to set this value at the value of 'max.message.bytes' * desired batch (in number of records) size
54 | * // to ensure that all records sent to {@link #sendSynchronously(List)} are batched together.
55 | * // NOTE: the setting of 'linger.ms' will supersede this value.
56 | * producerProps.put("batch.size", 16384);
57 | *
58 | * KafkaProducerWrapper<String, String> producerWrapper = new KafkaProducerWrapper<>(
59 | * new KafkaProducer<>(producerProps);
60 | *
61 | * String topic = "myTopic";
62 | * String key = "myKey";
63 | * String value = "myValue";
64 | *
65 | * ProducerRecord<String, String> record = new ProducerRecord<>(topic, key, value);
66 | *
67 | * // synchronously sends the specified record; the underlying KafkaProducer will be flushed
68 | * // immediately and the record will be sent.
69 | * producerWrapper.sendSynchronously(record);
70 | *
71 | * // alternatively, a collection of ProducerRecords can be submitted at a time to
72 | * // the sendSynchronous method and it will behave the same way and block until all messages
73 | * // have been submitted. If a KafkaExecutionException is thrown the context as to which
74 | * // message in the collection caused the failure will not be included. See the
75 | * // javadoc for KafkaExecutionException for more information about the potential errors
76 | *
77 | * String topic = "mySecondTopic";
78 | * String key = "mySecondKey";
79 | * String value = "mySecondValue";
80 | *
81 | * ProducerRecord<String, String> secondRecord = new ProducerRecord<>(topic, key, value);
82 | * List<ProducerRecord<String, String>> records = new ArrayList<>();
83 | * records.add(record);
84 | * records.add(secondRecord);
85 | *
86 | * producerWrapper.sendSynchronously(records);
87 | *
88 | * // when the producerWrapper is done being used call #close(), which will close
89 | * // the underlying KafkaProducer. NOTE: be aware of how the wrapper instance is being
90 | * // managed, as closing it may cause issues if the instance is being used
91 | * // across multiple threads.
92 | * producerWrapper.close();
93 | *
94 | *
95 | * The wrapper also supports the idea of sending incrementally instead of as a large collection. The synchronous
96 | * sending behavior can still be supported but feedback about a failure will not be provided until the wrapper has
97 | * been {@link #flush() flushed}. Similar to the description above the "linger.ms" and "batch.size" should be
98 | * configured appropriately for your desired throughput and latency.
99 | *
100 | *
101 | * KafkaProducerWrapper<String, String> producerWrapper = new KafkaProducerWrapper<>(
102 | * new KafkaProducer<>(producerProps);
103 | *
104 | * producerWrapper.send(new ProducerRecord<>(topic, key1, value1));
105 | * producerWrapper.send(new ProducerRecord<>(topic, key2, value2));
106 | * producerWrapper.send(new ProducerRecord<>(topic, key3, value3));
107 | *
108 | * //will block and return only when all sent events have been successfully flushed or an error has occurred.
109 | * try{}
110 | * producerWrapper.flush();
111 | * }catch(IOException ioe){
112 | * //an error occurred and all pending sent messages should be expected to be resent or handled
113 | * //appropriately.
114 | * }finally{
115 | * producerWrapper.close();
116 | * }
117 | *
118 | *
119 | *
120 | * This class is not thread safe.
121 | *
122 | * @author Stephen Durfey
123 | */
124 | public class KafkaProducerWrapper implements Closeable, Flushable {
125 |
126 | private static final Logger LOGGER = LoggerFactory.getLogger(KafkaProducerWrapper.class);
127 |
128 | private final Producer kafkaProducer;
129 |
130 | private final List> pendingWrites;
131 |
132 | /**
133 | * A timer for {@link #send(ProducerRecord)} calls
134 | */
135 | static final Timer SEND_TIMER = Metrics.newTimer(KafkaProducerWrapper.class, "send");
136 |
137 | /**
138 | * A timer for {@link #sendSynchronously(ProducerRecord)} or {@link #sendSynchronously(List)} calls
139 | */
140 | static final Timer SYNC_SEND_TIMER = Metrics.newTimer(KafkaProducerWrapper.class, "send-synchronously");
141 |
142 | /**
143 | * A timer for {@link #flush()} calls
144 | */
145 | static final Timer FLUSH_TIMER = Metrics.newTimer(KafkaProducerWrapper.class, "flush");
146 |
147 | /**
148 | * A histogram tracking batch size, updated by {@link #sendSynchronously(List)} or {@link #flush()}
149 | */
150 | static final Histogram BATCH_SIZE_HISTOGRAM = Metrics.newHistogram(KafkaProducerWrapper.class, "batch-size", true);
151 |
152 | /**
153 | * Creates an instance to manage interacting with the {@code kafkaProducer}.
154 | *
155 | * @param kafkaProducer
156 | * the {@link Producer} to interact with
157 | */
158 | public KafkaProducerWrapper(Producer kafkaProducer) {
159 | if (kafkaProducer == null) {
160 | throw new IllegalStateException("kafkaProducer cannot be null");
161 | }
162 |
163 | this.kafkaProducer = kafkaProducer;
164 | pendingWrites = new LinkedList<>();
165 | }
166 |
167 | /**
168 | * Sends the specified {@code record}. Data is not guaranteed to be written until {@link #flush()} is called.
169 | *
170 | * @param record the record to send to Kafka
171 | * @throws IllegalArgumentException the {@code record} cannot be {@code null}.
172 | * @throws org.apache.kafka.common.KafkaException if there is an issue sending the message to Kafka
173 | */
174 | public void send(ProducerRecord record){
175 | if(record == null){
176 | throw new IllegalArgumentException("The 'record' cannot be 'null'.");
177 | }
178 |
179 | TimerContext context = SEND_TIMER.time();
180 | try {
181 | pendingWrites.add(kafkaProducer.send(record));
182 | } finally {
183 | context.stop();
184 | }
185 | }
186 |
187 | /**
188 | * Synchronously sends the {@code record}. The underlying {@link Producer} is immediately flushed and the call will block until
189 | * the {@code record} is sent.
190 | *
191 | * @param record
192 | * the records to send to Kafka
193 | * @throws IOException
194 | * indicates that there was a Kafka error that led to the message not being sent.
195 | * @throws org.apache.kafka.common.KafkaException
196 | * if there is an issue sending the message to Kafka
197 | */
198 | public void sendSynchronously(ProducerRecord record) throws IOException {
199 | sendSynchronously(Collections.singletonList(record));
200 | }
201 |
202 | /**
203 | * Synchronously sends all {@code records}. The underlying {@link Producer} is immediately flushed and the call will block until
204 | * the {@code records} have been sent.
205 | *
206 | * @param records
207 | * the records to send to Kafka
208 | * @throws IOException
209 | * indicates that there was a Kafka error that lead to one of the messages not being sent.
210 | * @throws org.apache.kafka.common.KafkaException
211 | * if there is an issue sending the messages to Kafka
212 | */
213 | public void sendSynchronously(List> records) throws IOException {
214 | // Disregard empty batches.
215 | if (records.isEmpty()) {
216 | LOGGER.debug("records was empty; nothing to process");
217 | return;
218 | }
219 |
220 | BATCH_SIZE_HISTOGRAM.update(records.size());
221 |
222 | TimerContext context = SYNC_SEND_TIMER.time();
223 | try {
224 | List> futures = records.stream().map(kafkaProducer::send).collect(Collectors.toList());
225 |
226 | handlePendingWrites(futures);
227 | } finally {
228 | context.stop();
229 | }
230 | }
231 |
232 | /**
233 | * Flushes the underlying pending writes created by calls to {@link #send(ProducerRecord)}
234 | * to the {@link Producer} ensuring they persisted.
235 | *
236 | * If an exception occurs when flushing, all pending writes should be
237 | * {@link #send(ProducerRecord) sent again}.
238 | *
239 | * @throws IOException If there was an error persisting one of the pending writes
240 | */
241 | @Override
242 | public void flush() throws IOException{
243 | if (pendingWrites.isEmpty()) {
244 | LOGGER.debug("nothing to flush");
245 | return;
246 | }
247 |
248 | BATCH_SIZE_HISTOGRAM.update(pendingWrites.size());
249 |
250 | TimerContext context = FLUSH_TIMER.time();
251 | try {
252 | handlePendingWrites(pendingWrites);
253 | } finally{
254 | pendingWrites.clear();
255 | context.stop();
256 | }
257 | }
258 |
259 | private void handlePendingWrites(List> pendingWrites) throws IOException{
260 | // Future#get will sit and wait until 'linger.ms' has been reached
261 | // or flush is called, so flush here.
262 | try {
263 | kafkaProducer.flush();
264 | }catch(RuntimeException re){
265 | //Exception isn't retriable so wrap in known exception which represents fatal.
266 | throw new IOException("Unable to flush producer.",re);
267 | }
268 | for (final Future future : pendingWrites) {
269 | try {
270 | future.get();
271 | } catch (InterruptedException e) {
272 | throw new IOException(e);
273 | } catch (ExecutionException e) {
274 | throw new KafkaExecutionException(e);
275 | }
276 | }
277 | }
278 |
279 | @Override
280 | public void close() throws IOException {
281 | LOGGER.debug("Closing the producer wrapper and flushing outstanding writes.");
282 | flush();
283 | }
284 | }
--------------------------------------------------------------------------------
/common-kafka/src/main/java/com/cerner/common/kafka/producer/partitioners/FairPartitioner.java:
--------------------------------------------------------------------------------
1 | package com.cerner.common.kafka.producer.partitioners;
2 |
3 | import org.apache.kafka.clients.producer.Partitioner;
4 | import org.apache.kafka.common.Cluster;
5 | import org.apache.kafka.common.PartitionInfo;
6 | import org.slf4j.Logger;
7 | import org.slf4j.LoggerFactory;
8 |
9 | import java.util.List;
10 | import java.util.Map;
11 | import java.util.Random;
12 |
13 | /**
14 | * A basic default {@link Partitioner} implementation that uses a combination of {@link #getTimeHash() time-based} and
15 | * {@link #getKeyHash(Object) key-based} hashing to determine the target partition for a produced message. Additionally,
16 | * {@link Cluster#availablePartitionsForTopic(String) available} partitions are always preferred over non-available partitions.
17 | *
18 | * @author A. Olson
19 | */
20 | public class FairPartitioner implements Partitioner {
21 |
22 | /**
23 | * Logger.
24 | */
25 | private static final Logger LOG = LoggerFactory.getLogger(FairPartitioner.class);
26 |
27 | /**
28 | * Number of milliseconds between rotations of partition assignments.
29 | */
30 | public static final int ROTATE_MILLIS = 250;
31 |
32 | /**
33 | * Randomized clock skew to distribute load when there are multiple concurrent writers.
34 | */
35 | public static final int ROTATE_OFFSET = ROTATE_MILLIS * new Random().nextInt(Short.MAX_VALUE);
36 |
37 | @Override
38 | public void configure(Map configs) {
39 | //no-op because not configuring anything
40 | }
41 |
42 | /**
43 | * Returns a hash value roughly based on the current time. By default the returned value is incremented every
44 | * {@link #ROTATE_MILLIS} milliseconds, so that all messages produced in a single batch are very likely to be written
45 | * to the same partition.
46 | *
47 | * @return the time hash
48 | */
49 | protected int getTimeHash() {
50 | // Return a temporarily sticky value.
51 | return (int) ((System.currentTimeMillis() + ROTATE_OFFSET) / ROTATE_MILLIS);
52 | }
53 |
54 | /**
55 | * Returns a hash value based on the message key. By default the return value is 0 so that only {@link #getTimeHash()} is
56 | * used to determine which partition the message is written to.
57 | *
58 | * @param key
59 | * the message key
60 | * @return the message key hash
61 | */
62 | protected int getKeyHash(@SuppressWarnings("unused") Object key) {
63 | return 0;
64 | }
65 |
66 | @Override
67 | public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
68 | // Combine time and key hashes, then map to a partition.
69 | return getPartition(topic, cluster, getTimeHash() + getKeyHash(key));
70 | }
71 |
72 | @Override
73 | public void close() {
74 | //no-op because not maintaining resources
75 | }
76 |
77 | private static int getPartition(String topic, Cluster cluster, int hash) {
78 | List partitions = cluster.availablePartitionsForTopic(topic);
79 | if (partitions.isEmpty()) {
80 | LOG.warn("No available partitions for {} therefore using total partition count for calculations.", topic);
81 | partitions = cluster.partitionsForTopic(topic);
82 | }
83 |
84 | int index = Math.abs(hash) % partitions.size();
85 | return partitions.get(index).partition();
86 | }
87 | }
--------------------------------------------------------------------------------
/common-kafka/src/test/java/com/cerner/common/kafka/KafkaTests.java:
--------------------------------------------------------------------------------
1 | package com.cerner.common.kafka;
2 |
3 | import com.cerner.common.kafka.consumer.ProcessingKafkaConsumerITest;
4 | import com.cerner.common.kafka.producer.KafkaProducerPoolTest;
5 | import com.cerner.common.kafka.producer.KafkaProducerWrapperTest;
6 | import com.cerner.common.kafka.testing.AbstractKafkaTests;
7 | import org.junit.platform.suite.api.SelectClasses;
8 | import org.junit.platform.suite.api.Suite;
9 |
10 | /**
11 | * Suite of tests requiring an internal Kafka + Zookeeper cluster to be started up.
12 | *
13 | * IMPORTANT: New tests added to this project requiring Kafka and/or Zookeeper should be added to this suite so they will be
14 | * executed as part of the build.
15 | */
16 | @Suite
17 | @SelectClasses({
18 | // com.cerner.common.kafka.consumer
19 | ProcessingKafkaConsumerITest.class,
20 |
21 | // com.cerner.common.kafka.producer
22 | KafkaProducerPoolTest.class, KafkaProducerWrapperTest.class
23 | })
24 | public class KafkaTests extends AbstractKafkaTests {
25 | }
26 |
--------------------------------------------------------------------------------
/common-kafka/src/test/java/com/cerner/common/kafka/StandaloneTests.java:
--------------------------------------------------------------------------------
1 | package com.cerner.common.kafka;
2 |
3 | import com.cerner.common.kafka.consumer.ConsumerOffsetClientTest;
4 | import com.cerner.common.kafka.consumer.ProcessingConfigTest;
5 | import com.cerner.common.kafka.consumer.ProcessingKafkaConsumerTest;
6 | import com.cerner.common.kafka.consumer.ProcessingPartitionTest;
7 | import com.cerner.common.kafka.consumer.assignors.FairAssignorTest;
8 | import com.cerner.common.kafka.producer.partitioners.FairPartitionerTest;
9 | import org.junit.platform.suite.api.SelectClasses;
10 | import org.junit.platform.suite.api.Suite;
11 |
12 | /**
13 | * Suite of tests that can be run independently.
14 | *
15 | * IMPORTANT: New tests added to this project not requiring cluster services should be added to this suite so they will be
16 | * executed as part of the build.
17 | */
18 | @Suite
19 | @SelectClasses({
20 | // com.cerner.common.kafka.consumer
21 | ConsumerOffsetClientTest.class, ProcessingConfigTest.class, ProcessingKafkaConsumerTest.class,
22 | ProcessingPartitionTest.class,
23 |
24 | // com.cerner.common.kafka.consumer.assignors
25 | FairAssignorTest.class,
26 |
27 | // com.cerner.common.kafka.producer.partitioners
28 | FairPartitionerTest.class
29 | })
30 | public class StandaloneTests {
31 | }
--------------------------------------------------------------------------------
/common-kafka/src/test/java/com/cerner/common/kafka/consumer/ProcessingConfigTest.java:
--------------------------------------------------------------------------------
1 | package com.cerner.common.kafka.consumer;
2 |
3 | import static org.hamcrest.core.Is.is;
4 | import static org.hamcrest.MatcherAssert.assertThat;
5 | import static org.junit.jupiter.api.Assertions.assertThrows;
6 |
7 | import java.util.Map;
8 | import java.util.Properties;
9 |
10 | import org.apache.kafka.clients.consumer.ConsumerConfig;
11 | import org.apache.kafka.clients.consumer.OffsetResetStrategy;
12 | import org.junit.jupiter.api.BeforeEach;
13 | import org.junit.jupiter.api.Test;
14 |
15 | public class ProcessingConfigTest {
16 |
17 | private Properties properties;
18 | private ProcessingConfig config;
19 |
20 | @BeforeEach
21 | public void before() {
22 | properties = new Properties();
23 | properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, OffsetResetStrategy.EARLIEST.toString().toLowerCase());
24 | config = new ProcessingConfig(properties);
25 | }
26 |
27 | @Test
28 | public void constructor_nullProperties() {
29 | assertThrows(IllegalArgumentException.class,
30 | () -> new ProcessingConfig(null));
31 | }
32 |
33 | @Test
34 | public void constructor_offsetResetStrategyNotProvided() {
35 | properties.remove(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG);
36 | assertThrows(IllegalArgumentException.class,
37 | () -> new ProcessingConfig(properties));
38 | }
39 |
40 | @Test
41 | public void constructor_offsetResetStrategySetToNone() {
42 | properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none");
43 | assertThrows(IllegalArgumentException.class,
44 | () -> new ProcessingConfig(properties));
45 | }
46 |
47 | @Test
48 | public void constructor_commitInitialOffsetNotBoolean() {
49 | properties.setProperty(ProcessingConfig.COMMIT_INITIAL_OFFSET_PROPERTY, "not_boolean");
50 | assertThrows(IllegalArgumentException.class,
51 | () -> new ProcessingConfig(properties));
52 | }
53 |
54 | @Test
55 | public void constructor_commitSizeEqualsZero() {
56 | properties.setProperty(ProcessingConfig.COMMIT_SIZE_THRESHOLD_PROPERTY, "0");
57 | assertThrows(IllegalArgumentException.class,
58 | () -> new ProcessingConfig(properties));
59 | }
60 |
61 | @Test
62 | public void constructor_commitSizeLessThanZero() {
63 | properties.setProperty(ProcessingConfig.COMMIT_SIZE_THRESHOLD_PROPERTY, "-1");
64 | assertThrows(IllegalArgumentException.class,
65 | () -> new ProcessingConfig(properties));
66 | }
67 |
68 | @Test
69 | public void constructor_commitSizeNotANumber() {
70 | properties.setProperty(ProcessingConfig.COMMIT_SIZE_THRESHOLD_PROPERTY, "notANumber");
71 | assertThrows(IllegalArgumentException.class,
72 | () -> new ProcessingConfig(properties));
73 | }
74 |
75 | @Test
76 | public void constructor_commitTimeLessThanZero() {
77 | properties.setProperty(ProcessingConfig.COMMIT_TIME_THRESHOLD_PROPERTY, "-1");
78 | assertThrows(IllegalArgumentException.class,
79 | () -> new ProcessingConfig(properties));
80 | }
81 |
82 | @Test
83 | public void constructor_commitTimeNotANumber() {
84 | properties.setProperty(ProcessingConfig.COMMIT_TIME_THRESHOLD_PROPERTY, "notANumber");
85 | assertThrows(IllegalArgumentException.class,
86 | () -> new ProcessingConfig(properties));
87 | }
88 |
89 | @Test
90 | public void constructor_pauseThresholdLessThanZero() {
91 | properties.setProperty(ProcessingConfig.FAIL_THRESHOLD_PROPERTY, "-1");
92 | assertThrows(IllegalArgumentException.class,
93 | () -> new ProcessingConfig(properties));
94 | }
95 |
96 | @Test
97 | public void constructor_pauseThresholdGreaterThanOne() {
98 | properties.setProperty(ProcessingConfig.FAIL_THRESHOLD_PROPERTY, "2");
99 | assertThrows(IllegalArgumentException.class,
100 | () -> new ProcessingConfig(properties));
101 | }
102 |
103 | @Test
104 | public void constructor_pauseThresholdEqualsZero() {
105 | properties.setProperty(ProcessingConfig.FAIL_THRESHOLD_PROPERTY, "0");
106 | config = new ProcessingConfig(properties);
107 | assertThat(config.getFailThreshold(), is(0.0));
108 | }
109 |
110 | @Test
111 | public void constructor_pauseThresholdEqualsOne() {
112 | properties.setProperty(ProcessingConfig.FAIL_THRESHOLD_PROPERTY, "1");
113 | config = new ProcessingConfig(properties);
114 | assertThat(config.getFailThreshold(), is(1.0));
115 | }
116 |
117 | @Test
118 | public void constructor_pauseThresholdNotANumber() {
119 | properties.setProperty(ProcessingConfig.FAIL_THRESHOLD_PROPERTY, "notANumber");
120 | assertThrows(IllegalArgumentException.class,
121 | () -> new ProcessingConfig(properties));
122 | }
123 |
124 | @Test
125 | public void constructor_pauseSampleSizeEqualsZero() {
126 | properties.setProperty(ProcessingConfig.FAIL_SAMPLE_SIZE_PROPERTY, "0");
127 | assertThrows(IllegalArgumentException.class,
128 | () -> new ProcessingConfig(properties));
129 | }
130 |
131 | @Test
132 | public void constructor_pauseSampleSizeLessThanZero() {
133 | properties.setProperty(ProcessingConfig.FAIL_SAMPLE_SIZE_PROPERTY, "-1");
134 | assertThrows(IllegalArgumentException.class,
135 | () -> new ProcessingConfig(properties));
136 | }
137 |
138 | @Test
139 | public void constructor_pauseSampleSizeNotANumber() {
140 | properties.setProperty(ProcessingConfig.FAIL_SAMPLE_SIZE_PROPERTY, "notANumber");
141 | assertThrows(IllegalArgumentException.class,
142 | () -> new ProcessingConfig(properties));
143 | }
144 |
145 | @Test
146 | public void constructor_pauseTimeLessThanZero() {
147 | properties.setProperty(ProcessingConfig.FAIL_PAUSE_TIME_PROPERTY, "-1");
148 | assertThrows(IllegalArgumentException.class,
149 | () -> new ProcessingConfig(properties));
150 | }
151 |
152 | @Test
153 | public void constructor_pauseTimeNotANumber() {
154 | properties.setProperty(ProcessingConfig.FAIL_PAUSE_TIME_PROPERTY, "notANumber");
155 | assertThrows(IllegalArgumentException.class,
156 | () -> new ProcessingConfig(properties));
157 | }
158 |
159 | @Test
160 | public void constructor_maxPollIntervalNotANumber() {
161 | properties.setProperty(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, "notANumber");
162 | assertThrows(IllegalArgumentException.class,
163 | () -> new ProcessingConfig(properties));
164 | }
165 |
166 | @Test
167 | public void constructor_defaults() {
168 | assertThat(config.getCommitInitialOffset(), is(true));
169 | assertThat(config.getCommitSizeThreshold(), is(Long.parseLong(ProcessingConfig.COMMIT_SIZE_THRESHOLD_DEFAULT)));
170 | assertThat(config.getCommitTimeThreshold(), is(Long.parseLong(ProcessingConfig.COMMIT_TIME_THRESHOLD_DEFAULT)));
171 | assertThat(config.getFailPauseTime(), is(Long.parseLong(ProcessingConfig.FAIL_PAUSE_TIME_DEFAULT)));
172 | assertThat(config.getFailSampleSize(), is(Integer.parseInt(ProcessingConfig.FAIL_SAMPLE_SIZE_DEFAULT)));
173 | assertThat(config.getFailThreshold(), is(Double.parseDouble(ProcessingConfig.FAIL_THRESHOLD_DEFAULT)));
174 | assertThat(config.getOffsetResetStrategy(), is(OffsetResetStrategy.EARLIEST));
175 | assertThat(config.getMaxPollInterval(), is(300000L));
176 |
177 | // config properties should at least contain everything from properties
178 | for(Map.Entry entry : properties.entrySet()) {
179 | assertThat(config.getProperties().get(entry.getKey()), is(entry.getValue()));
180 | }
181 |
182 | assertThat(config.getProperties().getProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), is(Boolean.FALSE.toString()));
183 | }
184 |
185 | @Test
186 | public void constructor_customProperties() {
187 | properties.setProperty(ProcessingConfig.COMMIT_INITIAL_OFFSET_PROPERTY, String.valueOf(false));
188 | properties.setProperty(ProcessingConfig.COMMIT_SIZE_THRESHOLD_PROPERTY, "123");
189 | properties.setProperty(ProcessingConfig.COMMIT_TIME_THRESHOLD_PROPERTY, "234");
190 | properties.setProperty(ProcessingConfig.FAIL_PAUSE_TIME_PROPERTY, "456");
191 | properties.setProperty(ProcessingConfig.FAIL_SAMPLE_SIZE_PROPERTY, "567");
192 | properties.setProperty(ProcessingConfig.FAIL_THRESHOLD_PROPERTY, "0.1");
193 | properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, OffsetResetStrategy.LATEST.toString().toLowerCase());
194 | properties.setProperty(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, "10000");
195 |
196 | config = new ProcessingConfig(properties);
197 |
198 | assertThat(config.getCommitInitialOffset(), is(false));
199 | assertThat(config.getCommitSizeThreshold(), is(123L));
200 | assertThat(config.getCommitTimeThreshold(), is(234L));
201 | assertThat(config.getFailPauseTime(), is(456L));
202 | assertThat(config.getFailSampleSize(), is(567));
203 | assertThat(config.getFailThreshold(), is(0.1));
204 | assertThat(config.getOffsetResetStrategy(), is(OffsetResetStrategy.LATEST));
205 | assertThat(config.getMaxPollInterval(), is(10000L));
206 |
207 | // config properties should at least contain everything from properties
208 | for(Map.Entry entry : properties.entrySet()) {
209 | assertThat(config.getProperties().get(entry.getKey()), is(entry.getValue()));
210 | }
211 |
212 | assertThat(config.getProperties().getProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), is(Boolean.FALSE.toString()));
213 | }
214 |
215 | }
216 |
--------------------------------------------------------------------------------
/common-kafka/src/test/java/com/cerner/common/kafka/consumer/assignors/FairAssignorTest.java:
--------------------------------------------------------------------------------
1 | package com.cerner.common.kafka.consumer.assignors;
2 |
3 | import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
4 | import org.apache.kafka.common.TopicPartition;
5 | import org.junit.jupiter.api.Test;
6 |
7 | import java.util.Arrays;
8 | import java.util.Collections;
9 | import java.util.HashMap;
10 | import java.util.List;
11 | import java.util.Map;
12 |
13 | import static org.hamcrest.MatcherAssert.assertThat;
14 | import static org.hamcrest.core.Is.is;
15 | import static org.junit.jupiter.api.Assertions.assertEquals;
16 |
17 |
18 | public class FairAssignorTest {
19 |
20 | private FairAssignor assignor = new FairAssignor();
21 |
22 | @Test
23 | public void testOneConsumerNoTopic() {
24 | String consumerId = "consumer";
25 |
26 | Map partitionsPerTopic = new HashMap<>();
27 |
28 | Map> assignment = assignor.assign(partitionsPerTopic,
29 | Collections.singletonMap(consumerId, new ConsumerPartitionAssignor.Subscription(Collections.emptyList())));
30 | assertEquals(Collections.singleton(consumerId), assignment.keySet());
31 | assertThat(assignment.get(consumerId).isEmpty(), is(true));
32 | }
33 |
34 | @Test
35 | public void testOneConsumerNonexistentTopic() {
36 | String topic = "topic";
37 | String consumerId = "consumer";
38 |
39 | Map partitionsPerTopic = new HashMap<>();
40 | partitionsPerTopic.put(topic, 0);
41 |
42 | Map> assignment = assignor.assign(partitionsPerTopic,
43 | Collections.singletonMap(consumerId, new ConsumerPartitionAssignor.Subscription(Collections.singletonList(topic))));
44 |
45 | assertEquals(Collections.singleton(consumerId), assignment.keySet());
46 | assertThat(assignment.get(consumerId).isEmpty(), is(true));
47 | }
48 |
49 | @Test
50 | public void testOneConsumerOneTopic() {
51 | String topic = "topic";
52 | String consumerId = "consumer";
53 |
54 | Map partitionsPerTopic = new HashMap<>();
55 | partitionsPerTopic.put(topic, 3);
56 |
57 | Map> assignment = assignor.assign(partitionsPerTopic,
58 | Collections.singletonMap(consumerId, new ConsumerPartitionAssignor.Subscription(Collections.singletonList(topic))));
59 | assertEquals(Arrays.asList(
60 | new TopicPartition(topic, 0),
61 | new TopicPartition(topic, 1),
62 | new TopicPartition(topic, 2)), assignment.get(consumerId));
63 | }
64 |
65 | @Test
66 | public void testOnlyAssignsPartitionsFromSubscribedTopics() {
67 | String topic = "topic";
68 | String otherTopic = "other";
69 | String consumerId = "consumer";
70 |
71 | Map partitionsPerTopic = new HashMap<>();
72 | partitionsPerTopic.put(topic, 3);
73 | partitionsPerTopic.put(otherTopic, 3);
74 |
75 | Map> assignment = assignor.assign(partitionsPerTopic,
76 | Collections.singletonMap(consumerId, new ConsumerPartitionAssignor.Subscription(Collections.singletonList(topic))));
77 | assertEquals(Arrays.asList(
78 | new TopicPartition(topic, 0),
79 | new TopicPartition(topic, 1),
80 | new TopicPartition(topic, 2)), assignment.get(consumerId));
81 | }
82 |
83 | @Test
84 | public void testSubscriptionIncludesTopicThatDoesNotExist() {
85 | String topic = "topic";
86 | String topicDoesNotExist = "doesNotExist";
87 | String consumerId = "consumer";
88 |
89 | Map partitionsPerTopic = new HashMap<>();
90 | partitionsPerTopic.put(topic, 1);
91 |
92 | Map> assignment = assignor.assign(partitionsPerTopic,
93 | Collections.singletonMap(consumerId, new ConsumerPartitionAssignor.Subscription(Arrays.asList(topic, topicDoesNotExist))));
94 | assertEquals(Arrays.asList(new TopicPartition(topic, 0)), assignment.get(consumerId));
95 | }
96 |
97 | @Test
98 | public void testOneConsumerMultipleTopics() {
99 | String topic1 = "topic1";
100 | String topic2 = "topic2";
101 | String consumerId = "consumer";
102 |
103 | Map partitionsPerTopic = new HashMap<>();
104 | partitionsPerTopic.put(topic1, 1);
105 | partitionsPerTopic.put(topic2, 2);
106 |
107 | Map> assignment = assignor.assign(partitionsPerTopic,
108 | Collections.singletonMap(consumerId, new ConsumerPartitionAssignor.Subscription(Arrays.asList(topic1, topic2))));
109 | assertEquals(Arrays.asList(
110 | new TopicPartition(topic2, 0),
111 | new TopicPartition(topic2, 1),
112 | new TopicPartition(topic1, 0)), assignment.get(consumerId));
113 | }
114 |
115 | @Test
116 | public void testTwoConsumersOneTopicOnePartition() {
117 | String topic = "topic";
118 | String consumer1 = "consumer1";
119 | String consumer2 = "consumer2";
120 |
121 | Map partitionsPerTopic = new HashMap<>();
122 | partitionsPerTopic.put(topic, 1);
123 |
124 | Map consumers = new HashMap<>();
125 | consumers.put(consumer1, new ConsumerPartitionAssignor.Subscription(Collections.singletonList(topic)));
126 | consumers.put(consumer2, new ConsumerPartitionAssignor.Subscription(Collections.singletonList(topic)));
127 |
128 | Map> assignment = assignor.assign(partitionsPerTopic, consumers);
129 | assertEquals(Arrays.asList(new TopicPartition(topic, 0)), assignment.get(consumer1));
130 | assertEquals(Collections.emptyList(), assignment.get(consumer2));
131 | }
132 |
133 | @Test
134 | public void testTwoConsumersOneTopicTwoPartitions() {
135 | String topic = "topic";
136 | String consumer1 = "consumer1";
137 | String consumer2 = "consumer2";
138 |
139 | Map partitionsPerTopic = new HashMap<>();
140 | partitionsPerTopic.put(topic, 2);
141 |
142 | Map consumers = new HashMap<>();
143 | consumers.put(consumer1, new ConsumerPartitionAssignor.Subscription(Collections.singletonList(topic)));
144 | consumers.put(consumer2, new ConsumerPartitionAssignor.Subscription(Collections.singletonList(topic)));
145 |
146 | Map> assignment = assignor.assign(partitionsPerTopic, consumers);
147 | assertEquals(Arrays.asList(new TopicPartition(topic, 0)), assignment.get(consumer1));
148 | assertEquals(Arrays.asList(new TopicPartition(topic, 1)), assignment.get(consumer2));
149 | }
150 |
151 | @Test
152 | public void testMultipleConsumersMixedTopics() {
153 | String topic1 = "topic1";
154 | String topic2 = "topic2";
155 | String consumer1 = "consumer1";
156 | String consumer2 = "consumer2";
157 | String consumer3 = "consumer3";
158 |
159 | Map partitionsPerTopic = new HashMap<>();
160 | partitionsPerTopic.put(topic1, 3);
161 | partitionsPerTopic.put(topic2, 2);
162 |
163 | Map consumers = new HashMap<>();
164 | consumers.put(consumer1, new ConsumerPartitionAssignor.Subscription(Collections.singletonList(topic1)));
165 | consumers.put(consumer2, new ConsumerPartitionAssignor.Subscription(Arrays.asList(topic1, topic2)));
166 | consumers.put(consumer3, new ConsumerPartitionAssignor.Subscription(Collections.singletonList(topic1)));
167 |
168 | Map> assignment = assignor.assign(partitionsPerTopic, consumers);
169 | assertEquals(Arrays.asList(
170 | new TopicPartition(topic1, 0),
171 | new TopicPartition(topic1, 2)), assignment.get(consumer1));
172 | assertEquals(Arrays.asList(
173 | new TopicPartition(topic2, 0),
174 | new TopicPartition(topic2, 1)), assignment.get(consumer2));
175 | assertEquals(Arrays.asList(
176 | new TopicPartition(topic1, 1)), assignment.get(consumer3));
177 | }
178 |
179 | @Test
180 | public void testTwoConsumersTwoTopicsSixPartitions() {
181 | String topic1 = "topic1";
182 | String topic2 = "topic2";
183 | String consumer1 = "consumer1";
184 | String consumer2 = "consumer2";
185 |
186 | Map partitionsPerTopic = new HashMap<>();
187 | partitionsPerTopic.put(topic1, 3);
188 | partitionsPerTopic.put(topic2, 3);
189 |
190 | Map consumers = new HashMap<>();
191 | consumers.put(consumer1, new ConsumerPartitionAssignor.Subscription(Arrays.asList(topic1, topic2)));
192 | consumers.put(consumer2, new ConsumerPartitionAssignor.Subscription(Arrays.asList(topic1, topic2)));
193 |
194 | Map> assignment = assignor.assign(partitionsPerTopic, consumers);
195 | assertEquals(Arrays.asList(
196 | new TopicPartition(topic1, 0),
197 | new TopicPartition(topic1, 2),
198 | new TopicPartition(topic2, 1)), assignment.get(consumer1));
199 | assertEquals(Arrays.asList(
200 | new TopicPartition(topic1, 1),
201 | new TopicPartition(topic2, 0),
202 | new TopicPartition(topic2, 2)), assignment.get(consumer2));
203 | }
204 |
205 | @Test
206 | public void testMultipleConsumersUnbalancedSubscriptions() {
207 | String topic1 = "topic1";
208 | String topic2 = "topic2";
209 | String topic3 = "topic3";
210 | String topic4 = "topic4";
211 | String topic5 = "topic5";
212 | String consumer1 = "consumer1";
213 | String consumer2 = "consumer2";
214 | String consumer3 = "consumer3";
215 | String consumer4 = "consumer4";
216 | int oddTopicPartitions = 2;
217 | int evenTopicPartitions = 1;
218 |
219 | Map partitionsPerTopic = new HashMap<>();
220 | partitionsPerTopic.put(topic1, oddTopicPartitions);
221 | partitionsPerTopic.put(topic2, evenTopicPartitions);
222 | partitionsPerTopic.put(topic3, oddTopicPartitions);
223 | partitionsPerTopic.put(topic4, evenTopicPartitions);
224 | partitionsPerTopic.put(topic5, oddTopicPartitions);
225 |
226 | List oddTopics = Arrays.asList(topic1, topic3, topic5);
227 | List allTopics = Arrays.asList(topic1, topic2, topic3, topic4, topic5);
228 |
229 | Map consumers = new HashMap<>();
230 | consumers.put(consumer1, new ConsumerPartitionAssignor.Subscription(allTopics));
231 | consumers.put(consumer2, new ConsumerPartitionAssignor.Subscription(oddTopics));
232 | consumers.put(consumer3, new ConsumerPartitionAssignor.Subscription(oddTopics));
233 | consumers.put(consumer4, new ConsumerPartitionAssignor.Subscription(allTopics));
234 |
235 | Map> assignment = assignor.assign(partitionsPerTopic, consumers);
236 | assertEquals(Arrays.asList(
237 | new TopicPartition(topic2, 0),
238 | new TopicPartition(topic3, 0)), assignment.get(consumer1));
239 | assertEquals(Arrays.asList(
240 | new TopicPartition(topic1, 0),
241 | new TopicPartition(topic3, 1)), assignment.get(consumer2));
242 | assertEquals(Arrays.asList(
243 | new TopicPartition(topic1, 1),
244 | new TopicPartition(topic5, 0)), assignment.get(consumer3));
245 | assertEquals(Arrays.asList(
246 | new TopicPartition(topic4, 0),
247 | new TopicPartition(topic5, 1)), assignment.get(consumer4));
248 | }
249 | }
250 |
--------------------------------------------------------------------------------
/common-kafka/src/test/java/com/cerner/common/kafka/metrics/MeterPoolTest.java:
--------------------------------------------------------------------------------
1 | package com.cerner.common.kafka.metrics;
2 |
3 | import com.yammer.metrics.Metrics;
4 | import com.yammer.metrics.core.Meter;
5 | import com.yammer.metrics.core.MetricName;
6 | import com.yammer.metrics.core.MetricsRegistry;
7 | import org.junit.jupiter.api.Test;
8 | import org.junit.jupiter.api.BeforeEach;
9 |
10 | import static org.hamcrest.CoreMatchers.equalTo;
11 | import static org.hamcrest.CoreMatchers.is;
12 | import static org.hamcrest.CoreMatchers.not;
13 | import static org.hamcrest.MatcherAssert.assertThat;
14 | import static org.junit.jupiter.api.Assertions.assertSame;
15 | import static org.junit.jupiter.api.Assertions.assertThrows;
16 |
17 | /**
18 | * Tests for {@link MeterPool}.
19 | */
20 | public class MeterPoolTest {
21 |
22 | @BeforeEach
23 | public void clearMetrics() {
24 | MetricsRegistry metricsRegistry = Metrics.defaultRegistry();
25 | for (MetricName metric : metricsRegistry.allMetrics().keySet()) {
26 | metricsRegistry.removeMetric(metric);
27 | }
28 | }
29 |
30 | @Test
31 | public void nullClass() {
32 | assertThrows(IllegalArgumentException.class,
33 | () -> new MeterPool(null, "name"));
34 | }
35 |
36 | @Test
37 | public void nullName() {
38 | assertThrows(IllegalArgumentException.class,
39 | () -> new MeterPool(Object.class, null));
40 | }
41 |
42 | @Test
43 | public void getMeterNullScope() {
44 | assertThrows(IllegalArgumentException.class,
45 | () -> new MeterPool(Object.class, "name").getMeter(null));
46 | }
47 |
48 | @Test
49 | public void getMeterSameScope() {
50 | MeterPool pool = new MeterPool(Object.class, "name");
51 |
52 | Meter meter1 = pool.getMeter("scope");
53 | Meter meter2 = pool.getMeter("scope");
54 |
55 | assertSame(meter1, meter2);
56 | }
57 |
58 | @Test
59 | public void getMeterDifferentScope() {
60 | MeterPool pool = new MeterPool(Object.class, "name");
61 |
62 | Meter meter1 = pool.getMeter("scope1");
63 | Meter meter2 = pool.getMeter("scope2");
64 |
65 | assertThat(meter1, is(not(equalTo(meter2))));
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/common-kafka/src/test/java/com/cerner/common/kafka/producer/KafkaProducerPoolTest.java:
--------------------------------------------------------------------------------
1 | package com.cerner.common.kafka.producer;
2 |
3 | import com.cerner.common.kafka.consumer.ConsumerOffsetClient;
4 |
5 | import org.apache.kafka.clients.admin.AdminClient;
6 | import org.apache.kafka.clients.admin.NewTopic;
7 | import org.apache.kafka.clients.producer.Producer;
8 | import org.apache.kafka.clients.producer.ProducerRecord;
9 | import org.apache.kafka.common.serialization.StringSerializer;
10 | import org.junit.jupiter.api.AfterEach;
11 | import org.junit.jupiter.api.AfterAll;
12 | import org.junit.jupiter.api.BeforeEach;
13 | import org.junit.jupiter.api.BeforeAll;
14 | import org.junit.jupiter.api.Test;
15 |
16 | import com.cerner.common.kafka.KafkaExecutionException;
17 | import com.cerner.common.kafka.KafkaTests;
18 | import com.cerner.common.kafka.testing.KafkaTestUtils;
19 | import org.junit.jupiter.api.Timeout;
20 |
21 | import java.io.IOException;
22 | import java.util.Arrays;
23 | import java.util.Collections;
24 | import java.util.HashSet;
25 | import java.util.IdentityHashMap;
26 | import java.util.LinkedList;
27 | import java.util.List;
28 | import java.util.Map;
29 | import java.util.Properties;
30 | import java.util.Set;
31 | import java.util.UUID;
32 | import java.util.concurrent.Callable;
33 | import java.util.concurrent.CompletionService;
34 | import java.util.concurrent.ExecutionException;
35 | import java.util.concurrent.ExecutorCompletionService;
36 | import java.util.concurrent.ExecutorService;
37 | import java.util.concurrent.Executors;
38 | import java.util.concurrent.Future;
39 |
40 | import static com.cerner.common.kafka.producer.KafkaProducerPool.DEFAULT_KAFKA_PRODUCER_CONCURRENCY_INT;
41 | import static com.cerner.common.kafka.producer.KafkaProducerPool.KAFKA_PRODUCER_CONCURRENCY;
42 | import static org.apache.kafka.clients.producer.ProducerConfig.LINGER_MS_CONFIG;
43 | import static org.hamcrest.CoreMatchers.is;
44 | import static org.hamcrest.CoreMatchers.not;
45 | import static org.hamcrest.MatcherAssert.assertThat;
46 | import static org.junit.jupiter.api.Assertions.assertNotNull;
47 | import static org.junit.jupiter.api.Assertions.assertThrows;
48 | import static org.mockito.Mockito.mock;
49 | import static org.mockito.Mockito.times;
50 | import static org.mockito.Mockito.verify;
51 | import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG;
52 | import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG;
53 | import static org.apache.kafka.clients.producer.ProducerConfig.ACKS_CONFIG;
54 |
55 | /**
56 | * @author Stephen Durfey
57 | */
58 | public class KafkaProducerPoolTest {
59 |
60 | private KafkaProducerPool pool;
61 | private static AdminClient kafkaAdminClient;
62 |
63 | private static final String FORCE_PRODUCER_ROTATION_OVERFLOW = "producer.rotation.overflow";
64 |
65 | @BeforeAll
66 | public static void startup() throws Exception {
67 | KafkaTests.startTest();
68 | kafkaAdminClient = AdminClient.create(KafkaTests.getProps());
69 | }
70 |
71 | @AfterAll
72 | public static void shutdown() throws Exception {
73 | kafkaAdminClient.close();
74 | KafkaTests.endTest();
75 | }
76 |
77 | @BeforeEach
78 | public void initializePool() {
79 | pool = new KafkaProducerPool<>();
80 | }
81 |
82 | @AfterEach
83 | public void closePool() throws IOException {
84 | pool.close();
85 | }
86 |
87 | @Test
88 | public void nullProperties() {
89 | assertThrows(IllegalArgumentException.class, () -> pool.getProducer(null),
90 | "Expected pool.getProdcuer to throw illegal argument exception, but it wasn't thrown.");
91 | }
92 |
93 | @Test
94 | public void sameConfiguration() {
95 | Properties props = KafkaTests.getProps();
96 | props.setProperty(KAFKA_PRODUCER_CONCURRENCY, "1");
97 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
98 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
99 |
100 | Producer p1 = pool.getProducer(props);
101 | Producer p2 = pool.getProducer(props);
102 | assertThat(p1, is(p2));
103 | }
104 |
105 | @Test
106 | public void differentConfiguration() {
107 | Properties props1 = KafkaTests.getProps();
108 | props1.setProperty(KAFKA_PRODUCER_CONCURRENCY, "1");
109 | props1.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
110 | props1.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
111 |
112 | Properties props2 = new Properties();
113 | props2.putAll(props1);
114 | props2.setProperty("some.property", "property.value");
115 |
116 | Producer p1 = pool.getProducer(props1);
117 | Producer p2 = pool.getProducer(props2);
118 | assertThat(p1, is(not(p2)));
119 | }
120 |
121 | @Test
122 | public void sameConfigurationDefaultConcurrency() throws IOException {
123 | Properties props = KafkaTests.getProps();
124 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
125 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
126 | assertPoolConcurrency(props, DEFAULT_KAFKA_PRODUCER_CONCURRENCY_INT);
127 | }
128 |
129 | @Test
130 | public void sameConfigurationCustomConcurrency() throws IOException {
131 | int producerConcurrency = 25;
132 | Properties props = KafkaTests.getProps();
133 | props.setProperty(KAFKA_PRODUCER_CONCURRENCY, String.valueOf(producerConcurrency));
134 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
135 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
136 | assertPoolConcurrency(props, producerConcurrency);
137 | }
138 |
139 | @Test
140 | public void concurrencyConfigIsString() throws IOException {
141 | Properties props = KafkaTests.getProps();
142 | props.setProperty(KAFKA_PRODUCER_CONCURRENCY, "notANumber");
143 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
144 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
145 | assertPoolConcurrency(props, DEFAULT_KAFKA_PRODUCER_CONCURRENCY_INT);
146 | }
147 |
148 | @Test
149 | public void concurrencyConfigIsZero() throws IOException {
150 | Properties props = KafkaTests.getProps();
151 | props.setProperty(KAFKA_PRODUCER_CONCURRENCY, String.valueOf(0));
152 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
153 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
154 | assertPoolConcurrency(props, DEFAULT_KAFKA_PRODUCER_CONCURRENCY_INT);
155 | }
156 |
157 | @Test
158 | public void concurrencyConfigIsNegative() throws IOException {
159 | Properties props = KafkaTests.getProps();
160 | props.setProperty(KAFKA_PRODUCER_CONCURRENCY, String.valueOf(-12));
161 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
162 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
163 | assertPoolConcurrency(props, DEFAULT_KAFKA_PRODUCER_CONCURRENCY_INT);
164 | }
165 |
166 | @Test
167 | public void defaultUsed() throws IOException {
168 | Properties props = KafkaTests.getProps();
169 |
170 | MockProducerPool mockPool = new MockProducerPool();
171 | mockPool.getProducer(props);
172 |
173 | assertThat(mockPool.getProducerProperties().getProperty(ACKS_CONFIG), is(String.valueOf(-1)));
174 | }
175 |
176 | @Test
177 | public void defaultOverridden() throws IOException {
178 | Properties props = KafkaTests.getProps();
179 | props.setProperty(ACKS_CONFIG, String.valueOf(1));
180 |
181 | MockProducerPool mockPool = new MockProducerPool();
182 | mockPool.getProducer(props);
183 |
184 | assertThat(mockPool.getProducerProperties().getProperty(ACKS_CONFIG), is(String.valueOf(1)));
185 | }
186 |
187 | @Test
188 | public void closeEmptyPool() throws IOException {
189 | pool.close();
190 | }
191 |
192 | @Test
193 | public void closeClosesProducers() throws IOException, ExecutionException, InterruptedException {
194 | KafkaProducerPool mockPool = new MockProducerPool();
195 | Properties props = KafkaTests.getProps();
196 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
197 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
198 | Producer producer = mockPool.getProducer(props);
199 | mockPool.close();
200 | verify(producer).close();
201 | }
202 |
203 | @Test
204 | public void multipleCloses() throws IOException {
205 | KafkaProducerPool mockPool = new MockProducerPool();
206 | Properties props = KafkaTests.getProps();
207 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
208 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
209 | List> producers = new LinkedList<>();
210 | for (int i = 0; i < 10; ++i) {
211 | producers.add(mockPool.getProducer(props));
212 | }
213 | mockPool.close();
214 | mockPool.close();
215 | producers.forEach(producer -> verify(producer, times(1)).close());
216 | }
217 |
218 | @Test
219 | @Timeout(10)
220 | public void messageProductionWithProducerConfig() throws InterruptedException, KafkaExecutionException, ExecutionException {
221 | Properties props = KafkaTests.getProps();
222 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
223 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
224 | props.setProperty(LINGER_MS_CONFIG, String.valueOf(100));
225 |
226 | messageProduction(props);
227 | }
228 |
229 | @Test
230 | @Timeout(10)
231 | public void messageProductionWithProperties() throws InterruptedException, KafkaExecutionException, ExecutionException {
232 | Properties props = KafkaTests.getProps();
233 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
234 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
235 | props.setProperty(LINGER_MS_CONFIG, String.valueOf(100));
236 |
237 | messageProduction(props);
238 | }
239 |
240 | @Test
241 | public void producerRotationOverflow() throws IOException {
242 | KafkaProducerPool mockPool = new MockProducerPool();
243 | Properties props = KafkaTests.getProps();
244 | props.setProperty(FORCE_PRODUCER_ROTATION_OVERFLOW, String.valueOf(true));
245 |
246 | mockPool.getProducer(props);
247 | mockPool.close();
248 | }
249 |
250 | private void messageProduction(Properties config) throws InterruptedException, KafkaExecutionException, ExecutionException {
251 | String topicName = "topic_" + UUID.randomUUID().toString();
252 | NewTopic topic = new NewTopic(topicName, 4, (short) 1);
253 | Set topics = new HashSet<>();
254 | topics.add(topic);
255 |
256 | kafkaAdminClient.createTopics(topics);
257 |
258 | Producer producer = pool.getProducer(config);
259 |
260 | long messages = 10;
261 | for (long i = 0; i < messages; ++i) {
262 | producer.send(new ProducerRecord<>(topicName, String.valueOf(i), UUID.randomUUID().toString())).get();
263 | }
264 |
265 | // We loop here since the producer doesn't necessarily write to ZK immediately after receiving a write
266 | ConsumerOffsetClient consumerOffsetClient = new ConsumerOffsetClient(config);
267 | while(KafkaTestUtils.getTopicAndPartitionOffsetSum(topicName, consumerOffsetClient.getEndOffsets(Arrays.asList(topicName))) != messages)
268 | Thread.sleep(100);
269 | }
270 |
271 | @Test
272 | public void concurrencyTest() throws Exception {
273 | // Run 10 threads in parallel for approximately 3 seconds.
274 | int threads = 10;
275 | long stopTime = System.currentTimeMillis() + 3000;
276 |
277 | int producerConcurrency = 25;
278 | final Properties props = KafkaTests.getProps();
279 | props.setProperty(KAFKA_PRODUCER_CONCURRENCY, String.valueOf(producerConcurrency));
280 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
281 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
282 |
283 | final Set> producers = Collections.synchronizedSet(new HashSet<>());
284 |
285 | ExecutorService executor = Executors.newFixedThreadPool(threads);
286 | CompletionService service = new ExecutorCompletionService(executor);
287 | Map, Callable> futures = new IdentityHashMap, Callable>(threads);
288 |
289 | while (System.currentTimeMillis() < stopTime) {
290 | final Future future = service.submit(() -> {
291 | producers.add(pool.getProducer(props));
292 | return null;
293 | });
294 |
295 | // Keep track of submitted jobs.
296 | futures.put(future, () -> {
297 | Void nothing = future.get();
298 | return nothing;
299 | });
300 |
301 | // Collect any completed jobs.
302 | Future done;
303 | while ((done = service.poll()) != null) {
304 | futures.remove(done).call();
305 | }
306 |
307 | // Respect the concurrency limit.
308 | while (futures.size() >= threads) {
309 | done = service.take();
310 | futures.remove(done).call();
311 | }
312 | }
313 |
314 | // Allow pending jobs to complete.
315 | while (futures.size() > 0) {
316 | Future done = service.take();
317 | futures.remove(done).call();
318 | }
319 |
320 | executor.shutdown();
321 | assertThat(producers.size(), is(producerConcurrency));
322 | }
323 |
324 | private void assertPoolConcurrency(Properties props, int concurrency) {
325 | Set> producers = new HashSet<>();
326 | for (int i = 0; i < concurrency * 10; ++i) {
327 | producers.add(pool.getProducer(props));
328 | }
329 | assertThat(producers.size(), is(concurrency));
330 | }
331 |
332 | @SuppressWarnings("unchecked")
333 | private class MockProducerPool extends KafkaProducerPool {
334 | private Properties producerProperties;
335 |
336 | @Override
337 | Producer createProducer(Properties properties) {
338 | producerProperties = properties;
339 | return mock(Producer.class);
340 | }
341 |
342 | @Override
343 | public int getProducerRotation() {
344 | if (producerProperties.getProperty(FORCE_PRODUCER_ROTATION_OVERFLOW) != null) {
345 | return Integer.MAX_VALUE + 2;
346 | } else {
347 | return super.getProducerRotation();
348 | }
349 | }
350 |
351 | public Properties getProducerProperties() {
352 | assertNotNull(producerProperties);
353 | return producerProperties;
354 | }
355 | }
356 |
357 | }
358 |
--------------------------------------------------------------------------------
/common-kafka/src/test/java/com/cerner/common/kafka/producer/KafkaProducerWrapperTest.java:
--------------------------------------------------------------------------------
1 | package com.cerner.common.kafka.producer;
2 |
3 | import com.cerner.common.kafka.KafkaExecutionException;
4 | import com.cerner.common.kafka.KafkaTests;
5 | import org.apache.kafka.clients.admin.AdminClient;
6 | import org.apache.kafka.clients.admin.NewTopic;
7 | import org.apache.kafka.clients.producer.KafkaProducer;
8 | import org.apache.kafka.clients.producer.ProducerConfig;
9 | import org.apache.kafka.clients.producer.ProducerRecord;
10 | import org.apache.kafka.clients.producer.RecordMetadata;
11 | import org.apache.kafka.common.errors.RecordTooLargeException;
12 | import org.apache.kafka.common.errors.TimeoutException;
13 | import org.apache.kafka.common.serialization.StringSerializer;
14 | import org.junit.jupiter.api.AfterAll;
15 | import org.junit.jupiter.api.BeforeAll;
16 | import org.junit.jupiter.api.BeforeEach;
17 | import org.junit.jupiter.api.Test;
18 | import org.junit.jupiter.api.TestInfo;
19 | import org.junit.jupiter.api.extension.ExtendWith;
20 | import org.mockito.ArgumentMatchers;
21 | import org.mockito.Mock;
22 | import org.mockito.junit.jupiter.MockitoExtension;
23 |
24 | import java.io.IOException;
25 | import java.util.ArrayList;
26 | import java.util.HashSet;
27 | import java.util.List;
28 | import java.util.Properties;
29 | import java.util.Set;
30 | import java.util.UUID;
31 | import java.util.concurrent.ExecutionException;
32 | import java.util.concurrent.Future;
33 | import java.util.stream.IntStream;
34 |
35 | import static com.cerner.common.kafka.testing.AbstractKafkaTests.getProps;
36 | import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG;
37 | import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG;
38 | import static org.hamcrest.CoreMatchers.instanceOf;
39 | import static org.hamcrest.CoreMatchers.is;
40 | import static org.hamcrest.MatcherAssert.assertThat;
41 | import static org.junit.jupiter.api.Assertions.assertThrows;
42 | import static org.mockito.Mockito.doThrow;
43 | import static org.mockito.Mockito.never;
44 | import static org.mockito.Mockito.times;
45 | import static org.mockito.Mockito.verify;
46 | import static org.mockito.Mockito.when;
47 |
48 | /**
49 | * @author Stephen Durfey
50 | */
51 | @ExtendWith(MockitoExtension.class)
52 | public class KafkaProducerWrapperTest {
53 |
54 | private static AdminClient kafkaAdminClient;
55 |
56 | private static Properties customProps;
57 |
58 | private String testName;
59 |
60 | @Mock
61 | private KafkaProducer mockedProducer;
62 |
63 | @Mock
64 | private Future mockedFuture;
65 |
66 | private String topicName;
67 |
68 | @BeforeAll
69 | public static void startup() throws Exception {
70 | customProps = new Properties();
71 | customProps.setProperty("message.max.bytes", "1000");
72 | customProps.setProperty("group.max.session.timeout.ms", "2000");
73 | customProps.setProperty("group.min.session.timeout.ms", "500");
74 |
75 | // start Kafka with custom properties.
76 | KafkaTests.startKafka(customProps);
77 |
78 | kafkaAdminClient = AdminClient.create(KafkaTests.getProps());
79 | }
80 |
81 | @AfterAll
82 | public static void shutdown() throws Exception {
83 | kafkaAdminClient.close();
84 |
85 | // Restart Kafka with default properties.
86 | KafkaTests.stopKafka();
87 | KafkaTests.startKafka();
88 | }
89 |
90 | @BeforeEach
91 | public void setup(TestInfo testInfo){
92 | testName = testInfo.getDisplayName().replaceAll("[^a-zA-Z0-9]", "-").trim();
93 | topicName = "topic_" + testName;
94 | }
95 |
96 | @Test
97 | public void test_messageSentSynchronouslySuccessfully() throws IOException {
98 | long previousSendCount = KafkaProducerWrapper.SEND_TIMER.count();
99 | long previousSyncSendCount = KafkaProducerWrapper.SYNC_SEND_TIMER.count();
100 | long previousFlushCount = KafkaProducerWrapper.FLUSH_TIMER.count();
101 | long previousBatchSizeCount = KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.count();
102 | double previousBatchSizeSum = KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.sum();
103 |
104 | Set topics = new HashSet<>();
105 | topics.add(new NewTopic(topicName, 4, (short) 1));
106 | kafkaAdminClient.createTopics(topics);
107 |
108 | Properties props = KafkaTests.getProps();
109 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
110 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
111 | props.setProperty(ProducerConfig.BATCH_SIZE_CONFIG, "10000");
112 | props.setProperty(ProducerConfig.LINGER_MS_CONFIG, "60000");
113 |
114 | KafkaProducerWrapper producer = new KafkaProducerWrapper<>(new KafkaProducer<>(props));
115 |
116 | producer.sendSynchronously(
117 | new ProducerRecord<>(topicName, "key"+testName, "value"+ UUID.randomUUID()));
118 | producer.close();
119 |
120 | assertThat(KafkaProducerWrapper.SEND_TIMER.count(), is(previousSendCount));
121 | assertThat(KafkaProducerWrapper.SYNC_SEND_TIMER.count(), is(previousSyncSendCount + 1));
122 | assertThat(KafkaProducerWrapper.FLUSH_TIMER.count(), is(previousFlushCount));
123 | assertThat(KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.count(), is(previousBatchSizeCount + 1));
124 | assertThat(KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.sum(), is(previousBatchSizeSum + 1));
125 | }
126 |
127 | @Test
128 | public void test_multipleMessagesSentSynchronouslySuccessfully() throws IOException {
129 | long previousSendCount = KafkaProducerWrapper.SEND_TIMER.count();
130 | long previousSyncSendCount = KafkaProducerWrapper.SYNC_SEND_TIMER.count();
131 | long previousFlushCount = KafkaProducerWrapper.FLUSH_TIMER.count();
132 | long previousBatchSizeCount = KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.count();
133 | double previousBatchSizeSum = KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.sum();
134 |
135 | Set topics = new HashSet<>();
136 | topics.add(new NewTopic(topicName, 4, (short) 1));
137 | kafkaAdminClient.createTopics(topics);
138 |
139 | Properties props = KafkaTests.getProps();
140 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
141 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
142 | props.setProperty(ProducerConfig.BATCH_SIZE_CONFIG, "10000");
143 | props.setProperty(ProducerConfig.LINGER_MS_CONFIG, "60000");
144 |
145 | int batchSize = 10;
146 | KafkaProducerWrapper producer = new KafkaProducerWrapper<>(new KafkaProducer<>(props));
147 |
148 | List> records = new ArrayList<>();
149 |
150 | IntStream.range(0, batchSize).forEach(i ->
151 | records.add(new ProducerRecord<>(topicName, "key"+testName+i, "value"+i)));
152 |
153 | producer.sendSynchronously(records);
154 | producer.close();
155 |
156 | assertThat(KafkaProducerWrapper.SEND_TIMER.count(), is(previousSendCount));
157 | assertThat(KafkaProducerWrapper.SYNC_SEND_TIMER.count(), is(previousSyncSendCount + 1));
158 | assertThat(KafkaProducerWrapper.FLUSH_TIMER.count(), is(previousFlushCount));
159 | assertThat(KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.count(), is(previousBatchSizeCount + 1));
160 | assertThat(KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.sum(), is(previousBatchSizeSum + batchSize));
161 | }
162 |
163 | @Test
164 | public void test_flushRetriable() throws IOException {
165 | doThrow(new TimeoutException("boom")).when(mockedProducer).flush();
166 |
167 | KafkaProducerWrapper producer = new KafkaProducerWrapper<>(mockedProducer);
168 |
169 | producer.send(new ProducerRecord<>(topicName, "key"+testName,
170 | "value"+UUID.randomUUID()));
171 | assertThrows(IOException.class, producer::flush,
172 | "Expected producer.flush() to throw IO exception, but it wasn't thrown.");
173 | }
174 |
175 | @Test
176 | public void test_flushNonRetriable() throws IOException {
177 | doThrow(new RuntimeException("boom")).when(mockedProducer).flush();
178 |
179 | KafkaProducerWrapper producer = new KafkaProducerWrapper<>(mockedProducer);
180 |
181 | producer.send(new ProducerRecord<>(topicName, "key"+testName,
182 | "value"+UUID.randomUUID()));
183 | assertThrows(IOException.class, producer::flush,
184 | "Expected producer.flush() to throw IO exception, but it wasn't thrown.");
185 | }
186 |
187 | @Test
188 | public void test_flushFutureExecutionException() throws IOException, ExecutionException, InterruptedException {
189 | when(mockedProducer.send(ArgumentMatchers.any())).thenReturn(mockedFuture);
190 | when(mockedFuture.get()).thenThrow(new ExecutionException("boom", new IllegalStateException()));
191 |
192 | KafkaProducerWrapper producer = new KafkaProducerWrapper<>(mockedProducer);
193 |
194 | producer.send(new ProducerRecord<>(topicName, "key"+testName,
195 | "value"+UUID.randomUUID()));
196 | assertThrows(IOException.class, producer::flush,
197 | "Expected producer.flush() to throw IO exception, but it wasn't thrown.");
198 | }
199 |
200 | @Test
201 | public void test_messageSentSuccessfully() throws IOException {
202 | long previousSendCount = KafkaProducerWrapper.SEND_TIMER.count();
203 | long previousSyncSendCount = KafkaProducerWrapper.SYNC_SEND_TIMER.count();
204 | long previousFlushCount = KafkaProducerWrapper.FLUSH_TIMER.count();
205 | long previousBatchSizeCount = KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.count();
206 | double previousBatchSizeSum = KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.sum();
207 |
208 | Set topics = new HashSet<>();
209 | topics.add(new NewTopic(topicName, 4, (short) 1));
210 | kafkaAdminClient.createTopics(topics);
211 |
212 | Properties props = KafkaTests.getProps();
213 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
214 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
215 | props.setProperty(ProducerConfig.BATCH_SIZE_CONFIG, "10000");
216 | props.setProperty(ProducerConfig.LINGER_MS_CONFIG, "60000");
217 |
218 | KafkaProducerWrapper producer = new KafkaProducerWrapper<>(new KafkaProducer<>(props));
219 |
220 | producer.send(new ProducerRecord<>(topicName, "key"+testName, "value"+UUID.randomUUID()));
221 | producer.flush();
222 | producer.close();
223 |
224 | assertThat(KafkaProducerWrapper.SEND_TIMER.count(), is(previousSendCount + 1));
225 | assertThat(KafkaProducerWrapper.SYNC_SEND_TIMER.count(), is(previousSyncSendCount));
226 | assertThat(KafkaProducerWrapper.FLUSH_TIMER.count(), is(previousFlushCount + 1));
227 | assertThat(KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.count(), is(previousBatchSizeCount + 1));
228 | assertThat(KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.sum(), is(previousBatchSizeSum + 1));
229 | }
230 |
231 | @Test
232 | public void test_messageSentMultipleSuccessfully() throws IOException, ExecutionException, InterruptedException {
233 | long previousSendCount = KafkaProducerWrapper.SEND_TIMER.count();
234 | long previousSyncSendCount = KafkaProducerWrapper.SYNC_SEND_TIMER.count();
235 | long previousFlushCount = KafkaProducerWrapper.FLUSH_TIMER.count();
236 | long previousBatchSizeCount = KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.count();
237 | double previousBatchSizeSum = KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.sum();
238 |
239 | when(mockedProducer.send(ArgumentMatchers.any())).thenReturn(mockedFuture);
240 | int batchSize = 10;
241 | KafkaProducerWrapper producer = new KafkaProducerWrapper<>(mockedProducer);
242 |
243 | IntStream.range(0, batchSize).forEach(i ->
244 | producer.send(new ProducerRecord<>(topicName, "key"+testName+i, "value"+i)));
245 | producer.flush();
246 |
247 | verify(mockedFuture, times(batchSize)).get();
248 |
249 | producer.close();
250 |
251 | assertThat(KafkaProducerWrapper.SEND_TIMER.count(), is(previousSendCount + batchSize));
252 | assertThat(KafkaProducerWrapper.SYNC_SEND_TIMER.count(), is(previousSyncSendCount));
253 | assertThat(KafkaProducerWrapper.FLUSH_TIMER.count(), is(previousFlushCount + 1));
254 | assertThat(KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.count(), is(previousBatchSizeCount + 1));
255 | assertThat(KafkaProducerWrapper.BATCH_SIZE_HISTOGRAM.sum(), is(previousBatchSizeSum + batchSize));
256 | }
257 |
258 | @Test
259 | public void test_sendNullRecord() throws IOException, ExecutionException, InterruptedException {
260 | KafkaProducerWrapper producer = new KafkaProducerWrapper<>(mockedProducer);
261 | assertThrows(IllegalArgumentException.class, () -> producer.send(null),
262 | "Expected producer.send() to throw Illegal Argument exception, but it wasn't thrown.");
263 | }
264 |
265 | @Test
266 | public void test_WrapperNotCloseProducer() throws IOException, ExecutionException, InterruptedException {
267 | KafkaProducerWrapper producer = new KafkaProducerWrapper<>(mockedProducer);
268 |
269 | producer.close();
270 |
271 | verify(mockedProducer, never()).close();
272 | }
273 |
274 | @Test
275 | public void testSynchronous_messageTooLarge() throws IOException {
276 | Set topics = new HashSet<>();
277 | topics.add(new NewTopic(topicName, 4, (short) 1));
278 | kafkaAdminClient.createTopics(topics);
279 |
280 | Properties props = KafkaTests.getProps();
281 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
282 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
283 | props.setProperty(ProducerConfig.BATCH_SIZE_CONFIG, "10000");
284 | props.setProperty(ProducerConfig.LINGER_MS_CONFIG, "60000");
285 |
286 | // create a payload that is too large
287 | StringBuilder bldr = new StringBuilder();
288 | for(int i = 0; i < 100000; i++) {
289 | bldr.append(i);
290 | }
291 |
292 | List> records = new ArrayList<>(2);
293 | records.add(new ProducerRecord<>(topicName, "key", bldr.toString()));
294 | records.add(new ProducerRecord<>(topicName, "key2", "small"));
295 | boolean caughtRecordTooLargeException = false;
296 | try (KafkaProducerWrapper producer =
297 | new KafkaProducerWrapper<>(new KafkaProducer<>(props))) {
298 | producer.sendSynchronously(records);
299 | } catch (KafkaExecutionException e) {
300 | Throwable cause = e.getCause();
301 |
302 | assertThat(cause, instanceOf(ExecutionException.class));
303 |
304 | Throwable cause2 = cause.getCause();
305 |
306 | assertThat(cause2, instanceOf(RecordTooLargeException.class));
307 |
308 | caughtRecordTooLargeException = true;
309 | }
310 |
311 | assertThat(caughtRecordTooLargeException, is(true));
312 | }
313 |
314 | @Test
315 | public void test_messageSentSuccessfullyEvenWithFailure() throws IOException {
316 |
317 | Set topics = new HashSet<>();
318 | topics.add(new NewTopic(topicName, 4, (short) 1));
319 | kafkaAdminClient.createTopics(topics);
320 |
321 | Properties props = getProps();
322 | props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
323 | props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
324 | props.setProperty(ProducerConfig.BATCH_SIZE_CONFIG, "10000");
325 | props.setProperty(ProducerConfig.LINGER_MS_CONFIG, "60000");
326 |
327 | KafkaProducerWrapper producer = new KafkaProducerWrapper<>(new KafkaProducer<>(props));
328 |
329 | producer.send(new ProducerRecord<>(topicName, "key1", "value1"));
330 | producer.flush();
331 |
332 | // Just stopping Kafka brokers not the Zookeepers (as every time Zookeepers are stopped and brought up they do not have
333 | // same config so for this test we need same config)
334 | KafkaTests.stopOnlyKafkaBrokers();
335 |
336 | // to simulate after some transient error Kafka brokers come back up and resumes
337 | Thread kafkaThread = new StartKafkaThread();
338 |
339 | kafkaThread.start();
340 |
341 | try {
342 | producer.send(new ProducerRecord<>(topicName, "key2", "value2"));
343 | producer.flush();
344 | producer.close();
345 |
346 | } finally {
347 | try {
348 | kafkaThread.join();
349 | } catch (InterruptedException e) {
350 | e.printStackTrace();
351 | }
352 | }
353 | }
354 |
355 | // Separate thread class just to start Kafka cluster independent of test thread
356 | public class StartKafkaThread extends Thread {
357 |
358 | public void run() {
359 | try {
360 | try {
361 | // to have some delay between stopping and starting of the brokers
362 | Thread.sleep(3000L);
363 | } catch (InterruptedException e) {
364 | e.printStackTrace();
365 | }
366 |
367 | // Just start the Kafka brokers, Zookeepers should already be running
368 | KafkaTests.startOnlyKafkaBrokers();
369 |
370 | } catch (Exception e) {
371 | e.printStackTrace();
372 | }
373 | }
374 | }
375 |
376 | }
--------------------------------------------------------------------------------
/common-kafka/src/test/java/com/cerner/common/kafka/producer/partitioners/FairPartitionerTest.java:
--------------------------------------------------------------------------------
1 | package com.cerner.common.kafka.producer.partitioners;
2 |
3 | import org.apache.kafka.common.Cluster;
4 | import org.apache.kafka.common.Node;
5 | import org.apache.kafka.common.PartitionInfo;
6 | import org.junit.jupiter.api.Test;
7 | import org.junit.jupiter.api.BeforeEach;
8 | import org.junit.jupiter.api.TestInfo;
9 |
10 | import java.util.Collections;
11 | import java.util.HashSet;
12 | import java.util.List;
13 | import java.util.Set;
14 | import java.util.stream.Collectors;
15 | import java.util.stream.IntStream;
16 |
17 | import static org.hamcrest.core.Is.is;
18 | import static org.hamcrest.number.OrderingComparison.greaterThanOrEqualTo;
19 | import static org.hamcrest.number.OrderingComparison.lessThan;
20 | import static org.hamcrest.MatcherAssert.assertThat;
21 |
22 | public class FairPartitionerTest {
23 |
24 | private FairPartitioner partitioner;
25 | private String topic;
26 | private Object key;
27 | private byte[] keyBytes;
28 | private Object value;
29 | private byte[] valueBytes;
30 | private Cluster cluster;
31 | private List allPartitions;
32 | private List notAvailablePartitions;
33 | private Node node;
34 |
35 | @BeforeEach
36 | public void setup(TestInfo testInfo) throws InterruptedException {
37 | partitioner = new FairPartitioner();
38 | topic = testInfo.getDisplayName().replaceAll("[^a-zA-Z0-9]", "-").trim();
39 | key = new Object();
40 | keyBytes = new byte[0];
41 | value = new Object();
42 | valueBytes = new byte[0];
43 |
44 | node = new Node(1, "example.com", 6667);
45 |
46 | allPartitions =
47 | IntStream.range(0, 8).mapToObj(i -> {
48 | //null leader means not available
49 | Node leader = null;
50 | if(i % 2 == 0){
51 | //a non-null leader means it is available
52 | leader = node;
53 | }
54 | return new PartitionInfo(topic, i, leader, null, null);
55 | }).collect(Collectors.toList());
56 | notAvailablePartitions = allPartitions.stream().filter(p -> p.leader() == null).collect(Collectors.toList());
57 |
58 | cluster = new Cluster("clusterId", Collections.singleton(node), allPartitions,
59 | Collections.emptySet(), Collections.emptySet());
60 |
61 | // Wait until next clock window tick.
62 | long millis = System.currentTimeMillis() / FairPartitioner.ROTATE_MILLIS;
63 | while (System.currentTimeMillis() / FairPartitioner.ROTATE_MILLIS == millis) {
64 | Thread.sleep(1);
65 | }
66 | }
67 |
68 | @Test
69 | public void partitionAvailable() {
70 | int partition = partitioner.partition(topic, key, keyBytes, value, valueBytes, cluster);
71 | assertThat(partition, is(lessThan(allPartitions.size())));
72 | assertThat(partition, is(greaterThanOrEqualTo(0)));
73 | assertThat(partition % 2, is(0));
74 | }
75 |
76 | @Test
77 | public void partitionNotAvailable() {
78 | cluster = new Cluster("clusterId", Collections.singleton(node), notAvailablePartitions,
79 | Collections.emptySet(), Collections.emptySet());
80 | int partition = partitioner.partition(topic, key, keyBytes, value, valueBytes, cluster);
81 | assertThat(partition, is(lessThan(allPartitions.size())));
82 | assertThat(partition, is(greaterThanOrEqualTo(0)));
83 | assertThat(partition % 2, is(1));
84 | }
85 |
86 | @Test
87 | public void partitionSameTimeWindow() {
88 | int messages = allPartitions.size();
89 | Set partitions = new HashSet<>();
90 |
91 | for (int i = 0; i < messages; ++i) {
92 | int partition = partitioner.partition(topic, key, keyBytes, value, valueBytes, cluster);
93 | assertThat(partition, is(greaterThanOrEqualTo(0)));
94 | assertThat(partition, is(lessThan(allPartitions.size())));
95 | partitions.add(partition);
96 | }
97 |
98 | // Since all messages were produced in the same time window, they should be assigned to the same partition.
99 | assertThat(partitions.size(), is(1));
100 | }
101 |
102 | @Test
103 | public void partitionDifferentTimeWindows() throws InterruptedException {
104 | int messages = allPartitions.size();
105 | Set partitions = new HashSet<>();
106 |
107 | for (int i = 0; i < messages; ++i) {
108 | int partition = partitioner.partition(topic, key, keyBytes, value, valueBytes, cluster);
109 | assertThat(partition, is(greaterThanOrEqualTo(0)));
110 | assertThat(partition, is(lessThan(allPartitions.size())));
111 | partitions.add(partition);
112 | Thread.sleep(FairPartitioner.ROTATE_MILLIS);
113 | }
114 |
115 | // Verify that partition is periodically rotated across all available partitions as expected.
116 | assertThat(partitions.size(), is(allPartitions.size() - notAvailablePartitions.size()));
117 | }
118 | }
--------------------------------------------------------------------------------
/common-kafka/src/test/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | log4j.rootLogger=INFO, A1
2 | log4j.appender.A1=org.apache.log4j.ConsoleAppender
3 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout
4 |
5 | # Print the date in ISO 8601 format
6 | log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
7 |
8 | # Limit Kafka logging
9 | log4j.logger.kafka=WARN
10 | log4j.logger.org.apache.kafka=WARN
11 | log4j.logger.org.apache.kafka.clients.consumer.ConsumerConfig=ERROR
12 | log4j.logger.org.apache.kafka.clients.producer.ProducerConfig=ERROR
13 | log4j.logger.org.apache.kafka.clients.NetworkClient=ERROR
14 |
15 | # Limit Zookeeper logging
16 | log4j.logger.org.apache.zookeeper=WARN
17 | log4j.logger.org.I0Itec.zkclient=WARN
--------------------------------------------------------------------------------