> readyTests = new ArrayList<>();
46 | private int lastAutoPort = 39999;
47 | private int dockerComposeTimeoutSeconds = 1800;
48 |
49 | private Builder(String config) {
50 | this.config = Objects.requireNonNull(config);
51 | }
52 |
53 | /**
54 | * Add low level ready-test that checks if a single TCP port can be connected to.
55 | * @param host
56 | * @param port
57 | * @return this builder
58 | */
59 | public Builder readyWhenPortIsOpen(String host, int port) {
60 | this.readyTests.add(() -> {
61 | try (Socket s = new Socket(InetAddress.getByName(host), port)) {
62 | log.info("Port is ready: {}:{}", host, port);
63 | return true;
64 | } catch (IOException e) {
65 | return false;
66 | }
67 | });
68 | return this;
69 | }
70 |
71 | /**
72 | * Same as {@link #readyWhenPortIsOpen(String, int)}, but uses a previously configured env variable
73 | * as the port, which can be a variable set with {@link #addAutoPortVariable(String)}.
74 | * @param host typically "localhost"
75 | * @param portVariable name of env variable
76 | * @throws NullPointerException if env variable is not previously configured
77 | * @throws NumberFormatException if env variable is not parseable as an integer
78 | * @return this builder
79 | */
80 | public Builder readyWhenPortIsOpen(String host, String portVariable) {
81 | final int port = Integer.parseInt(env.get(portVariable));
82 | return readyWhenPortIsOpen(host, port);
83 | }
84 |
85 | /**
86 | * Add ready when an http-GET to a URL yields status code 2xx.
87 | * @param url
88 | * @return this builder
89 | */
90 | public Builder readyOnHttpGet2xx(String url) {
91 | if (!Objects.requireNonNull(url, "URL cannot be null").startsWith("http")) {
92 | throw new IllegalArgumentException("URL must start with 'http'");
93 | }
94 | this.readyTests.add(() -> {
95 | HttpURLConnection httpConnection = null;
96 | try {
97 | httpConnection = (HttpURLConnection)new URL(url).openConnection();
98 | httpConnection.setConnectTimeout(1000);
99 | httpConnection.setReadTimeout(2000);
100 | final int responseCode = httpConnection.getResponseCode();
101 | if (responseCode >= 200 && responseCode < 300) {
102 | log.info("Http-service is ready: {}", url);
103 | return true;
104 | } else {
105 | return false;
106 | }
107 | } catch (IOException e) {
108 | return false;
109 | } finally {
110 | try {
111 | if (httpConnection != null) {
112 | httpConnection.getInputStream().close();
113 | }
114 | } catch (IOException io) {}
115 | }
116 | });
117 | return this;
118 | }
119 |
120 | /**
121 | * Add test which is ready when an http-GET to a URL yields status code 2xx.
122 | * @param urlTemplate the URL, where "{VALUE}" is replaced by the value of a an env-variable, which
123 | * must be previously set and can be an auto-port-variable.
124 | * @param envVariable variable to use
125 | * @return this builder
126 | */
127 | public Builder readyOnHttpGet2xx(String urlTemplate, String envVariable) {
128 | return readyOnHttpGet2xx(urlTemplate.replaceAll(Pattern.quote("{VALUE}"), env.get(envVariable)));
129 | }
130 |
131 | /**
132 | * Add simple ready-test which simply returns {@code false} until the desired amount of time has elapsed since
133 | * the call to this method method.
134 | */
135 | public Builder readyAfter(long duration, TimeUnit timeUnit) {
136 | final AtomicLong start = new AtomicLong(-1);
137 | final long millisToWait = timeUnit.toMillis(duration);
138 | this.readyTests.add(() -> {
139 | if (start.compareAndSet(-1, System.currentTimeMillis())) {
140 | return false;
141 | }
142 | if (System.currentTimeMillis() - start.get() > millisToWait) {
143 | log.info("Waited for at least {} milliseconds, signalling ready", millisToWait);
144 | return true;
145 | } else {
146 | return false;
147 | }
148 | });
149 | return this;
150 | }
151 |
152 | /**
153 | * Add environment variable to export to docker-compose process.
154 | * The value of this variable can later be retrieved with {@link #getEnvVariable(String)}
155 | * @param variable name, not null
156 | * @param value value
157 | * @return the builder
158 | */
159 | public Builder addEnvVariable(String variable, Object value) {
160 | env.put(Objects.requireNonNull(variable), value != null ? value.toString() : "null");
161 | return this;
162 | }
163 |
164 | /**
165 | * Get value of a previously set variable, which can be an auto-port-variable.
166 | * @param variable
167 | * @return
168 | */
169 | public String getEnvVariable(String variable) {
170 | return env.get(variable);
171 | }
172 |
173 | /**
174 | * Expose an environment variable having a random (currently) free port number as value.
175 | * The value of this variable can later be retrieved with {@link #getEnvVariable(String)}
176 | * @param portVariable an environment variable name
177 | * @return this builder
178 | */
179 | public Builder addAutoPortVariable(String portVariable) {
180 | final int freePort = chooseFreePort(lastAutoPort);
181 | this.env.put(Objects.requireNonNull(portVariable), String.valueOf(freePort));
182 | lastAutoPort = freePort;
183 | return this;
184 | }
185 |
186 | /**
187 | * Add a custom ready test as a supplier which can test if any external docker services are ready.
188 | * The builder instance itself is provided as the function argument to be able to extract
189 | * values from auto-port variables, see {@link #getEnvVariable(String)}.
190 | * The ready-test-supplier be called potentially multiple times at some interval and should
191 | * supply {@code true} when a service is ready.
192 | * By default, a ready-test which simply waits 5 seconds is used.
193 | * @param readyTestSupplier a function which accepts this build as argument and returns a new ready-test supplier
194 | * @return this builder
195 | */
196 | public Builder withCustomReadyTest(Function> readyTestSupplier) {
197 | Supplier readyTest = readyTestSupplier.apply(this);
198 | if (readyTest != null) {
199 | this.readyTests.add(readyTest);
200 | }
201 | return this;
202 | }
203 |
204 | /**
205 | * Set a directory where docker-compose stdout/stderr logs will be stored.
206 | * Default is nothing, and no logs are kept.
207 | * @param directoryPath
208 | * @return this builder
209 | */
210 | public Builder dockerComposeLogDir(String directoryPath) {
211 | this.logdir = directoryPath;
212 | return this;
213 | }
214 |
215 | /**
216 | * Set wait limit in seconds for docker-compose to finish bringing up containers. This time may involve
217 | * Docker downloading images from the internet, take that into account.
218 | * Default value: {@code 1800} seconds
219 | * @return this builder
220 | */
221 | public Builder dockerComposeTimeout(int timeoutSeconds) {
222 | this.dockerComposeTimeoutSeconds = timeoutSeconds;
223 | return this;
224 | }
225 |
226 | /**
227 | * Bring up configured env. This call will block for some amount of time to invoke docker-compose, and then
228 | * to wait using any set ready-test.
229 | * @return a hopefully ready-to-use docker-compose environment
230 | */
231 | public DockerComposeEnv up() throws Exception {
232 | if (readyTests.isEmpty()) {
233 | readyAfter(5, TimeUnit.SECONDS);
234 | }
235 | return new DockerComposeEnv(this.config, this.dockerComposeTimeoutSeconds, this.logdir, this.env, this.readyTests).up();
236 | }
237 | }
238 |
239 | /**
240 | * Get a builder for a docker-compose test environment.
241 | * @param dockerComposeConfigFile
242 | * @return
243 | */
244 | static DockerComposeEnv.Builder builder(String dockerComposeConfigFile) {
245 | return new Builder(dockerComposeConfigFile);
246 | }
247 |
248 | private final Path dockerComposeLogDir;
249 | private final String configFile;
250 | private final long timeoutSeconds;
251 | private final Map env;
252 | private final List> readyTests;
253 |
254 | private DockerComposeEnv(String configFile, long timeoutSeconds, String logdir, Map env, List> readyTests) {
255 | if (logdir != null) {
256 | File dir = new File(logdir);
257 | if (!dir.isDirectory() || !dir.canWrite()) {
258 | throw new IllegalArgumentException("Invalid directory for logs (does not exist or is not writable): " + logdir);
259 | }
260 | this.dockerComposeLogDir = dir.toPath();
261 | } else {
262 | this.dockerComposeLogDir = null;
263 | }
264 | this.configFile = configFile;
265 | this.timeoutSeconds = timeoutSeconds;
266 | this.env = env;
267 | this.readyTests = readyTests;
268 | }
269 |
270 | /**
271 | * Factory for constructing a new docker-compose test environment.
272 | * Waits for all configured ready-tests to complete before returning.
273 | *
274 | * @return the new instance
275 | * @throws Exception in case something goes awry during setup.
276 | */
277 | private DockerComposeEnv up() throws Exception {
278 | log.info("Bringing up local docker-compose environment, max wait={} seconds, env={}", this.timeoutSeconds, this.env);
279 | dockerCompose("up", "-d").start().onExit().thenAccept(process -> {
280 | if (process.exitValue() != 0) {
281 | throw new RuntimeException("docker-compose failed with status " + process.exitValue());
282 | }
283 | }).get(this.timeoutSeconds, TimeUnit.SECONDS);
284 |
285 | log.info("Waiting for {} ready-test(s) to complete ..", readyTests.size());
286 | List> allReadyTestsComplete = readyTests.stream().map(
287 | test -> CompletableFuture.runAsync(()-> {
288 | int count = 0;
289 | while (!test.get() && count++ < 300) {
290 | try {
291 | Thread.sleep(1000);
292 | } catch (InterruptedException ie) {
293 | throw new RuntimeException("Interrupted while waiting for ready-test");
294 | }
295 | }
296 | if (count >= 300) {
297 | throw new RuntimeException("Ready-test give up after 300 attempts");
298 | }
299 | }
300 | )).collect(Collectors.toList());
301 |
302 | try {
303 | CompletableFuture.allOf(allReadyTestsComplete.toArray(new CompletableFuture[]{}))
304 | .thenRun(() -> log.info("All ready-tests completed."))
305 | .get(240, TimeUnit.SECONDS);
306 | } catch (Exception e) {
307 | log.error("At least one ready-test failed, or we timed out while waiting", e);
308 | down();
309 | throw e;
310 | }
311 |
312 | return this;
313 | }
314 |
315 | /**
316 | * Return value of a registered environment variable.
317 | * @param variable the variable
318 | * @return the value, or {@code null} if unknown
319 | */
320 | public String getEnvVariable(String variable) {
321 | return env.get(variable);
322 | }
323 |
324 | /**
325 | * Take down local docker environment using docker-compose.
326 | * @throws Exception when something bad happens, in which case you will have to clean up manually.
327 | */
328 | public void down() throws Exception {
329 | log.info("Taking down local docker-compose environment ..");
330 | dockerCompose("down", "--volumes").start().onExit().get(120, TimeUnit.SECONDS);
331 | }
332 |
333 | static class DockerComposeCommand {
334 | public final String executable;
335 | private final String[] args;
336 | public final String version;
337 |
338 | DockerComposeCommand(String executable, String[] args, String version) {
339 | this.executable = Objects.requireNonNull(executable);
340 | this.args = Objects.requireNonNull(args);
341 | this.version = Objects.requireNonNull(version);
342 | }
343 |
344 | List executableAndDefaultArgs() {
345 | return Arrays.stream(new String[][]{
346 | new String[]{executable},
347 | args,
348 | compareVersions(version, "1.29") < 0 ? new String[]{"--no-ansi"} : new String[]{"--ansi", "never"}
349 | }).flatMap(e -> Arrays.stream(e)).toList();
350 | }
351 |
352 | private static int compareVersions(String v1, String v2) {
353 | List v1Components = Arrays.stream(v1.split("\\."))
354 | .map(n -> Integer.parseInt(n)).collect(Collectors.toList());
355 | List v2Components = Arrays.stream(v2.split("\\."))
356 | .map(n -> Integer.parseInt(n)).collect(Collectors.toList());
357 |
358 | final int maxLength = Math.max(v1Components.size(), v2Components.size());
359 | for (int i=0; i commandArgs = new ArrayList<>(dcExec.executableAndDefaultArgs());
437 | commandArgs.addAll(List.of("-f", Path.of(this.configFile).toString(), "-p", dockerComposeProjectName()));
438 | commandArgs.addAll(Arrays.asList(composeCommandAndArgs));
439 | ProcessBuilder pb = new ProcessBuilder(commandArgs);
440 | this.env.forEach((k,v) -> pb.environment().put(k, v));
441 | if (this.dockerComposeLogDir != null){
442 | pb.redirectOutput(appendTo(this.dockerComposeLogDir.resolve(dockerComposeProjectName() + "-stdout.log").toFile()));
443 | pb.redirectError(appendTo(this.dockerComposeLogDir.resolve(dockerComposeProjectName() + "-stderr.log").toFile()));
444 | } else {
445 | pb.redirectOutput(ProcessBuilder.Redirect.DISCARD);
446 | pb.redirectError(ProcessBuilder.Redirect.DISCARD);
447 | }
448 |
449 | return pb;
450 | }
451 |
452 | private String dockerComposeProjectName() {
453 | return getClass().getSimpleName().toLowerCase() + "-" + obtainPid();
454 | }
455 |
456 | private static long obtainPid() {
457 | return ProcessHandle.current().pid();
458 | }
459 |
460 | private static int chooseFreePort(int higherThan) {
461 | int next = Math.min(65535, higherThan + 1 + (int)(Math.random()*100));
462 | do {
463 | try (ServerSocket s = ServerSocketFactory.getDefault().createServerSocket(
464 | next, 1, InetAddress.getLocalHost())) {
465 | return s.getLocalPort();
466 | } catch (IOException e) {
467 | }
468 | } while ((next += (int)(Math.random()*100)) <= 65535);
469 | throw new IllegalStateException("Unable to find free network port on localhost");
470 | }
471 |
472 | @Override
473 | public void close() throws Exception {
474 | down();
475 | }
476 |
477 | }
478 |
--------------------------------------------------------------------------------
/clients/src/test/java/no/nav/kafka/sandbox/DockerComposeEnvTest.java:
--------------------------------------------------------------------------------
1 | package no.nav.kafka.sandbox;
2 |
3 | import no.nav.kafka.sandbox.DockerComposeEnv.DockerComposeCommand;
4 | import org.junit.jupiter.api.Test;
5 |
6 | import java.util.List;
7 |
8 | import static org.junit.jupiter.api.Assertions.assertEquals;
9 |
10 | class DockerComposeEnvTest {
11 |
12 | @Test
13 | void dockerComposeCommand() {
14 | assertEquals(List.of("docker-compose", "--no-ansi"),
15 | new DockerComposeCommand("docker-compose", new String[0], "1.28").executableAndDefaultArgs());
16 | assertEquals(List.of("docker-compose", "--no-ansi"),
17 | new DockerComposeCommand("docker-compose", new String[0], "1").executableAndDefaultArgs());
18 |
19 | assertEquals(List.of("docker-compose", "--ansi", "never"),
20 | new DockerComposeCommand("docker-compose", new String[0], "1.29").executableAndDefaultArgs());
21 | assertEquals(List.of("docker", "compose", "--ansi", "never"),
22 | new DockerComposeCommand("docker", new String[]{"compose"}, "2").executableAndDefaultArgs());
23 | }
24 |
25 | }
26 |
--------------------------------------------------------------------------------
/clients/src/test/java/no/nav/kafka/sandbox/KafkaSandboxTest.java:
--------------------------------------------------------------------------------
1 | package no.nav.kafka.sandbox;
2 |
3 | import no.nav.kafka.sandbox.messages.ConsoleMessages.Message;
4 | import no.nav.kafka.sandbox.consumer.JsonMessageConsumer;
5 | import no.nav.kafka.sandbox.producer.JsonMessageProducer;
6 | import org.apache.kafka.clients.admin.AdminClient;
7 | import org.apache.kafka.clients.admin.AdminClientConfig;
8 | import org.apache.kafka.clients.admin.NewTopic;
9 | import org.apache.kafka.clients.consumer.ConsumerConfig;
10 | import org.apache.kafka.clients.consumer.KafkaConsumer;
11 | import org.apache.kafka.clients.producer.KafkaProducer;
12 | import org.apache.kafka.clients.producer.ProducerConfig;
13 | import org.apache.kafka.clients.producer.ProducerRecord;
14 | import org.apache.kafka.common.errors.InterruptException;
15 | import org.apache.kafka.common.serialization.StringDeserializer;
16 | import org.apache.kafka.common.serialization.StringSerializer;
17 | import org.junit.jupiter.api.*;
18 | import org.slf4j.Logger;
19 | import org.slf4j.LoggerFactory;
20 |
21 | import java.time.Duration;
22 | import java.util.ArrayList;
23 | import java.util.Collections;
24 | import java.util.List;
25 | import java.util.Map;
26 | import java.util.concurrent.*;
27 | import java.util.concurrent.atomic.AtomicInteger;
28 | import java.util.function.Supplier;
29 |
30 | import static org.junit.jupiter.api.Assertions.*;
31 |
32 | /**
33 | * Demonstrates use of {@link DockerComposeEnv}, and a few simple tests using a running Kafka instance.
34 | */
35 | class KafkaSandboxTest {
36 |
37 | private static DockerComposeEnv dockerComposeEnv;
38 |
39 | private static AdminClient adminClient;
40 |
41 | private static Logger LOG = LoggerFactory.getLogger(KafkaSandboxTest.class);
42 |
43 | private final String testTopic;
44 |
45 | public KafkaSandboxTest() {
46 | testTopic = nextTestTopic();
47 | }
48 |
49 | private static final AtomicInteger testTopicCounter = new AtomicInteger(0);
50 | private static String nextTestTopic() {
51 | return "test-topic-" + testTopicCounter.incrementAndGet();
52 | }
53 |
54 | @BeforeAll
55 | static void dockerComposeUp() throws Exception {
56 | Assumptions.assumeTrue(DockerComposeEnv.dockerComposeAvailable(),
57 | "This test needs a working 'docker compose' command");
58 |
59 | dockerComposeEnv = DockerComposeEnv.builder("src/test/resources/KafkaDockerComposeEnv.yml")
60 | .addAutoPortVariable("KAFKA_PORT")
61 | .dockerComposeLogDir("target/")
62 | .readyWhenPortIsOpen("localhost", "KAFKA_PORT")
63 | .up();
64 |
65 | adminClient = newAdminClient();
66 | }
67 |
68 | @AfterAll
69 | static void dockerComposeDown() throws Exception {
70 | if (dockerComposeEnv != null) {
71 | adminClient.close();
72 | dockerComposeEnv.down();
73 | }
74 | }
75 |
76 | static int kafkaPort() {
77 | return Integer.parseInt(dockerComposeEnv.getEnvVariable("KAFKA_PORT"));
78 | }
79 |
80 | @BeforeEach
81 | void createTestTopic() throws Exception {
82 | adminClient.createTopics(Collections.singleton(new NewTopic(testTopic, 1, (short)1))).all().get();
83 | }
84 |
85 | @AfterEach
86 | void deleteTestTopic() throws Exception {
87 | adminClient.deleteTopics(Collections.singleton(testTopic)).all().get();
88 | }
89 |
90 | @Test
91 | void waitForMessagesBeforeSending() throws Exception {
92 | LOG.debug("waitForMessagesBeforeSending start");
93 | final List messages = List.of("one", "two", "three", "four");
94 |
95 | final CountDownLatch waitForAllMessages = new CountDownLatch(messages.size());
96 |
97 | Executors.newSingleThreadExecutor().execute(() -> {
98 | try (KafkaConsumer consumer = newConsumer("test-group")) {
99 | consumer.subscribe(Collections.singleton(testTopic));
100 | int pollCount = 0;
101 | while (waitForAllMessages.getCount() > 0 && pollCount++ < 5) {
102 | consumer.poll(Duration.ofSeconds(10)).forEach(record -> {
103 | LOG.debug("Received message: " + record.value());
104 | waitForAllMessages.countDown();
105 | });
106 | }
107 | }
108 | });
109 |
110 | // Consumer is now ready and waiting for data, produce something without waiting for acks
111 | try (KafkaProducer producer = newProducer()) {
112 | messages.forEach(val -> {
113 | LOG.debug("Sending message: " + val);
114 | producer.send(new ProducerRecord<>(testTopic, val));
115 | try {
116 | TimeUnit.SECONDS.sleep(1);
117 | } catch (InterruptedException ie) {
118 | }
119 | });
120 | }
121 |
122 | // Wait for consumer to receive the expected number of messages from Kafka
123 | assertTrue(waitForAllMessages.await(30, TimeUnit.SECONDS), "Did not receive expected number of messages within time frame");
124 | }
125 |
126 | @Test
127 | void sendThenReceiveMessages() throws Exception {
128 | LOG.debug("sendThenReceiveMessage start");
129 | final List messages = List.of("one", "two", "three", "four");
130 |
131 | final CountDownLatch productionFinished = new CountDownLatch(messages.size());
132 | try (KafkaProducer producer = newProducer()) {
133 | messages.forEach(val -> {
134 | producer.send(new ProducerRecord<>(testTopic, val), (metadata, exception) -> {
135 | if (exception == null) {
136 | productionFinished.countDown();
137 | }
138 | });
139 | });
140 | productionFinished.await(30, TimeUnit.SECONDS); // Wait for shipment of messages
141 | }
142 |
143 | final List received = new ArrayList<>();
144 | try (KafkaConsumer consumer = newConsumer("test-group")) {
145 | consumer.subscribe(Collections.singleton(testTopic));
146 | int pollCount = 0;
147 | while (received.size() < messages.size() && pollCount++ < 10) {
148 | consumer.poll(Duration.ofSeconds(1)).forEach(cr -> {
149 | received.add(cr.value());
150 | });
151 | }
152 | }
153 |
154 | LOG.debug("Received messages: {}", received);
155 |
156 | assertEquals(messages.size(), received.size(), "Number of messages received not equal to number of messages sent");
157 | }
158 |
159 | @Test
160 | void testJsonMessageConsumerAndProducer() throws Exception {
161 |
162 | final BlockingQueue outbox = new ArrayBlockingQueue<>(1);
163 | final Supplier supplier = () -> {
164 | try { return outbox.take(); } catch (InterruptedException ie) {
165 | throw new InterruptException(ie);
166 | }
167 | };
168 |
169 | final ExecutorService executor = Executors.newFixedThreadPool(2);
170 |
171 | final var producer = new JsonMessageProducer(testTopic, null, kafkaProducerTestConfig(kafkaPort()),
172 | Bootstrap.objectMapper(), supplier , m -> null, true);
173 | final Future producerLoop = executor.submit(producer::produceLoop);
174 |
175 | outbox.offer(new Message("Hello", "test-sender"));
176 |
177 | // ---- now fetch message using consumer ----
178 |
179 | final BlockingQueue inbox = new ArrayBlockingQueue<>(1);
180 | final var consumer = new JsonMessageConsumer(testTopic,
181 | Message.class,
182 | kafkaConsumerTestConfig(kafkaPort(), "testGroup"),
183 | Bootstrap.objectMapper(), m -> inbox.offer(m) );
184 | final Future consumeLoop = executor.submit(consumer::consumeLoop);
185 |
186 | Message success = inbox.take();
187 | assertEquals("Hello", success.text);
188 | assertNull(inbox.poll());
189 |
190 | producerLoop.cancel(true);
191 | consumeLoop.cancel(true);
192 | executor.shutdown();
193 | }
194 |
195 | private static AdminClient newAdminClient() {
196 | return AdminClient.create(Map.of(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:"+ kafkaPort()));
197 | }
198 |
199 | private static KafkaConsumer newConsumer(String group) {
200 | return new KafkaConsumer<>(kafkaConsumerTestConfig(kafkaPort(), group));
201 | }
202 |
203 | private static KafkaProducer newProducer() {
204 | return new KafkaProducer<>(kafkaProducerTestConfig(kafkaPort()));
205 | }
206 |
207 | private static Map kafkaConsumerTestConfig(int kafkaPort, String consumerGroup) {
208 | return Map.of(
209 | ConsumerConfig.GROUP_ID_CONFIG, consumerGroup,
210 | ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + kafkaPort,
211 | ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName(),
212 | ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName(),
213 | ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"
214 | );
215 | }
216 |
217 | private static Map kafkaProducerTestConfig(int kafkaPort) {
218 | return Map.of(
219 | ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + kafkaPort,
220 | ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName(),
221 | ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()
222 | );
223 | }
224 |
225 | }
226 |
--------------------------------------------------------------------------------
/clients/src/test/resources/KafkaDockerComposeEnv.yml:
--------------------------------------------------------------------------------
1 | ---
2 | version: '3'
3 | services:
4 | zookeeper:
5 | hostname: zookeeper
6 | container_name: zookeeper-test-${KAFKA_PORT}
7 | image: confluentinc/cp-zookeeper:latest
8 | environment:
9 | ZOOKEEPER_CLIENT_PORT: 2181
10 | ZOOKEEPER_TICK_TIME: 2000
11 | healthcheck:
12 | test: ["CMD-SHELL", "echo ruok | nc -w 2 localhost 2181"]
13 | interval: 10s
14 | timeout: 30s
15 | retries: 5
16 |
17 | broker:
18 | image: confluentinc/cp-kafka:latest
19 | hostname: broker
20 | container_name: broker-test-${KAFKA_PORT}
21 | ports:
22 | - "${KAFKA_PORT:-9092}:${KAFKA_PORT:-9092}"
23 | depends_on:
24 | - zookeeper
25 | environment:
26 | KAFKA_BROKER_ID: 1
27 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
28 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
29 | KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker:29092,EXTERNAL://localhost:${KAFKA_PORT:-9092}
30 | KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
31 | CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
32 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
33 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
34 | healthcheck:
35 | test: ["CMD-SHELL", "echo healthcheck | kafka-console-producer --broker-list broker:29092 --topic healthchecktopic"]
36 | interval: 10s
37 | timeout: 30s
38 | retries: 5
39 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | ---
2 | version: '3'
3 | services:
4 | zookeeper:
5 | hostname: zookeeper
6 | container_name: zookeeper
7 | image: confluentinc/cp-zookeeper:latest
8 | ports:
9 | - "${ZK_PORT:-2181}"
10 | environment:
11 | ZOOKEEPER_CLIENT_PORT: ${ZK_PORT:-2181}
12 | ZOOKEEPER_TICK_TIME: 2000
13 | healthcheck:
14 | test: ["CMD-SHELL", "echo ruok | nc -w 2 localhost ${ZK_PORT:-2181}"]
15 | interval: 30s
16 | timeout: 30s
17 | retries: 5
18 |
19 | broker:
20 | image: confluentinc/cp-kafka:latest
21 | hostname: broker
22 | container_name: broker
23 | ports:
24 | - "${KAFKA_PORT:-9092}:${KAFKA_PORT:-9092}"
25 | - "29092"
26 | depends_on:
27 | - zookeeper
28 | environment:
29 | KAFKA_BROKER_ID: 1
30 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:${ZK_PORT:-2181}'
31 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
32 | KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker:29092,EXTERNAL://localhost:${KAFKA_PORT:-9092}
33 | KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
34 | CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
35 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
36 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
37 | healthcheck:
38 | test: ["CMD-SHELL", "echo healthcheck | kafka-console-producer --broker-list localhost:${KAFKA_PORT:-9092} --topic healthchecktopic"]
39 | interval: 30s
40 | timeout: 30s
41 | retries: 5
42 |
--------------------------------------------------------------------------------
/messages/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 | 4.0.0
6 |
7 |
8 | kafka-sandbox
9 | no.nav.kafka
10 | 1.0-SNAPSHOT
11 |
12 |
13 | messages
14 |
15 |
16 |
17 | com.fasterxml.jackson.core
18 | jackson-annotations
19 |
20 |
21 | org.fusesource.jansi
22 | jansi
23 |
24 |
25 |
26 |
27 |
--------------------------------------------------------------------------------
/messages/src/main/java/module-info.java:
--------------------------------------------------------------------------------
1 | open module no.nav.kafka.sandbox.messages {
2 | requires com.fasterxml.jackson.annotation;
3 | requires org.fusesource.jansi;
4 | exports no.nav.kafka.sandbox.messages;
5 | }
6 |
--------------------------------------------------------------------------------
/messages/src/main/java/no/nav/kafka/sandbox/messages/ConsoleMessages.java:
--------------------------------------------------------------------------------
1 | package no.nav.kafka.sandbox.messages;
2 |
3 | import com.fasterxml.jackson.annotation.JsonCreator;
4 | import com.fasterxml.jackson.annotation.JsonProperty;
5 | import org.fusesource.jansi.AnsiConsole;
6 | import org.fusesource.jansi.AnsiRenderer;
7 |
8 | import java.io.Console;
9 | import java.util.Objects;
10 | import java.util.concurrent.atomic.AtomicBoolean;
11 | import java.util.function.Consumer;
12 | import java.util.function.Supplier;
13 |
14 | public class ConsoleMessages {
15 |
16 | public static class Message {
17 | @JsonProperty
18 | public final String text;
19 | @JsonProperty
20 | public final String senderId;
21 |
22 | @JsonCreator
23 | public Message(@JsonProperty("text") String text,
24 | @JsonProperty("senderId") String id) {
25 | this.text = text;
26 | this.senderId = id;
27 | }
28 |
29 | @Override
30 | public boolean equals(Object o) {
31 | if (this == o) return true;
32 | if (o == null || getClass() != o.getClass()) return false;
33 | Message message = (Message) o;
34 | return Objects.equals(text, message.text) && Objects.equals(senderId, message.senderId);
35 | }
36 |
37 | @Override
38 | public int hashCode() {
39 | return Objects.hash(text, senderId);
40 | }
41 | }
42 |
43 | public static Consumer consoleMessageConsumer() {
44 | return message -> AnsiConsole.out().println(
45 | AnsiRenderer.render(String.format("@|magenta,bold %s|@: @|yellow %s|@", message.senderId, message.text)));
46 | }
47 |
48 | public static Supplier consoleMessageSupplier() {
49 | final Console console = System.console();
50 | if (console == null) {
51 | throw new IllegalStateException("Unable to get system console");
52 | }
53 | final String senderId = "sender-" + ProcessHandle.current().pid();
54 | final AtomicBoolean first = new AtomicBoolean(true);
55 | return () -> {
56 | if (first.getAndSet(false)) {
57 | console.writer().println("Send messages to Kafka, use CTRL+D to exit gracefully.");
58 | }
59 | try {
60 | Thread.sleep(200);
61 | } catch (InterruptedException ie) {
62 | Thread.currentThread().interrupt();
63 | return new Message("interrupted", senderId);
64 | }
65 | console.writer().print("Type message> ");
66 | console.writer().flush();
67 | String text = console.readLine();
68 | if (text == null){
69 | Thread.currentThread().interrupt();
70 | return new Message("leaving", senderId);
71 | } else {
72 | return new Message(text, senderId);
73 | }
74 | };
75 | }
76 |
77 | }
78 |
--------------------------------------------------------------------------------
/messages/src/main/java/no/nav/kafka/sandbox/messages/Measurements.java:
--------------------------------------------------------------------------------
1 | package no.nav.kafka.sandbox.messages;
2 |
3 | import com.fasterxml.jackson.annotation.JsonCreator;
4 | import com.fasterxml.jackson.annotation.JsonProperty;
5 | import org.fusesource.jansi.AnsiConsole;
6 | import org.fusesource.jansi.AnsiRenderer;
7 |
8 | import java.time.LocalDateTime;
9 | import java.time.OffsetDateTime;
10 | import java.util.NoSuchElementException;
11 | import java.util.Objects;
12 | import java.util.concurrent.atomic.AtomicInteger;
13 | import java.util.function.Supplier;
14 |
15 | public class Measurements {
16 |
17 | public static class SensorEvent {
18 |
19 | private final String deviceId;
20 | private final String measureType;
21 | private final String unitType;
22 | private final OffsetDateTime timestamp;
23 | private final Integer value;
24 |
25 | @JsonCreator
26 | public SensorEvent(@JsonProperty("deviceId") String deviceId,
27 | @JsonProperty("measureType") String measureType,
28 | @JsonProperty("unitType") String unitType,
29 | @JsonProperty("timestamp") OffsetDateTime timestamp,
30 | @JsonProperty("value") Integer value) {
31 | this.deviceId = Objects.requireNonNull(deviceId);
32 | this.measureType = Objects.requireNonNull(measureType);
33 | this.unitType = Objects.requireNonNull(unitType);
34 | this.timestamp = Objects.requireNonNull(timestamp);
35 | this.value = Objects.requireNonNull(value);
36 | }
37 |
38 | public String getDeviceId() {
39 | return deviceId;
40 | }
41 |
42 | public String getMeasureType() {
43 | return measureType;
44 | }
45 |
46 | public String getUnitType() {
47 | return unitType;
48 | }
49 |
50 | public OffsetDateTime getTimestamp() {
51 | return timestamp;
52 | }
53 |
54 | public Integer getValue() {
55 | return value;
56 | }
57 |
58 | @Override
59 | public String toString() {
60 | return "SensorEvent{" +
61 | "deviceId='" + deviceId + '\'' +
62 | ", measureType='" + measureType + '\'' +
63 | ", unitType='" + unitType + '\'' +
64 | ", timestamp=" + timestamp +
65 | ", value=" + value +
66 | '}';
67 | }
68 |
69 | @Override
70 | public boolean equals(Object o) {
71 | if (this == o) return true;
72 | if (o == null || getClass() != o.getClass()) return false;
73 | SensorEvent that = (SensorEvent) o;
74 | return Objects.equals(deviceId, that.deviceId)
75 | && Objects.equals(measureType, that.measureType)
76 | && Objects.equals(unitType, that.unitType)
77 | && Objects.equals(timestamp, that.timestamp)
78 | && Objects.equals(value, that.value);
79 | }
80 |
81 | @Override
82 | public int hashCode() {
83 | return Objects.hash(deviceId, measureType, unitType, timestamp, value);
84 | }
85 | }
86 |
87 | /**
88 | * @param maxNumberOfElements max number of elements to supply
89 | * @return a sensor event supplier providing up to n values.
90 | * @throws NoSuchElementException when no more events can be supplied.
91 | */
92 | public static Supplier eventSupplier(final int maxNumberOfElements) {
93 | final AtomicInteger counter = new AtomicInteger();
94 | return () -> {
95 | if (counter.incrementAndGet() > maxNumberOfElements) {
96 | throw new NoSuchElementException("No more data can be supplied");
97 | }
98 | return generateEvent();
99 | };
100 | }
101 |
102 | /**
103 | * @return a sensor event supplier providing infinite number of values with a delay.
104 | */
105 | public static Supplier delayedInfiniteEventSupplier() {
106 | return () -> {
107 | try {
108 | Thread.sleep((long) (Math.random() * 1000) + 1000);
109 | } catch (InterruptedException ie) {
110 | Thread.currentThread().interrupt();
111 | }
112 | return generateEvent();
113 | };
114 | }
115 |
116 | /**
117 | * @return a generated sensor event
118 | */
119 | public static SensorEvent generateEvent() {
120 | final long pid = ProcessHandle.current().pid();
121 | final int temperatureBase = (int)(pid % 100);
122 | final int temperaturVariance = (int)(pid % 10);
123 | final String sensorId = "sensor-" + pid;
124 |
125 | final int temp = (int)(Math.random()*temperaturVariance + temperatureBase);
126 | return new SensorEvent(sensorId,"temperature", "celcius", OffsetDateTime.now(), temp);
127 | }
128 |
129 | public static void sensorEventToConsole(SensorEvent m) {
130 | final String ansiOutput = AnsiRenderer.render(String.format(
131 | "@|cyan Device|@: @|magenta,bold %s|@, value: @|blue,bold %d\u00B0|@ %s, timestamp: @|green %s|@",
132 | m.getDeviceId(), m.getValue(), m.getUnitType(), m.getTimestamp()));
133 |
134 | AnsiConsole.out().println(ansiOutput);
135 | }
136 |
137 | }
138 |
--------------------------------------------------------------------------------
/messages/src/main/java/no/nav/kafka/sandbox/messages/SequenceValidation.java:
--------------------------------------------------------------------------------
1 | package no.nav.kafka.sandbox.messages;
2 |
3 | import org.fusesource.jansi.AnsiConsole;
4 | import org.fusesource.jansi.AnsiRenderer;
5 |
6 | import java.io.*;
7 | import java.util.concurrent.TimeUnit;
8 | import java.util.concurrent.atomic.AtomicLong;
9 | import java.util.function.Consumer;
10 | import java.util.function.Supplier;
11 |
12 | /**
13 | * Supplier and consumer that does message sequence validation.
14 | *
15 | * Can be used to test loss or reordering of Kafka messages
16 | *
17 | * The supplier maintains state on disk wrt. next sequence number. Delete the persistence file to reset state.
18 | */
19 | public class SequenceValidation {
20 |
21 | public static Supplier sequenceSupplier(File persistence, long delay, TimeUnit timeUnit) {
22 | return new SequenceSupplier(persistence, delay, timeUnit);
23 | }
24 |
25 | public static Consumer sequenceValidatorConsolePrinter() {
26 | return new SequenceValidator();
27 | }
28 |
29 | private static class SequenceSupplier implements Supplier {
30 | final AtomicLong sequence;
31 | final long delayMillis;
32 | final File persistence;
33 |
34 | private SequenceSupplier(File persistence, long delay, TimeUnit timeUnit) {
35 | this.delayMillis = timeUnit.toMillis(delay);
36 | this.persistence = persistence;
37 | this.sequence = new AtomicLong(loadValue(0));
38 | }
39 |
40 | private void saveValue(long value) {
41 | try (DataOutputStream out = new DataOutputStream(new FileOutputStream(persistence))) {
42 | out.writeLong(value);
43 | } catch (Exception e) {}
44 | }
45 |
46 | private long loadValue(long defaultValue) {
47 | try (DataInputStream in = new DataInputStream(new FileInputStream(persistence))) {
48 | return in.readLong();
49 | } catch (Exception e) {
50 | return defaultValue;
51 | }
52 | }
53 |
54 | @Override
55 | public Long get() {
56 | try {
57 | Thread.sleep(delayMillis);
58 | } catch (InterruptedException ie) {
59 | Thread.currentThread().interrupt();
60 | }
61 | final long toBeDelivered = sequence.longValue();
62 | toConsole("@|cyan [SEQ]|@ Supplying sequence(%d)", toBeDelivered);
63 | saveValue(sequence.incrementAndGet());
64 | return toBeDelivered;
65 | }
66 | }
67 |
68 | private static class SequenceValidator implements Consumer {
69 |
70 | long expect = -1;
71 | long errorCount = 0;
72 |
73 | @Override
74 | public void accept(Long received) {
75 | if (expect == -1) {
76 | expect = received + 1;
77 | toConsole("@|cyan [SEQ]|@ @|yellow Synchronized sequence|@: received(%d), next expect(%d), errors(%d)", received, expect, errorCount);
78 | } else if (received < expect) {
79 | ++errorCount;
80 | toConsole("@|cyan [SEQ]|@ @|red ERROR|@: Received lower sequence than expected: received(%d) < expect(%d), resync next, errors(%d)",
81 | received, expect, errorCount);
82 | expect = -1;
83 | } else if (received > expect){
84 | ++errorCount;
85 | toConsole("@|cyan [SEQ]|@ @|red ERROR|@: Received higher sequence than expected: received(%d) > expect(%d), resync next, errors(%d)",
86 | received, expect, errorCount);
87 | expect = -1;
88 | } else {
89 | toConsole("@|cyan [SEQ]|@ @|blue,bold In sync|@: received(%d) = expect(%d), errors(%d)", received, expect, errorCount);
90 | ++expect;
91 | }
92 | }
93 | }
94 |
95 | private static void toConsole(String format, Object...args) {
96 | AnsiConsole.out().println(AnsiRenderer.render(String.format(format, args)));
97 | }
98 | }
99 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 | 4.0.0
6 |
7 | no.nav.kafka
8 | kafka-sandbox
9 | 1.0-SNAPSHOT
10 | pom
11 |
12 |
13 | messages
14 | clients
15 | clients-spring
16 |
17 |
18 |
19 | UTF-8
20 | 17
21 |
22 |
24 | 3.2.0
25 | 2.4.1
26 |
27 |
28 | 3.3.0
29 | 3.11.0
30 | 3.2.2
31 | 3.4.1
32 | 3.6.1
33 |
34 |
35 |
36 |
37 |
38 | org.springframework.boot
39 | spring-boot-dependencies
40 | ${spring-boot.version}
41 | pom
42 | import
43 |
44 |
45 | org.fusesource.jansi
46 | jansi
47 | ${jansi.version}
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 | org.apache.maven.plugins
57 | maven-compiler-plugin
58 | ${maven-compiler-plugin.version}
59 |
60 | ${java.version}
61 | true
62 |
63 |
64 |
65 | org.apache.maven.plugins
66 | maven-surefire-plugin
67 | ${maven-surefire-plugin.version}
68 |
69 |
70 | org.apache.maven.plugins
71 | maven-dependency-plugin
72 | ${maven-dependency-plugin.version}
73 |
74 |
75 | org.apache.maven.plugins
76 | maven-jar-plugin
77 | ${maven-jar-plugin.version}
78 |
79 |
80 | org.springframework.boot
81 | spring-boot-maven-plugin
82 | ${spring-boot.version}
83 |
84 |
85 | org.apache.maven.plugins
86 | maven-enforcer-plugin
87 | ${maven-enforcer-plugin.version}
88 |
89 |
90 | enforce-versions
91 |
92 | enforce
93 |
94 |
95 |
96 |
97 | [${java.version},)
98 |
99 |
100 | [3.8,)
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 | org.apache.maven.plugins
113 | maven-enforcer-plugin
114 |
115 |
116 |
117 |
118 |
119 |
--------------------------------------------------------------------------------
/run:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | set -e
3 |
4 | cd "$(dirname "$0")"
5 |
6 | if ! test -f messages/target/messages-*.jar -a -f clients/target/clients-*.jar; then
7 | mvn -B install
8 | fi
9 |
10 | exec java -jar clients/target/clients-*.jar "$@"
11 |
--------------------------------------------------------------------------------