├── README.md ├── dashboard.jpg ├── dashboard ├── Dockerfile ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── wordpress │ │ └── abhirockzz │ │ └── kafEEne │ │ └── websocket │ │ ├── Consumer.java │ │ ├── ConsumerTrigger.java │ │ ├── KafkaWebsocketEndpoint.java │ │ ├── Peers.java │ │ └── model │ │ └── Payload.java │ └── webapp │ ├── WEB-INF │ └── beans.xml │ └── index.html ├── docker-compose.yml └── producer ├── .gitignore ├── Dockerfile ├── pom.xml └── src └── main └── java └── com └── wordpress └── abhirockzz └── kafEEne └── websocket └── producer ├── Producer.java └── ProducerBootstrap.java /README.md: -------------------------------------------------------------------------------- 1 | Check [the blog](https://abhirockzz.wordpress.com/2017/05/22/kafeene-1-websocket-kafka) for more details 2 | 3 | ## Start with Docker Compose 4 | 5 | - `git clone https://github.com/abhirockzz/kafka-websocket.git` 6 | - `cd dashboard` and `mvn clean install` - creates `kafka-websocket.war` in `target` directory 7 | - `cd producer` and `mvn clean install` - creates `kafka-producer.jar` in `target` directory 8 | - `cd ..` and `docker-compose up --build` - starts Kafka, Zookeeper, Payara and Producer containers (you can switch to any other [Java EE runtime](https://github.com/abhirockzz/kafka-websocket/blob/master/dashboard/Dockerfile#L1)) 9 | - Kafka accessible @ `9092` 10 | - Zookeeper @ `2181` 11 | - Producer starts pushing records to Kafka topics (auto creates topic-1, topic-2) 12 | - Port `8080` exposed from Payara container 13 | 14 | Wait for the containers to startup before you move to the testing part... 15 | 16 | ## Test 17 | 18 | - `docker-machine ip` - get the IP address of your Docker host. Let's call it `APP_HOST` 19 | - Open your browser and go to `http://:8080/kafka-websocket/`. Enter topic-1 or topic-2 in `subscription` input text box. You will start seeing the records being produced by the producer for that topic 20 | - You can open multiple such clients/browser windows/tabs and subscribe to either of these topics 21 | 22 | ![](dashboard.jpg) 23 | 24 | - `docker-compose down -v` once you're done.... 25 | -------------------------------------------------------------------------------- /dashboard.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abhirockzz/kafka-websocket/690590b98b64b396a86ffee67dd1fa6065b8e7ab/dashboard.jpg -------------------------------------------------------------------------------- /dashboard/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM airhacks/payara 2 | ENV WAR kafka-websocket.war 3 | COPY target/${WAR} ${DEPLOYMENT_DIR} -------------------------------------------------------------------------------- /dashboard/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | com.wordpress.abhirockzz 6 | kafka-websocket 7 | 1.0 8 | war 9 | 10 | 11 | 12 | 13 | java.net-Public 14 | Maven Java Net Snapshots and Releases 15 | https://maven.java.net/content/groups/public/ 16 | 17 | 18 | 19 | 20 | yasson-snapshots 21 | Yasson Snapshots repository 22 | https://repo.eclipse.org/content/repositories/yasson-snapshots 23 | 24 | 25 | 26 | kafka-websocket 27 | 28 | 29 | ${project.build.directory}/endorsed 30 | UTF-8 31 | 32 | 33 | 34 | 35 | javax 36 | javaee-api 37 | 7.0 38 | provided 39 | 40 | 41 | org.apache.kafka 42 | kafka-clients 43 | 0.10.2.1 44 | 45 | 46 | 47 | javax.json.bind 48 | javax.json.bind-api 49 | 1.0-SNAPSHOT 50 | 51 | 52 | 53 | org.eclipse 54 | yasson 55 | 1.0-SNAPSHOT 56 | 57 | 58 | 59 | org.glassfish 60 | javax.json 61 | 1.1.0-SNAPSHOT 62 | 63 | 64 | 65 | 66 | kafka-websocket 67 | 68 | 69 | org.apache.maven.plugins 70 | maven-compiler-plugin 71 | 3.1 72 | 73 | 1.8 74 | 1.8 75 | 76 | ${endorsed.dir} 77 | 78 | 79 | 80 | 81 | org.apache.maven.plugins 82 | maven-war-plugin 83 | 2.3 84 | 85 | false 86 | 87 | 88 | 89 | org.apache.maven.plugins 90 | maven-dependency-plugin 91 | 2.6 92 | 93 | 94 | validate 95 | 96 | copy 97 | 98 | 99 | ${endorsed.dir} 100 | true 101 | 102 | 103 | javax 104 | javaee-endorsed-api 105 | 7.0 106 | jar 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | -------------------------------------------------------------------------------- /dashboard/src/main/java/com/wordpress/abhirockzz/kafEEne/websocket/Consumer.java: -------------------------------------------------------------------------------- 1 | package com.wordpress.abhirockzz.kafEEne.websocket; 2 | 3 | import com.wordpress.abhirockzz.kafEEne.websocket.model.Payload; 4 | import java.util.Arrays; 5 | import java.util.Properties; 6 | import java.util.concurrent.atomic.AtomicBoolean; 7 | import javax.annotation.PostConstruct; 8 | import javax.annotation.PreDestroy; 9 | import javax.ejb.Stateless; 10 | import javax.enterprise.event.Event; 11 | import javax.inject.Inject; 12 | 13 | import org.apache.kafka.clients.consumer.ConsumerConfig; 14 | import org.apache.kafka.clients.consumer.ConsumerRecord; 15 | import org.apache.kafka.clients.consumer.ConsumerRecords; 16 | import org.apache.kafka.clients.consumer.KafkaConsumer; 17 | import org.apache.kafka.common.errors.WakeupException; 18 | 19 | @Stateless 20 | public class Consumer { 21 | 22 | private KafkaConsumer kConsumer; 23 | private static final String CONSUMER_GROUP = "test-group"; 24 | private final AtomicBoolean stopped = new AtomicBoolean(false); 25 | 26 | @PostConstruct 27 | public void init() { 28 | 29 | String kafkaCluster = System.getenv().getOrDefault("KAFKA_BROKER", "192.168.99.100:9092"); 30 | System.out.println("Kafka Cluster " + kafkaCluster); 31 | 32 | Properties consumerProps = new Properties(); 33 | consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaCluster); 34 | consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, CONSUMER_GROUP); 35 | consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); 36 | consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); 37 | 38 | kConsumer = new KafkaConsumer<>(consumerProps); 39 | kConsumer.subscribe(Arrays.asList("topic-1", "topic-2")); 40 | 41 | System.out.println("init() complete"); 42 | 43 | } 44 | 45 | @PreDestroy 46 | public void close() { 47 | if (stopped.get()) { 48 | System.out.println("Consumer already stopped"); 49 | } else { 50 | if (kConsumer != null) { 51 | stopped.set(true); 52 | kConsumer.wakeup(); 53 | System.out.println("Invoked Consumer wakeup() from thread " + Thread.currentThread().getName()); 54 | } 55 | } 56 | 57 | } 58 | 59 | @Inject 60 | Event event; 61 | 62 | public void consume() { 63 | System.out.println("Consumer loop wil be triggered..."); 64 | try { 65 | while (!stopped.get()) { 66 | 67 | ConsumerRecords records = kConsumer.poll(Integer.MAX_VALUE); 68 | for (ConsumerRecord record : records) { 69 | Payload payload = new Payload(record.key(), record.value(), record.topic()); 70 | event.fire(payload); 71 | System.out.println("Event for payload " + payload + " fired....."); 72 | } 73 | 74 | } 75 | } catch (WakeupException e) { 76 | System.out.println("Consumer loop interrupted from thread " + Thread.currentThread().getName()); 77 | } catch (Exception e) { 78 | System.out.println("Error: " + e.getMessage()); 79 | } finally { 80 | kConsumer.close(); 81 | System.out.println("Consumer shutdown"); 82 | } 83 | 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /dashboard/src/main/java/com/wordpress/abhirockzz/kafEEne/websocket/ConsumerTrigger.java: -------------------------------------------------------------------------------- 1 | package com.wordpress.abhirockzz.kafEEne.websocket; 2 | 3 | import javax.annotation.PostConstruct; 4 | import javax.annotation.Resource; 5 | import javax.ejb.Singleton; 6 | import javax.ejb.Startup; 7 | import javax.ejb.Timeout; 8 | import javax.ejb.TimerConfig; 9 | import javax.ejb.TimerService; 10 | import javax.inject.Inject; 11 | 12 | @Singleton 13 | @Startup 14 | public class ConsumerTrigger { 15 | @Resource 16 | TimerService ts; 17 | 18 | @PostConstruct 19 | public void schedule(){ 20 | ts.createSingleActionTimer(10000, new TimerConfig()); 21 | System.out.println("Setup one-time timer"); 22 | } 23 | 24 | @Inject 25 | Consumer consumer; 26 | 27 | @Timeout 28 | public void trigger(){ 29 | System.out.println("Timer triggered. invoking consumer...."); 30 | consumer.consume(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dashboard/src/main/java/com/wordpress/abhirockzz/kafEEne/websocket/KafkaWebsocketEndpoint.java: -------------------------------------------------------------------------------- 1 | package com.wordpress.abhirockzz.kafEEne.websocket; 2 | 3 | import com.wordpress.abhirockzz.kafEEne.websocket.model.Payload; 4 | import javax.enterprise.event.Observes; 5 | import javax.inject.Inject; 6 | import javax.json.bind.Jsonb; 7 | import javax.json.bind.JsonbBuilder; 8 | import javax.websocket.OnClose; 9 | import javax.websocket.OnOpen; 10 | import javax.websocket.Session; 11 | import javax.websocket.server.PathParam; 12 | import javax.websocket.server.ServerEndpoint; 13 | 14 | @ServerEndpoint("/{topic}/") 15 | public class KafkaWebsocketEndpoint { 16 | 17 | @Inject 18 | private Peers peers; 19 | 20 | @OnOpen 21 | public void open(@PathParam("topic") String topic, Session peer) { 22 | System.out.println("Peer " + peer.getId() + " joined & subscribed to topic " + topic); 23 | peer.getUserProperties().put("topic", topic); 24 | peers.add(peer); 25 | } 26 | 27 | @OnClose 28 | public void close(Session peer) { 29 | peers.remove(peer); 30 | System.out.println("Peer " + peer.getId() + " left"); 31 | } 32 | 33 | public void broadcast(@Observes Payload eventPayload) { 34 | System.out.println("Broadcasting payload " + eventPayload); 35 | 36 | try (Jsonb jsonb = JsonbBuilder.create();) { 37 | 38 | peers.peers().stream() 39 | .filter(s -> s.isOpen()) 40 | .filter(s -> s.getUserProperties().get("topic").equals(eventPayload.getTopic())) 41 | .forEach(s -> s.getAsyncRemote().sendText(jsonb.toJson(eventPayload))); 42 | } catch (Exception e) { 43 | System.out.println("Error during payload broadcast "+ e.getMessage()); 44 | } 45 | 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /dashboard/src/main/java/com/wordpress/abhirockzz/kafEEne/websocket/Peers.java: -------------------------------------------------------------------------------- 1 | package com.wordpress.abhirockzz.kafEEne.websocket; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.Collections; 6 | import java.util.List; 7 | import java.util.logging.Level; 8 | import java.util.logging.Logger; 9 | import javax.annotation.PostConstruct; 10 | import javax.annotation.PreDestroy; 11 | import javax.ejb.Lock; 12 | import javax.ejb.LockType; 13 | import javax.ejb.Singleton; 14 | import javax.websocket.Session; 15 | 16 | @Singleton 17 | public class Peers { 18 | private List peers; 19 | 20 | @PostConstruct 21 | public void init(){ 22 | peers = new ArrayList<>(); 23 | } 24 | 25 | public void add(Session peer){ 26 | peers.add(peer); 27 | System.out.println("Added peer "+ peer.getId() + " to list of peers"); 28 | } 29 | 30 | public void remove(Session peer){ 31 | peers.remove(peer); 32 | System.out.println("Removed peer "+ peer.getId() + " from list of peers"); 33 | } 34 | 35 | @Lock(LockType.READ) 36 | public List peers(){ 37 | System.out.println("Getting connected peers.."); 38 | return Collections.unmodifiableList(peers); 39 | } 40 | 41 | @PreDestroy 42 | public void closeAll(){ 43 | 44 | peers.stream() 45 | .filter(s -> s.isOpen()) 46 | .forEach(s -> { 47 | try { 48 | s.close(); 49 | } catch (IOException ex) { 50 | Logger.getLogger(Peers.class.getName()).log(Level.SEVERE, null, ex); 51 | } 52 | }); 53 | 54 | System.out.println("Closed all peer connections"); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /dashboard/src/main/java/com/wordpress/abhirockzz/kafEEne/websocket/model/Payload.java: -------------------------------------------------------------------------------- 1 | package com.wordpress.abhirockzz.kafEEne.websocket.model; 2 | 3 | public class Payload { 4 | 5 | private String key; 6 | private String val; 7 | private String topic; 8 | 9 | public Payload(String key, String val, String topic) { 10 | this.key = key; 11 | this.val = val; 12 | this.topic = topic; 13 | } 14 | 15 | public Payload() { 16 | } 17 | 18 | public void setKey(String key) { 19 | this.key = key; 20 | } 21 | 22 | public void setVal(String val) { 23 | this.val = val; 24 | } 25 | 26 | public void setTopic(String topic) { 27 | this.topic = topic; 28 | } 29 | 30 | public String getKey() { 31 | return key; 32 | } 33 | 34 | public String getVal() { 35 | return val; 36 | } 37 | 38 | public String getTopic() { 39 | return topic; 40 | } 41 | 42 | @Override 43 | public String toString() { 44 | return "Payload{" + "key=" + key + ", val=" + val + ", topic=" + topic + '}'; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /dashboard/src/main/webapp/WEB-INF/beans.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | -------------------------------------------------------------------------------- /dashboard/src/main/webapp/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | kafEEne 7 | 8 | 9 | 10 | 22 | 23 | 24 | 25 |
26 | 27 |
28 |

kafEEne - Kafka + Websocket

29 | Topic name: 30 | 31 | 32 |

33 |

34 |

35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 |
TopicKeyValue
48 |
49 | 50 | 78 | 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | zookeeper: 2 | image: wurstmeister/zookeeper 3 | container_name: "zk" 4 | ports: 5 | - "2181:2181" 6 | kafka: 7 | image: wurstmeister/kafka 8 | container_name: "kafka" 9 | ports: 10 | - "9092:9092" 11 | environment: 12 | KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100 13 | KAFKA_ZOOKEEPER_CONNECT: zk:2181 14 | KAFKA_CREATE_TOPICS: "topic-1:10:1,topic-2:10:1" 15 | links: 16 | - zookeeper:zk 17 | volumes: 18 | - /var/run/docker.sock:/var/run/docker.sock 19 | 20 | dashboard: 21 | build: dashboard 22 | container_name: "dashboard" 23 | environment: 24 | - KAFKA_BROKER=kafka:9092 25 | ports: 26 | - "8080:8080" 27 | links: 28 | - kafka:kafka 29 | 30 | producer: 31 | build: producer 32 | container_name: "producer" 33 | environment: 34 | - KAFKA_BROKER=kafka:9092 35 | links: 36 | - kafka:kafka -------------------------------------------------------------------------------- /producer/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ -------------------------------------------------------------------------------- /producer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM anapsix/alpine-java:latest 2 | 3 | RUN mkdir app 4 | 5 | WORKDIR "/app" 6 | 7 | COPY target/kafka-producer.jar . 8 | 9 | CMD ["java", "-jar", "kafka-producer.jar"] 10 | -------------------------------------------------------------------------------- /producer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | com.wordpress.abhirockzz 5 | kafka-producer 6 | 1.0 7 | jar 8 | 9 | kafka-producer 10 | 11 | UTF-8 12 | 1.8 13 | 1.8 14 | 15 | 16 | 17 | org.apache.kafka 18 | kafka-clients 19 | 0.10.2.1 20 | 21 | 22 | 23 | 24 | kafka-producer 25 | 26 | 27 | 28 | org.apache.maven.plugins 29 | maven-shade-plugin 30 | 2.3 31 | 32 | true 33 | 34 | 35 | *:* 36 | 37 | META-INF/*.SF 38 | META-INF/*.DSA 39 | META-INF/*.RSA 40 | 41 | 42 | 43 | 44 | 45 | 46 | package 47 | 48 | shade 49 | 50 | 51 | 52 | 53 | 54 | com.wordpress.abhirockzz.kafEEne.websocket.producer.ProducerBootstrap 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /producer/src/main/java/com/wordpress/abhirockzz/kafEEne/websocket/producer/Producer.java: -------------------------------------------------------------------------------- 1 | package com.wordpress.abhirockzz.kafEEne.websocket.producer; 2 | 3 | import java.util.Properties; 4 | import java.util.Random; 5 | import java.util.logging.Level; 6 | import java.util.logging.Logger; 7 | 8 | import org.apache.kafka.clients.producer.KafkaProducer; 9 | import org.apache.kafka.clients.producer.ProducerConfig; 10 | import org.apache.kafka.clients.producer.ProducerRecord; 11 | 12 | public class Producer implements Runnable { 13 | 14 | private static final Logger LOGGER = Logger.getLogger(Producer.class.getName()); 15 | private KafkaProducer kafkaProducer = null; 16 | 17 | public Producer() { 18 | LOGGER.log(Level.INFO, "Kafka Producer running in thread {0}", Thread.currentThread().getName()); 19 | 20 | Properties kafkaProps = new Properties(); 21 | 22 | String kafkaCluster = System.getenv().get("KAFKA_BROKER"); 23 | LOGGER.log(Level.INFO, "Kafka cluster {0}", kafkaCluster); 24 | 25 | kafkaProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaCluster); 26 | kafkaProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); 27 | kafkaProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); 28 | kafkaProps.put(ProducerConfig.ACKS_CONFIG, "0"); 29 | 30 | this.kafkaProducer = new KafkaProducer<>(kafkaProps); 31 | 32 | } 33 | 34 | @Override 35 | public void run() { 36 | try { 37 | produce(); 38 | } catch (Exception e) { 39 | LOGGER.log(Level.SEVERE, e.getMessage(), e); 40 | } 41 | } 42 | 43 | private void produce() throws Exception { 44 | 45 | try { 46 | while (true) { 47 | 48 | kafkaProducer.send(record("topic-1")); 49 | kafkaProducer.send(record("topic-2")); 50 | 51 | Thread.sleep(5000); 52 | 53 | } 54 | } catch (Exception e) { 55 | LOGGER.log(Level.SEVERE, "Producer thread was interrupted"); 56 | } finally { 57 | kafkaProducer.close(); 58 | 59 | LOGGER.log(Level.INFO, "Producer closed"); 60 | } 61 | 62 | } 63 | static Random rnd = new Random(); 64 | private ProducerRecord record(String topic) { 65 | 66 | String key = "key-" + rnd.nextInt(10); 67 | String value = "value-" + rnd.nextInt(10); 68 | ProducerRecord record = new ProducerRecord<>(topic, key, value);; 69 | return record; 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /producer/src/main/java/com/wordpress/abhirockzz/kafEEne/websocket/producer/ProducerBootstrap.java: -------------------------------------------------------------------------------- 1 | package com.wordpress.abhirockzz.kafEEne.websocket.producer; 2 | 3 | public class ProducerBootstrap { 4 | 5 | public static void main(String[] args) throws Exception { 6 | 7 | new Thread(new Producer()).start(); 8 | 9 | } 10 | } 11 | --------------------------------------------------------------------------------