├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── pom.xml └── src └── main ├── java └── com │ └── utiltube │ └── kafka │ ├── app │ └── VideoConsumerApplication.java │ ├── config │ └── ConsumerConfigFactory.java │ └── video │ ├── consumer │ ├── ConsumerThreadPool.java │ └── VideoConsumer.java │ └── model │ └── Video.java └── resources └── application.properties /.gitignore: -------------------------------------------------------------------------------- 1 | ### Maven ### 2 | target/ 3 | pom.xml.tag 4 | pom.xml.releaseBackup 5 | pom.xml.versionsBackup 6 | pom.xml.next 7 | release.properties 8 | 9 | 10 | ### Java ### 11 | *.class 12 | 13 | # Mobile Tools for Java (J2ME) 14 | .mtj.tmp/ 15 | 16 | # Package Files # 17 | *.jar 18 | *.war 19 | *.ear 20 | 21 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 22 | hs_err_pid* 23 | 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - oraclejdk7 4 | - openjdk7 5 | 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 José David Baena 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # spring-boot-kafka-video-consumer 2 | Spring boot example for consuming a video from a kafka topic 3 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.utiltube.video 7 | video-consumer 8 | 0.0.1-SNAPSHOT 9 | jar 10 | 11 | video-consumer 12 | Kafka video consumer project for Spring Boot 13 | 14 | 15 | org.springframework.boot 16 | spring-boot-starter-parent 17 | 1.2.2.RELEASE 18 | 19 | 20 | 21 | 22 | UTF-8 23 | consumer.VideoConsumerApplication 24 | 1.7 25 | 18.0 26 | 0.8.1.1 27 | 28 | 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-starter-remote-shell 33 | 34 | 35 | 36 | org.springframework.boot 37 | spring-boot-starter-test 38 | test 39 | 40 | 41 | 42 | org.apache.kafka 43 | kafka_2.8.0 44 | ${kafka.version} 45 | 46 | 47 | 48 | com.google.guava 49 | guava 50 | ${guava.version} 51 | 52 | 53 | 54 | 55 | 56 | 57 | org.springframework.boot 58 | spring-boot-maven-plugin 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /src/main/java/com/utiltube/kafka/app/VideoConsumerApplication.java: -------------------------------------------------------------------------------- 1 | package com.utiltube.kafka.app; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.annotation.ComponentScan; 6 | 7 | @SpringBootApplication 8 | @ComponentScan("com.utiltube") 9 | public class VideoConsumerApplication { 10 | 11 | public static void main(String[] args) { 12 | SpringApplication.run(VideoConsumerApplication.class, args); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/com/utiltube/kafka/config/ConsumerConfigFactory.java: -------------------------------------------------------------------------------- 1 | package com.utiltube.kafka.config; 2 | 3 | import java.util.Properties; 4 | 5 | import javax.annotation.PostConstruct; 6 | 7 | import kafka.consumer.ConsumerConfig; 8 | 9 | import org.springframework.stereotype.Component; 10 | 11 | @Component 12 | public class ConsumerConfigFactory { 13 | 14 | private static final String ZK_CONNECT = "localhost:2181"; 15 | 16 | private ConsumerConfig consumerConfig; 17 | 18 | @PostConstruct 19 | private void createConsumerConfig() { 20 | Properties props = new Properties(); 21 | props.put("zookeeper.connect", ZK_CONNECT); 22 | props.put("group.id", "Video-cg-0"); 23 | props.put("zookeeper.session.timeout.ms", "400"); 24 | props.put("zookeeper.sync.time.ms", "200"); 25 | props.put("auto.commit.interval.ms", "1000"); 26 | consumerConfig = new ConsumerConfig(props); 27 | } 28 | 29 | public ConsumerConfig getConsumerConfig() { 30 | return consumerConfig; 31 | } 32 | } -------------------------------------------------------------------------------- /src/main/java/com/utiltube/kafka/video/consumer/ConsumerThreadPool.java: -------------------------------------------------------------------------------- 1 | package com.utiltube.kafka.video.consumer; 2 | 3 | import static kafka.consumer.Consumer.createJavaConsumerConnector; 4 | 5 | import java.util.HashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | import java.util.concurrent.ExecutorService; 9 | import java.util.concurrent.Executors; 10 | 11 | import javax.annotation.PostConstruct; 12 | 13 | import kafka.consumer.ConsumerConfig; 14 | import kafka.consumer.KafkaStream; 15 | import kafka.javaapi.consumer.ConsumerConnector; 16 | 17 | import org.springframework.beans.factory.annotation.Autowired; 18 | import org.springframework.stereotype.Component; 19 | 20 | import com.utiltube.kafka.config.ConsumerConfigFactory; 21 | 22 | @Component 23 | public class ConsumerThreadPool { 24 | 25 | private static final String TOPIC = "video_test"; 26 | private static final Integer NUM_THREADS = 1; 27 | 28 | @Autowired 29 | private ConsumerConfigFactory consumerConfigFactory; 30 | 31 | private ConsumerConnector consumer; 32 | private ExecutorService threadPool; 33 | 34 | public ConsumerThreadPool() { 35 | threadPool = Executors.newFixedThreadPool(NUM_THREADS); 36 | } 37 | 38 | @PostConstruct 39 | public void startConsuming() { 40 | ConsumerConfig consumerConfig = consumerConfigFactory.getConsumerConfig(); 41 | consumer = createJavaConsumerConnector(consumerConfig); 42 | 43 | consume(); 44 | } 45 | 46 | public void consume() { 47 | Map topicCountMap = new HashMap(); 48 | topicCountMap.put(TOPIC, NUM_THREADS); 49 | Map>> consumerMap = consumer.createMessageStreams(topicCountMap); 50 | List> streams = consumerMap.get(TOPIC); 51 | 52 | int threadNumber = 0; 53 | for (final KafkaStream stream : streams) { 54 | threadPool.submit(new VideoConsumer(stream, threadNumber)); 55 | threadNumber++; 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/utiltube/kafka/video/consumer/VideoConsumer.java: -------------------------------------------------------------------------------- 1 | package com.utiltube.kafka.video.consumer; 2 | import java.io.IOException; 3 | 4 | import com.fasterxml.jackson.core.JsonParseException; 5 | import com.fasterxml.jackson.databind.JsonMappingException; 6 | import com.fasterxml.jackson.databind.ObjectMapper; 7 | import com.utiltube.kafka.video.model.Video; 8 | 9 | import kafka.consumer.ConsumerIterator; 10 | import kafka.consumer.KafkaStream; 11 | 12 | public class VideoConsumer implements Runnable { 13 | private ObjectMapper objectMapper; 14 | private KafkaStream kafkaStream; 15 | private int threadNumber; 16 | 17 | public VideoConsumer(KafkaStream kafkaStream, int threadNumber) { 18 | this.threadNumber = threadNumber; 19 | this.kafkaStream = kafkaStream; 20 | this.objectMapper = new ObjectMapper(); 21 | } 22 | @Override 23 | public void run() { 24 | ConsumerIterator it = kafkaStream.iterator(); 25 | 26 | while (it.hasNext()) { 27 | byte[] messageData = it.next().message(); 28 | try { 29 | Video videoFromMessage = objectMapper.readValue(messageData, Video.class); 30 | System.out.println("Thread:" + threadNumber + ".Consuming video: " + videoFromMessage); 31 | } catch (JsonParseException | JsonMappingException e) { 32 | e.printStackTrace(); 33 | } catch (IOException e) { 34 | e.printStackTrace(); 35 | } 36 | 37 | } 38 | 39 | System.out.println("Shutting down Thread: " + kafkaStream); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/utiltube/kafka/video/model/Video.java: -------------------------------------------------------------------------------- 1 | package com.utiltube.kafka.video.model; 2 | 3 | public class Video { 4 | 5 | private static final int IMPOSSIBLE_ID = -1; 6 | 7 | private int id = IMPOSSIBLE_ID; 8 | private String identifier; 9 | private String provider; 10 | private String title; 11 | private String description; 12 | 13 | public int getId() { 14 | return id; 15 | } 16 | 17 | public String getIdentifier() { 18 | return identifier; 19 | } 20 | 21 | public String getProvider() { 22 | return provider; 23 | } 24 | 25 | public String getTitle() { 26 | return title; 27 | } 28 | 29 | public String getDescription() { 30 | return description; 31 | } 32 | 33 | @Override 34 | public String toString() { 35 | return "Video [identifier=" + identifier + ", provider=" + provider 36 | + ", title=" + title + "]"; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josedab/spring-boot-kafka-video-consumer/2149f168d1f9bdf8f542b74b29b314fbcf33450c/src/main/resources/application.properties --------------------------------------------------------------------------------