├── .gitignore ├── README.md ├── pom.xml └── src └── main ├── java └── com │ └── aggregator │ ├── actors │ ├── DbAggregatorActor.java │ ├── DbSaverActor.java │ └── InitialBaseActor.java │ ├── configuration │ ├── AppContextBoot.java │ ├── KafkaConfig.java │ ├── SpringActorProducer.java │ └── SpringExtension.java │ ├── converters │ ├── WeatherAvgNotificationConverter.java │ └── WeatherNotificationConverter.java │ ├── db │ ├── IWeatherRepository.java │ └── WeatherRepository.java │ ├── kafka │ ├── AbstractKafkaConsumer.java │ ├── DefaultKafkaProducer.java │ ├── IConsumer.java │ ├── IProducer.java │ ├── WeatherAvgNotificationConsumer.java │ ├── WeatherAvgNotificationKafkaConsumer.java │ ├── WeatherNotificationConsumer.java │ └── WeatherNotificationKafkaConsumer.java │ └── model │ ├── WeatherAverageNotification.java │ └── WeatherNotification.java └── resources ├── application.properties └── logback.xml /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | 3 | # Mobile Tools for Java (J2ME) 4 | .mtj.tmp/ 5 | 6 | .idea/ 7 | target/ 8 | .DS_Store 9 | 10 | # Package Files # 11 | *.jar 12 | *.war 13 | *.ear 14 | 15 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 16 | hs_err_pid* 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # akka-spring-boot 2 | Small app using spring, akka, kafka, mongo db 3 | 4 | - you need mongo db and kafka to run the app 5 | - start zookeeper and kafka broker 6 | - start mongo 7 | - run the app 8 | 9 | 10 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | com.aggregator 7 | aggregator 8 | 1.0 9 | 10 | 11 | 4.1.6.RELEASE 12 | 2.10 13 | UTF-8 14 | 0.8.2.2 15 | 1.8.2.RELEASE 16 | 1.2.3.RELEASE 17 | 2.2.4 18 | 2.3.14 19 | 1.7.5 20 | 1.1.3 21 | 3.3 22 | 1.8 23 | 24 | 25 | 26 | 27 | org.slf4j 28 | jul-to-slf4j 29 | ${slf4j.version} 30 | 31 | 32 | 33 | org.slf4j 34 | jcl-over-slf4j 35 | ${slf4j.version} 36 | 37 | 38 | 39 | org.slf4j 40 | log4j-over-slf4j 41 | ${slf4j.version} 42 | 43 | 44 | 45 | org.slf4j 46 | slf4j-api 47 | ${slf4j.version} 48 | 49 | 50 | 51 | ch.qos.logback 52 | logback-classic 53 | ${logback.version} 54 | 55 | 56 | 57 | ch.qos.logback 58 | logback-core 59 | ${logback.version} 60 | 61 | 62 | 63 | org.springframework 64 | spring-core 65 | ${springframework.version} 66 | 67 | 68 | commons-logging 69 | commons-logging 70 | 71 | 72 | 73 | 74 | 75 | org.springframework 76 | spring-context 77 | ${springframework.version} 78 | 79 | 80 | 81 | org.apache.kafka 82 | kafka_${scala-binaries.version} 83 | ${kafka.version} 84 | compile 85 | 86 | 87 | jmxri 88 | com.sun.jmx 89 | 90 | 91 | jms 92 | javax.jms 93 | 94 | 95 | jmxtools 96 | com.sun.jdmk 97 | 98 | 99 | slf4j-log4j12 100 | org.slf4j 101 | 102 | 103 | slf4j-api 104 | org.slf4j 105 | 106 | 107 | log4j 108 | log4j 109 | 110 | 111 | 112 | 113 | 114 | com.google.code.gson 115 | gson 116 | ${gson.version} 117 | 118 | 119 | 120 | org.springframework.boot 121 | spring-boot-starter-data-mongodb 122 | ${boot} 123 | 124 | 125 | 126 | org.springframework.boot 127 | spring-boot-starter 128 | ${boot} 129 | 130 | 131 | org.springframework.boot 132 | spring-boot-starter-logging 133 | 134 | 135 | 136 | 137 | 138 | org.springframework.data 139 | spring-data-mongodb 140 | ${spring.data.version} 141 | 142 | 143 | org.slf4j 144 | slf4j-api 145 | 146 | 147 | org.slf4j 148 | jcl-over-slf4j 149 | 150 | 151 | 152 | 153 | 154 | com.typesafe.akka 155 | akka-actor_${scala-binaries.version} 156 | ${akka.version} 157 | 158 | 159 | 160 | com.typesafe.akka 161 | akka-slf4j_2.10 162 | ${akka.version} 163 | 164 | 165 | 166 | org.scala-lang 167 | scala-library 168 | ${scala-binaries.version}.3 169 | 170 | 171 | 172 | javax.inject 173 | javax.inject 174 | 1 175 | 176 | 177 | 178 | 179 | 180 | aggregator 181 | 182 | 183 | org.springframework.boot 184 | spring-boot-maven-plugin 185 | ${boot} 186 | 187 | 188 | org.apache.maven.plugins 189 | maven-compiler-plugin 190 | ${maven-compiler-plugin.version} 191 | 192 | ${maven.compiler} 193 | ${maven.compiler} 194 | 195 | 196 | 197 | 198 | 199 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/actors/DbAggregatorActor.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.actors; 2 | 3 | import akka.actor.UntypedActor; 4 | import com.aggregator.db.IWeatherRepository; 5 | import com.aggregator.kafka.IProducer; 6 | import com.aggregator.model.WeatherAverageNotification; 7 | import com.aggregator.model.WeatherNotification; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.context.annotation.Scope; 12 | 13 | import javax.inject.Named; 14 | 15 | /** 16 | * Author: Yuliia Vovk 17 | * Date: 22.02.16 18 | * Time: 22:46 19 | */ 20 | @Named("DbAggregatorActor") 21 | @Scope("prototype") 22 | public class DbAggregatorActor extends UntypedActor { 23 | 24 | private static final Logger LOGGER = LoggerFactory.getLogger(DbAggregatorActor.class); 25 | 26 | @Autowired 27 | private IWeatherRepository weatherRepository; 28 | 29 | @Autowired 30 | private IProducer weatherAvgNotProducer; 31 | 32 | @Override 33 | public void preStart() { 34 | LOGGER.info("Starting db aggregator actor..."); 35 | } 36 | 37 | @Override 38 | public void onReceive(Object msg) throws Exception { 39 | WeatherNotification notification = (WeatherNotification) msg; 40 | LOGGER.debug("Aggregating by notification parameters: {}, {}", notification.getCity(), notification.getStreet()); 41 | WeatherAverageNotification weatherAvgNotification = 42 | weatherRepository.getAverageInfoByCityAndStreet(notification.getCity(), 43 | notification.getStreet()); 44 | LOGGER.debug("Saving to message broker: {}", weatherAvgNotification); 45 | weatherAvgNotProducer.produceWeatherAvgNotification(weatherAvgNotification); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/actors/DbSaverActor.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.actors; 2 | 3 | import akka.actor.ActorRef; 4 | import akka.actor.UntypedActor; 5 | import com.aggregator.db.IWeatherRepository; 6 | import com.aggregator.model.WeatherNotification; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.beans.factory.annotation.Qualifier; 11 | import org.springframework.context.annotation.Scope; 12 | 13 | import javax.inject.Named; 14 | 15 | /** 16 | * Author: Yuliia Vovk 17 | * Date: 22.02.16 18 | * Time: 22:45 19 | */ 20 | @Named("DbSaverActor") 21 | @Scope("prototype") 22 | public class DbSaverActor extends UntypedActor { 23 | 24 | private static final Logger LOGGER = LoggerFactory.getLogger(DbSaverActor.class); 25 | 26 | @Autowired 27 | private IWeatherRepository weatherRepository; 28 | 29 | @Autowired 30 | @Qualifier("dbAggregatorActor") 31 | private ActorRef childDbAggregatorActor; 32 | 33 | @Override 34 | public void preStart() { 35 | LOGGER.info("Starting db saver actor..."); 36 | } 37 | 38 | @Override 39 | public void onReceive(Object msg) { 40 | LOGGER.debug("Saving to db notification: {}", msg); 41 | weatherRepository.insert((WeatherNotification) msg); 42 | childDbAggregatorActor.tell(msg, getSelf()); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/actors/InitialBaseActor.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.actors; 2 | 3 | import akka.actor.ActorRef; 4 | import akka.actor.UntypedActor; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.beans.factory.annotation.Qualifier; 7 | import org.springframework.context.annotation.Scope; 8 | 9 | import javax.inject.Named; 10 | 11 | /** 12 | * Author: Yuliia Vovk 13 | * Date: 22.02.16 14 | * Time: 22:42 15 | */ 16 | @Named("InitialBaseActor") 17 | @Scope("prototype") 18 | public class InitialBaseActor extends UntypedActor { 19 | 20 | @Autowired 21 | @Qualifier("dbSaverActor") 22 | private ActorRef childDbSaverActor; 23 | 24 | @Override 25 | public void onReceive(Object msg) throws Exception { 26 | childDbSaverActor.tell(msg, getSelf()); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/configuration/AppContextBoot.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.configuration; 2 | 3 | import akka.actor.ActorRef; 4 | import akka.actor.ActorSystem; 5 | import com.mongodb.MongoClient; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.boot.SpringApplication; 9 | import org.springframework.boot.autoconfigure.SpringBootApplication; 10 | import org.springframework.context.ApplicationContext; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.context.annotation.ComponentScan; 13 | import org.springframework.context.annotation.Lazy; 14 | import org.springframework.core.env.Environment; 15 | import org.springframework.data.mongodb.core.MongoOperations; 16 | import org.springframework.data.mongodb.core.MongoTemplate; 17 | import org.springframework.scheduling.annotation.EnableAsync; 18 | 19 | import java.util.concurrent.ExecutorService; 20 | import java.util.concurrent.Executors; 21 | 22 | /** 23 | * Author: Yuliia Vovk 24 | * Date: 22.02.16 25 | * Time: 16:12 26 | */ 27 | @SpringBootApplication 28 | @EnableAsync(proxyTargetClass = true) 29 | @ComponentScan("com.aggregator") 30 | public class AppContextBoot { 31 | 32 | @Autowired 33 | private Environment environment; 34 | 35 | @Autowired 36 | private ApplicationContext applicationContext; 37 | 38 | public static void main(String... args) { 39 | SpringApplication.run(AppContextBoot.class, args); 40 | } 41 | 42 | @Bean 43 | public MongoOperations mongoDbFactory() throws Exception { 44 | return new MongoTemplate(new MongoClient( 45 | environment.getProperty("spring.data.mongodb.host")), 46 | "spring.data.mongodb.database"); 47 | } 48 | 49 | @Bean(name = "notificationsExecutor") 50 | public ExecutorService messageExecutorService(@Value("${app.executor.size}") Integer executorSize) { 51 | return Executors.newFixedThreadPool(executorSize); 52 | } 53 | 54 | @Bean(destroyMethod = "shutdown") 55 | @Lazy(false) 56 | public ActorSystem provideActorSystem() { 57 | ActorSystem system = ActorSystem.create("akka-weather-actor"); 58 | // initialize the application context in the Akka Spring Extension 59 | SpringExtension.SpringExtProvider.get(system).initialize(applicationContext); 60 | return system; 61 | } 62 | 63 | @Lazy(false) 64 | @Bean(name = "initBaseActor") 65 | public ActorRef provideParentActorRef() { 66 | return provideActorSystem().actorOf(SpringExtension.SpringExtProvider 67 | .get(provideActorSystem()).props("InitialBaseActor"), "parent-actor"); 68 | } 69 | 70 | @Lazy(false) 71 | @Bean(name = "dbSaverActor") 72 | public ActorRef provideChildDbActorRef() { 73 | return provideActorSystem().actorOf(SpringExtension.SpringExtProvider 74 | .get(provideActorSystem()).props("DbSaverActor"), "child-db-saver-actor"); 75 | } 76 | 77 | @Lazy(false) 78 | @Bean(name = "dbAggregatorActor") 79 | public ActorRef provideChildDbAggregatorActorRef() { 80 | return provideActorSystem().actorOf(SpringExtension.SpringExtProvider 81 | .get(provideActorSystem()).props("DbAggregatorActor"), "child-db-aggregator-actor"); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/configuration/KafkaConfig.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.configuration; 2 | 3 | import com.aggregator.converters.WeatherAvgNotificationConverter; 4 | import com.aggregator.converters.WeatherNotificationConverter; 5 | import com.aggregator.kafka.*; 6 | import com.aggregator.model.WeatherAverageNotification; 7 | import com.aggregator.model.WeatherNotification; 8 | import kafka.consumer.Consumer; 9 | import kafka.consumer.ConsumerConfig; 10 | import kafka.consumer.KafkaStream; 11 | import kafka.javaapi.consumer.ConsumerConnector; 12 | import kafka.javaapi.producer.Producer; 13 | import kafka.producer.ProducerConfig; 14 | import kafka.serializer.Decoder; 15 | import kafka.serializer.StringDecoder; 16 | import kafka.utils.VerifiableProperties; 17 | import org.springframework.beans.factory.annotation.Autowired; 18 | import org.springframework.beans.factory.annotation.Value; 19 | import org.springframework.context.annotation.Bean; 20 | import org.springframework.context.annotation.Configuration; 21 | import org.springframework.context.annotation.Lazy; 22 | import org.springframework.context.annotation.Scope; 23 | import org.springframework.core.env.Environment; 24 | 25 | import java.util.*; 26 | import java.util.function.Supplier; 27 | 28 | /** 29 | * Author: Yuliia Vovk 30 | * Date: 22.02.16 31 | * Time: 16:12 32 | */ 33 | @Configuration 34 | public class KafkaConfig { 35 | 36 | @Autowired 37 | private Environment env; 38 | 39 | @Value("${threads.count:1}") 40 | private int threadCount; 41 | 42 | @Value("${metadata.broker.list}") 43 | private String brokerList; 44 | 45 | @Value("${zookeeper.connect}") 46 | private String zookeeperConnect; 47 | 48 | @Value("${weather.not.serializer.class}") 49 | private String weatherNotSerializer; 50 | 51 | @Value("${weather.average.not.serializer.class}") 52 | private String weatherAvgNotSerializer; 53 | 54 | @Value("${zookeeper.session.timeout.ms}") 55 | private int zookeeperSessionTimeout; 56 | 57 | @Value("${zookeeper.connection.timeout.ms}") 58 | private int zookeeperConnTimeout; 59 | 60 | @Value("${zookeeper.sync.time.ms}") 61 | private int zookeeperSyncTime; 62 | 63 | @Value("${auto.commit.interval.ms}") 64 | private int autoCommitInt; 65 | 66 | @Bean(name="weatherAvgNotConsumer") 67 | public IConsumer provideCommandUpdateConsumer() { 68 | return new WeatherAvgNotificationConsumer(); 69 | } 70 | 71 | @Bean(name="weatherNotConsumer") 72 | public IConsumer provideCommandConsumer() { 73 | return new WeatherNotificationConsumer(); 74 | } 75 | 76 | @Bean 77 | @Scope("prototype") 78 | public AbstractKafkaConsumer commandWeatherAvgNotConsumer() { 79 | return new WeatherAvgNotificationKafkaConsumer(); 80 | } 81 | 82 | @Bean 83 | @Scope("prototype") 84 | public AbstractKafkaConsumer commandWeatherNotConsumer() { 85 | return new WeatherNotificationKafkaConsumer(); 86 | } 87 | 88 | @Bean(name = "weatherAvgNotProducer", destroyMethod = "close") 89 | @Lazy(false) 90 | public Producer notificationWeatherAvgProducer() { 91 | Properties properties = new Properties(); 92 | properties.put("metadata.broker.list", brokerList); 93 | properties.put("key.serializer.class", "kafka.serializer.StringEncoder"); 94 | properties.put("serializer.class", weatherAvgNotSerializer); 95 | properties.put("partitioner.class", "kafka.producer.DefaultPartitioner"); 96 | return new Producer<>(new ProducerConfig(properties)); 97 | } 98 | 99 | @Bean(destroyMethod = "shutdown") 100 | @Lazy(false) 101 | public ConsumerConnector commandWeatherAvgNotConsumerConnector() { 102 | String groupId = "weather-avg-notification" + UUID.randomUUID().toString(); 103 | return createAndSubscribe(groupId, "weather-avg-notification", this::commandWeatherAvgNotConsumer, 104 | new WeatherAvgNotificationConverter(new VerifiableProperties())); 105 | } 106 | 107 | @Bean(destroyMethod = "shutdown") 108 | @Lazy(false) 109 | public ConsumerConnector commandWeatherNotConsumerConnector() { 110 | String groupId = "weather-notification" + UUID.randomUUID().toString(); 111 | return createAndSubscribe(groupId, "weather-notification", this::commandWeatherNotConsumer, 112 | new WeatherNotificationConverter(new VerifiableProperties())); 113 | } 114 | 115 | private ConsumerConnector createAndSubscribe(String groupId, String topicName, 116 | Supplier> consumerCreator, Decoder decoder) { 117 | Properties properties = consumerSharedProps(); 118 | properties.put("group.id", groupId); 119 | ConsumerConnector connector = Consumer.createJavaConsumerConnector(new ConsumerConfig(properties)); 120 | 121 | Map topicCountMap = new HashMap<>(); 122 | topicCountMap.put(topicName, threadCount); 123 | 124 | Map>> streams = connector.createMessageStreams(topicCountMap, 125 | new StringDecoder(new VerifiableProperties()), decoder); 126 | List> stream = streams.get(topicName); 127 | 128 | int thread = 0; 129 | for (final KafkaStream sm : stream) { 130 | AbstractKafkaConsumer consumer = consumerCreator.get(); 131 | consumer.subscribe(sm, thread); 132 | thread++; 133 | } 134 | return connector; 135 | } 136 | 137 | private Properties consumerSharedProps() { 138 | Properties props = new Properties(); 139 | props.put("zookeeper.connect", zookeeperConnect); 140 | props.put("zookeeper.session.timeout.ms", env.getProperty("zookeeper.session.timeout.ms")); 141 | props.put("zookeeper.connection.timeout.ms", env.getProperty("zookeeper.connection.timeout.ms")); 142 | props.put("zookeeper.sync.time.ms", env.getProperty("zookeeper.sync.time.ms")); 143 | props.put("auto.commit.interval.ms", env.getProperty("auto.commit.interval.ms")); 144 | return props; 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/configuration/SpringActorProducer.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.configuration; 2 | 3 | import akka.actor.Actor; 4 | import akka.actor.IndirectActorProducer; 5 | import org.springframework.context.ApplicationContext; 6 | 7 | /** 8 | * Author: Yuliia Vovk 9 | * Date: 23.02.16 10 | * Time: 15:37 11 | * 12 | * An actor producer that lets Spring create the Actor instances. 13 | */ 14 | public class SpringActorProducer implements IndirectActorProducer { 15 | 16 | final ApplicationContext applicationContext; 17 | final String actorBeanName; 18 | 19 | public SpringActorProducer(ApplicationContext applicationContext, 20 | String actorBeanName) { 21 | this.applicationContext = applicationContext; 22 | this.actorBeanName = actorBeanName; 23 | } 24 | 25 | @Override 26 | public Actor produce() { 27 | return (Actor) applicationContext.getBean(actorBeanName); 28 | } 29 | 30 | @Override 31 | public Class actorClass() { 32 | return (Class) applicationContext.getType(actorBeanName); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/configuration/SpringExtension.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.configuration; 2 | 3 | import akka.actor.AbstractExtensionId; 4 | import akka.actor.ExtendedActorSystem; 5 | import akka.actor.Extension; 6 | import akka.actor.Props; 7 | import org.springframework.context.ApplicationContext; 8 | 9 | /** 10 | * Author: Yuliia Vovk 11 | * Date: 23.02.16 12 | * Time: 15:38 13 | * 14 | * An Akka Extension to provide access to Spring managed Actor Beans. 15 | */ 16 | public class SpringExtension extends 17 | AbstractExtensionId { 18 | 19 | /** 20 | * The identifier used to access the SpringExtension. 21 | */ 22 | public static SpringExtension SpringExtProvider = new SpringExtension(); 23 | 24 | /** 25 | * Is used by Akka to instantiate the Extension identified by this 26 | * ExtensionId, internal use only. 27 | */ 28 | @Override 29 | public SpringExt createExtension(ExtendedActorSystem system) { 30 | return new SpringExt(); 31 | } 32 | 33 | /** 34 | * The Extension implementation. 35 | */ 36 | public static class SpringExt implements Extension { 37 | 38 | private volatile ApplicationContext applicationContext; 39 | 40 | /** 41 | * Used to initialize the Spring application context for the extension. 42 | * @param applicationContext 43 | */ 44 | public void initialize(ApplicationContext applicationContext) { 45 | this.applicationContext = applicationContext; 46 | } 47 | 48 | /** 49 | * Create a Props for the specified actorBeanName using the 50 | * SpringActorProducer class. 51 | * 52 | * @param actorBeanName The name of the actor bean to create Props for 53 | * @return a Props that will create the named actor bean using Spring 54 | */ 55 | public Props props(String actorBeanName) { 56 | return Props.create(SpringActorProducer.class, 57 | applicationContext, actorBeanName); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/converters/WeatherAvgNotificationConverter.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.converters; 2 | 3 | import com.aggregator.model.WeatherAverageNotification; 4 | import com.google.gson.Gson; 5 | import com.google.gson.GsonBuilder; 6 | import kafka.serializer.Decoder; 7 | import kafka.serializer.Encoder; 8 | import kafka.utils.VerifiableProperties; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.UnsupportedEncodingException; 13 | 14 | /** 15 | * Author: Yuliia Vovk 16 | * Date: 22.02.16 17 | * Time: 16:12 18 | */ 19 | public class WeatherAvgNotificationConverter implements Encoder, Decoder { 20 | 21 | private Gson gson; 22 | 23 | private static final Logger LOGGER = LoggerFactory.getLogger(WeatherAvgNotificationConverter.class); 24 | 25 | public WeatherAvgNotificationConverter(VerifiableProperties verifiableProperties) { 26 | gson = new GsonBuilder().disableHtmlEscaping().create(); 27 | } 28 | 29 | @Override 30 | public byte[] toBytes(WeatherAverageNotification weatherAvgNotification) { 31 | return toJsonString(weatherAvgNotification).getBytes(); 32 | } 33 | 34 | public String toJsonString(WeatherAverageNotification weatherAvgNotification) { 35 | return gson.toJson(weatherAvgNotification); 36 | } 37 | 38 | @Override 39 | public WeatherAverageNotification fromBytes(byte[] bytes) { 40 | try { 41 | return gson.fromJson(new String(bytes, "UTF-8"), WeatherAverageNotification.class); 42 | } catch (UnsupportedEncodingException e) { 43 | LOGGER.error("Can't convert json to object ", e.getMessage()); 44 | } 45 | return null; 46 | } 47 | 48 | public WeatherAverageNotification fromString(String string) { 49 | return gson.fromJson(string, WeatherAverageNotification.class); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/converters/WeatherNotificationConverter.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.converters; 2 | 3 | import com.aggregator.model.WeatherNotification; 4 | import com.google.gson.Gson; 5 | import com.google.gson.GsonBuilder; 6 | import kafka.serializer.Decoder; 7 | import kafka.serializer.Encoder; 8 | import kafka.utils.VerifiableProperties; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.UnsupportedEncodingException; 13 | 14 | /** 15 | * Author: Yuliia Vovk 16 | * Date: 22.02.16 17 | * Time: 16:12 18 | */ 19 | public class WeatherNotificationConverter implements Encoder, Decoder { 20 | 21 | private Gson gson; 22 | 23 | private static final Logger LOGGER = LoggerFactory.getLogger(WeatherNotificationConverter.class); 24 | 25 | public WeatherNotificationConverter(VerifiableProperties verifiableProperties) { 26 | gson = new GsonBuilder().disableHtmlEscaping().create(); 27 | } 28 | 29 | @Override 30 | public byte[] toBytes(WeatherNotification weatherNotification) { 31 | return toJsonString(weatherNotification).getBytes(); 32 | } 33 | 34 | public String toJsonString(WeatherNotification weatherNotification) { 35 | return gson.toJson(weatherNotification); 36 | } 37 | 38 | @Override 39 | public WeatherNotification fromBytes(byte[] bytes) { 40 | try { 41 | return gson.fromJson(new String(bytes, "UTF-8"), WeatherNotification.class); 42 | } catch (UnsupportedEncodingException e) { 43 | LOGGER.error("Can't convert json to object ", e.getMessage()); 44 | } 45 | return null; 46 | } 47 | 48 | public WeatherNotification fromString(String string) { 49 | return gson.fromJson(string, WeatherNotification.class); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/db/IWeatherRepository.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.db; 2 | 3 | import com.aggregator.model.WeatherAverageNotification; 4 | import com.aggregator.model.WeatherNotification; 5 | import org.springframework.data.mongodb.core.query.Query; 6 | 7 | import java.util.List; 8 | 9 | /** 10 | * Author: Yuliia Vovk 11 | * Date: 22.02.16 12 | * Time: 16:32 13 | */ 14 | public interface IWeatherRepository { 15 | 16 | void insert(WeatherNotification notification); 17 | List findAll(Query query); 18 | WeatherAverageNotification getAverageInfoByCityAndStreet(String city, String street); 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/db/WeatherRepository.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.db; 2 | 3 | import com.aggregator.model.WeatherAverageNotification; 4 | import com.aggregator.model.WeatherNotification; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.data.mongodb.core.MongoOperations; 7 | import org.springframework.data.mongodb.core.query.Query; 8 | import org.springframework.stereotype.Repository; 9 | 10 | import java.util.List; 11 | 12 | import static org.springframework.data.mongodb.core.query.Criteria.where; 13 | import static org.springframework.data.mongodb.core.query.Query.query; 14 | 15 | /** 16 | * Author: Yuliia Vovk 17 | * Date: 22.02.16 18 | * Time: 16:50 19 | */ 20 | @Repository 21 | public class WeatherRepository implements IWeatherRepository { 22 | 23 | private static final String COLLECTION_NAME = "notifications"; 24 | 25 | @Autowired 26 | private MongoOperations mongoOperations; 27 | 28 | @Override 29 | public void insert(WeatherNotification notification) { 30 | mongoOperations.insert(notification, COLLECTION_NAME); 31 | } 32 | 33 | @Override 34 | public List findAll(Query query) { 35 | return mongoOperations.find(query, WeatherNotification.class, COLLECTION_NAME); 36 | } 37 | 38 | @Override 39 | public WeatherAverageNotification getAverageInfoByCityAndStreet(String city, String street) { 40 | List notifications = findAll(query(where("city").is(city).and("street").is(street))); 41 | 42 | float avgTemperature = notifications 43 | .parallelStream() 44 | .map(WeatherNotification::getTemp) 45 | .reduce(Float::sum) 46 | .get()/notifications.size(); 47 | 48 | float avgLight = notifications 49 | .parallelStream() 50 | .map(WeatherNotification::getLight) 51 | .reduce(Float::sum) 52 | .get()/notifications.size(); 53 | 54 | return new WeatherAverageNotification(city, street, avgTemperature, avgLight); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/kafka/AbstractKafkaConsumer.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.kafka; 2 | 3 | import kafka.consumer.KafkaStream; 4 | import kafka.message.MessageAndMetadata; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.scheduling.annotation.Async; 8 | 9 | /** 10 | * Author: Yuliia Vovk 11 | * Date: 22.02.16 12 | * Time: 16:32 13 | */ 14 | public abstract class AbstractKafkaConsumer implements IConsumer { 15 | 16 | private static final Logger LOGGER = LoggerFactory.getLogger(AbstractKafkaConsumer.class); 17 | 18 | @Async("notificationsExecutor") 19 | public void subscribe(KafkaStream a_stream, int a_threadNumber) { 20 | Thread.currentThread().setName("kafka-consumer-" + Thread.currentThread().getName()); 21 | LOGGER.info("Kafka consumer started, thread {} ", a_threadNumber); 22 | for (MessageAndMetadata anA_stream : (Iterable>) a_stream) { 23 | T message = anA_stream.message(); 24 | LOGGER.debug("Message arrived -> 'thread_name': {}, 'thread_number': {}, 'message': {}", Thread.currentThread().getName(), a_threadNumber, message); 25 | submitMessage(message); 26 | } 27 | LOGGER.info("Shutting down Thread: " + a_threadNumber); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/kafka/DefaultKafkaProducer.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.kafka; 2 | 3 | import com.aggregator.model.WeatherAverageNotification; 4 | import kafka.javaapi.producer.Producer; 5 | import kafka.producer.KeyedMessage; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.beans.factory.annotation.Qualifier; 8 | import org.springframework.stereotype.Component; 9 | 10 | /** 11 | * Author: Yuliia Vovk 12 | * Date: 22.02.16 13 | * Time: 16:12 14 | */ 15 | @Component 16 | public class DefaultKafkaProducer implements IProducer { 17 | 18 | @Autowired 19 | @Qualifier("weatherAvgNotProducer") 20 | private Producer notificationProducer; 21 | 22 | public void produceWeatherAvgNotification(WeatherAverageNotification message) { 23 | notificationProducer.send(new KeyedMessage<>("weather-avg-notification", message.getCity(), message)); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/kafka/IConsumer.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.kafka; 2 | 3 | /** 4 | * Author: Yuliia Vovk 5 | * Date: 22.02.16 6 | * Time: 16:32 7 | */ 8 | public interface IConsumer { 9 | 10 | void submitMessage(T message); 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/kafka/IProducer.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.kafka; 2 | 3 | import com.aggregator.model.WeatherAverageNotification; 4 | 5 | /** 6 | * Author: Yuliia Vovk 7 | * Date: 22.02.16 8 | * Time: 16:12 9 | */ 10 | public interface IProducer { 11 | 12 | void produceWeatherAvgNotification(WeatherAverageNotification message); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/kafka/WeatherAvgNotificationConsumer.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.kafka; 2 | 3 | import com.aggregator.model.WeatherAverageNotification; 4 | 5 | /** 6 | * Author: Y. Vovk 7 | * 08.02.16. 8 | */ 9 | public class WeatherAvgNotificationConsumer implements IConsumer { 10 | 11 | @Override 12 | public void submitMessage(final WeatherAverageNotification message) { 13 | // 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/kafka/WeatherAvgNotificationKafkaConsumer.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.kafka; 2 | 3 | import com.aggregator.model.WeatherAverageNotification; 4 | import org.springframework.beans.factory.annotation.Autowired; 5 | import org.springframework.beans.factory.annotation.Qualifier; 6 | 7 | /** 8 | * Author: Yuliia Vovk 9 | * Date: 22.02.16 10 | * Time: 16:32 11 | */ 12 | public class WeatherAvgNotificationKafkaConsumer extends AbstractKafkaConsumer { 13 | 14 | @Autowired 15 | @Qualifier("weatherAvgNotConsumer") 16 | private IConsumer consumer; 17 | 18 | @Override 19 | public void submitMessage(final WeatherAverageNotification message) { 20 | consumer.submitMessage(message); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/kafka/WeatherNotificationConsumer.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.kafka; 2 | 3 | import akka.actor.ActorRef; 4 | import com.aggregator.model.WeatherNotification; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.beans.factory.annotation.Qualifier; 7 | 8 | /** 9 | * Author: Yuliia Vovk 10 | * Date: 22.02.16 11 | * Time: 17:18 12 | */ 13 | public class WeatherNotificationConsumer implements IConsumer { 14 | 15 | @Autowired 16 | @Qualifier("initBaseActor") 17 | private ActorRef mainActor; 18 | 19 | @Override 20 | public void submitMessage(final WeatherNotification message) { 21 | mainActor.tell(message, null); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/kafka/WeatherNotificationKafkaConsumer.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.kafka; 2 | 3 | import com.aggregator.model.WeatherNotification; 4 | import org.springframework.beans.factory.annotation.Autowired; 5 | import org.springframework.beans.factory.annotation.Qualifier; 6 | 7 | /** 8 | * Author: Yuliia Vovk 9 | * Date: 22.02.16 10 | * Time: 17:18 11 | */ 12 | public class WeatherNotificationKafkaConsumer extends AbstractKafkaConsumer{ 13 | 14 | @Autowired 15 | @Qualifier("weatherNotConsumer") 16 | private IConsumer consumer; 17 | 18 | @Override 19 | public void submitMessage(final WeatherNotification message) { 20 | consumer.submitMessage(message); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/model/WeatherAverageNotification.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.model; 2 | 3 | /** 4 | * Author: Yuliia Vovk 5 | * Date: 22.02.16 6 | * Time: 16:25 7 | */ 8 | public class WeatherAverageNotification { 9 | 10 | private String city; 11 | private String street; 12 | private float averageTemp; 13 | private float averageLight; 14 | 15 | public WeatherAverageNotification() { 16 | } 17 | 18 | public WeatherAverageNotification(String city, String street, float averageTemp, float averageLight) { 19 | this.city = city; 20 | this.street = street; 21 | this.averageTemp = averageTemp; 22 | this.averageLight = averageLight; 23 | } 24 | 25 | public String getCity() { 26 | return city; 27 | } 28 | 29 | public void setCity(String city) { 30 | this.city = city; 31 | } 32 | 33 | public String getStreet() { 34 | return street; 35 | } 36 | 37 | public void setStreet(String street) { 38 | this.street = street; 39 | } 40 | 41 | public float getAverageTemp() { 42 | return averageTemp; 43 | } 44 | 45 | public void setAverageTemp(float averageTemp) { 46 | this.averageTemp = averageTemp; 47 | } 48 | 49 | public float getAverageLight() { 50 | return averageLight; 51 | } 52 | 53 | public void setAverageLight(float averageLight) { 54 | this.averageLight = averageLight; 55 | } 56 | 57 | @Override 58 | public String toString() { 59 | final StringBuilder sb = new StringBuilder("WeatherAverageNotification{"); 60 | sb.append("city='").append(city).append('\''); 61 | sb.append(", street='").append(street).append('\''); 62 | sb.append(", averageTemp=").append(averageTemp); 63 | sb.append(", averageLight=").append(averageLight); 64 | sb.append('}'); 65 | return sb.toString(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/aggregator/model/WeatherNotification.java: -------------------------------------------------------------------------------- 1 | package com.aggregator.model; 2 | 3 | /** 4 | * Author: Yuliia Vovk 5 | * Date: 22.02.16 6 | * Time: 16:23 7 | */ 8 | public class WeatherNotification { 9 | 10 | private long timestamp; 11 | private String city; 12 | private String street; 13 | private float temp; 14 | private float light; 15 | 16 | public long getTimestamp() { 17 | return timestamp; 18 | } 19 | 20 | public void setTimestamp(long timestamp) { 21 | this.timestamp = timestamp; 22 | } 23 | 24 | public String getCity() { 25 | return city; 26 | } 27 | 28 | public void setCity(String city) { 29 | this.city = city; 30 | } 31 | 32 | public String getStreet() { 33 | return street; 34 | } 35 | 36 | public void setStreet(String street) { 37 | this.street = street; 38 | } 39 | 40 | public float getTemp() { 41 | return temp; 42 | } 43 | 44 | public void setTemp(float temp) { 45 | this.temp = temp; 46 | } 47 | 48 | public float getLight() { 49 | return light; 50 | } 51 | 52 | public void setLight(float light) { 53 | this.light = light; 54 | } 55 | 56 | @Override 57 | public String toString() { 58 | final StringBuilder sb = new StringBuilder("WeatherNotification{"); 59 | sb.append("timestamp=").append(timestamp); 60 | sb.append(", city='").append(city).append('\''); 61 | sb.append(", street='").append(street).append('\''); 62 | sb.append(", temp=").append(temp); 63 | sb.append(", light=").append(light); 64 | sb.append('}'); 65 | return sb.toString(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # Kafka properties 2 | metadata.broker.list=127.0.0.1:9092 3 | zookeeper.connect=127.0.0.1:2181 4 | threads.count=1 5 | 6 | #Kafka producer properties 7 | weather.not.serializer.class=com.aggregator.converters.WeatherNotificationConverter 8 | weather.average.not.serializer.class=com.aggregator.converters.WeatherAvgNotificationConverter 9 | 10 | #Kafka consumer properties 11 | zookeeper.session.timeout.ms=1000 12 | zookeeper.connection.timeout.ms=20000 13 | zookeeper.sync.time.ms=200 14 | auto.commit.interval.ms=1000 15 | 16 | #MongoDB 17 | spring.data.mongodb.host=localhost 18 | spring.data.mongodb.database=weather-markers 19 | 20 | #consumer-executor 21 | app.executor.size=20 22 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${entry.pattern} 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | --------------------------------------------------------------------------------