├── .gitignore ├── README.md ├── pom.xml └── src ├── main ├── documentation │ ├── data-model.png │ └── target-architecture.png ├── java │ └── com │ │ └── example │ │ ├── configuration │ │ └── ApplicationConfiguration.java │ │ ├── converter │ │ └── MessagingToDomainObjectConverter.java │ │ ├── domain │ │ └── UserActivityEvent.java │ │ ├── mapper │ │ └── JSONDeserialiser.java │ │ ├── messaging │ │ ├── CassandraDataSink.java │ │ └── KafkaConsumer.java │ │ ├── model │ │ ├── Event.java │ │ ├── EventByCorrelationId.java │ │ ├── EventByReference.java │ │ ├── EventByType.java │ │ ├── builder │ │ │ └── EventBuilder.java │ │ └── primarykey │ │ │ ├── CorrelationIdDateTime.java │ │ │ ├── EventTypeDateTime.java │ │ │ └── ReferenceDateTime.java │ │ ├── repository │ │ ├── EventsByCorrelationIdRepository.java │ │ ├── EventsByReferenceRepository.java │ │ └── EventsByTypeRepository.java │ │ └── service │ │ └── UserActivityEventsPersister.java └── resources │ ├── cassandra.properties │ └── log4j.properties └── test ├── java └── com │ └── example │ ├── configuration │ └── TestConfiguration.java │ ├── converter │ └── MessagingToDomainObjectConverterTest.java │ ├── mapper │ └── JSONDeserialiserTest.java │ └── repository │ ├── CQLScriptIntegrationTest.java │ └── CassandraRepositoriesIntegrationTest.java └── resources ├── cassandra-test.properties ├── kafka_event.json ├── log4j.properties └── users_activity_db.cql /.gitignore: -------------------------------------------------------------------------------- 1 | /flink-kafka-consumer.iml 2 | /.idea 3 | ### Java template 4 | *.class 5 | 6 | # Mobile Tools for Java (J2ME) 7 | .mtj.tmp/ 8 | 9 | # Package Files # 10 | *.jar 11 | *.war 12 | *.ear 13 | 14 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 15 | hs_err_pid* 16 | ### Eclipse template 17 | *.pydevproject 18 | .metadata 19 | .gradle 20 | bin/ 21 | tmp/ 22 | *.tmp 23 | *.bak 24 | *.swp 25 | *~.nib 26 | local.properties 27 | .settings/ 28 | .loadpath 29 | 30 | # Eclipse Core 31 | .project 32 | 33 | # External tool builders 34 | .externalToolBuilders/ 35 | 36 | # Locally stored "Eclipse launch configurations" 37 | *.launch 38 | 39 | # CDT-specific 40 | .cproject 41 | 42 | # JDT-specific (Eclipse Java Development Tools) 43 | .classpath 44 | 45 | # Java annotation processor (APT) 46 | .factorypath 47 | 48 | # PDT-specific 49 | .buildpath 50 | 51 | # sbteclipse plugin 52 | .target 53 | 54 | # TeXlipse plugin 55 | .texlipse 56 | 57 | # Created by .ignore support plugin (hsz.mobi) 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Description 2 | 3 | Demonstrates how one can integrate [kafka](http://kafka.apache.org/ "apache kafka"), [flink](https://flink.apache.org/ "apache flink") and [cassandra](http://cassandra.apache.org/ "apache cassandra") with [spring data](http://projects.spring.io/spring-data-cassandra/ "Spring data cassandra"). 4 | 5 | Please check the [producer module](https://github.com/viswanath7/flink-kafka-producer "Related producer module") in conjunction with this consumer for completion. 6 | 7 | ## Target architecture 8 | 9 | In the target architecture diagram shown below, this module implements kafka flink consumers that persist json events as entities in column families of cassandra using spring data cassandra. 10 | 11 | ![Target architecture](src/main/documentation/target-architecture.png "Target architecture") 12 | 13 | ## Data model 14 | 15 | Storage engine visualisation of column families stored in cassandra is as shown below 16 | 17 | ![Data model](src/main/documentation/data-model.png "Data model") 18 | 19 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 19 | 21 | 4.0.0 22 | 23 | com.example 24 | flink-kafka-consumer 25 | 0.1 26 | jar 27 | 28 | Flink kafka consumer 29 | http://www.myorganisation.org 30 | 31 | 32 | UTF-8 33 | 0.10.1 34 | 16.0.1 35 | 2.6.1 36 | 4.2.4.RELEASE 37 | 1.9.2.RELEASE 38 | 1.3.2.RELEASE 39 | 40 | 1.7.12 41 | 2.1.9 42 | 2.1.9.2 43 | 2.0-0 44 | 45 | 46 | 47 | 48 | 49 | apache.snapshots 50 | Apache Development Snapshot Repository 51 | https://repository.apache.org/content/repositories/snapshots/ 52 | 53 | false 54 | 55 | 56 | true 57 | 58 | 59 | 60 | 61 | 84 | 85 | 86 | 87 | org.apache.flink 88 | flink-java 89 | ${flink.version} 90 | 91 | 92 | org.apache.flink 93 | flink-streaming-java 94 | ${flink.version} 95 | 96 | 97 | org.apache.flink 98 | flink-clients 99 | ${flink.version} 100 | 101 | 102 | org.apache.flink 103 | flink-connector-kafka 104 | ${flink.version} 105 | 106 | 107 | org.springframework.data 108 | spring-data-cassandra 109 | ${spring.data.cassandra.version} 110 | 111 | 112 | org.springframework 113 | spring-core 114 | ${spring.version} 115 | 116 | 117 | org.hectorclient 118 | hector-core 119 | ${hector-core.version} 120 | 121 | 122 | com.datastax.cassandra 123 | cassandra-driver-core 124 | ${cassandra-driver-core.version} 125 | true 126 | 127 | 128 | org.slf4j 129 | slf4j-api 130 | ${org.slf4j.version} 131 | 132 | 133 | com.google.guava 134 | guava 135 | ${guava.version} 136 | 137 | 138 | com.fasterxml.jackson.datatype 139 | jackson-datatype-jsr310 140 | ${jackson-jsr310.version} 141 | 142 | 143 | 144 | 145 | 146 | org.springframework 147 | spring-test 148 | ${spring.version} 149 | test 150 | 151 | 152 | org.cassandraunit 153 | cassandra-unit-spring 154 | ${cassandra-unit.version} 155 | test 156 | 157 | 158 | org.cassandraunit 159 | cassandra-unit 160 | 161 | 162 | 163 | 164 | org.cassandraunit 165 | cassandra-unit-shaded 166 | ${cassandra-unit.version} 167 | test 168 | 169 | 170 | 171 | 172 | 173 | 174 | 177 | 178 | org.apache.maven.plugins 179 | maven-shade-plugin 180 | 2.4.1 181 | 182 | 183 | 184 | package 185 | 186 | shade 187 | 188 | 189 | 190 | 191 | 194 | org.apache.flink:flink-shaded-* 195 | org.apache.flink:flink-core 196 | org.apache.flink:flink-java 197 | org.apache.flink:flink-scala 198 | org.apache.flink:flink-runtime 199 | org.apache.flink:flink-optimizer 200 | org.apache.flink:flink-clients 201 | org.apache.flink:flink-avro 202 | org.apache.flink:flink-java-examples 203 | org.apache.flink:flink-scala-examples 204 | org.apache.flink:flink-streaming-examples 205 | org.apache.flink:flink-streaming-java 206 | 207 | 213 | org.scala-lang:scala-library 214 | org.scala-lang:scala-compiler 215 | org.scala-lang:scala-reflect 216 | com.amazonaws:aws-java-sdk 217 | com.typesafe.akka:akka-actor_* 218 | com.typesafe.akka:akka-remote_* 219 | com.typesafe.akka:akka-slf4j_* 220 | io.netty:netty-all 221 | io.netty:netty 222 | org.eclipse.jetty:jetty-server 223 | org.eclipse.jetty:jetty-continuation 224 | org.eclipse.jetty:jetty-http 225 | org.eclipse.jetty:jetty-io 226 | org.eclipse.jetty:jetty-util 227 | org.eclipse.jetty:jetty-security 228 | org.eclipse.jetty:jetty-servlet 229 | commons-fileupload:commons-fileupload 230 | org.apache.avro:avro 231 | commons-collections:commons-collections 232 | org.codehaus.jackson:jackson-core-asl 233 | org.codehaus.jackson:jackson-mapper-asl 234 | com.thoughtworks.paranamer:paranamer 235 | org.xerial.snappy:snappy-java 236 | org.apache.commons:commons-compress 237 | org.tukaani:xz 238 | com.esotericsoftware.kryo:kryo 239 | com.esotericsoftware.minlog:minlog 240 | org.objenesis:objenesis 241 | com.twitter:chill_* 242 | com.twitter:chill-java 243 | com.twitter:chill-avro_* 244 | com.twitter:chill-bijection_* 245 | com.twitter:bijection-core_* 246 | com.twitter:bijection-avro_* 247 | commons-lang:commons-lang 248 | junit:junit 249 | de.javakaffee:kryo-serializers 250 | joda-time:joda-time 251 | org.apache.commons:commons-lang3 252 | org.slf4j:slf4j-api 253 | org.slf4j:slf4j-log4j12 254 | log4j:log4j 255 | org.apache.commons:commons-math 256 | org.apache.sling:org.apache.sling.commons.json 257 | commons-logging:commons-logging 258 | org.apache.httpcomponents:httpclient 259 | org.apache.httpcomponents:httpcore 260 | commons-codec:commons-codec 261 | 262 | com.fasterxml.jackson.core:jackson-databind 263 | com.fasterxml.jackson.core:jackson-annotations 264 | org.codehaus.jettison:jettison 265 | stax:stax-api 266 | com.typesafe:config 267 | org.uncommons.maths:uncommons-maths 268 | com.github.scopt:scopt_* 269 | org.mortbay.jetty:servlet-api 270 | commons-io:commons-io 271 | commons-cli:commons-cli 272 | 273 | 274 | 275 | 276 | org.apache.flink:* 277 | 278 | org/apache/flink/shaded/** 279 | web-docs/** 280 | 281 | 282 | 283 | 285 | *:* 286 | 287 | META-INF/*.SF 288 | META-INF/*.DSA 289 | META-INF/*.RSA 290 | 291 | 292 | 293 | 294 | 295 | 296 | com.example.messaging.KafkaConsumer 297 | 298 | 299 | false 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | org.apache.maven.plugins 308 | maven-jar-plugin 309 | 2.5 310 | 311 | 312 | 313 | com.example.messaging.KafkaConsumer 314 | 315 | 316 | 317 | 318 | 319 | 320 | org.apache.maven.plugins 321 | maven-compiler-plugin 322 | 3.1 323 | 324 | 1.8 325 | 1.8 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | maven-compiler-plugin 336 | 337 | 1.8 338 | 1.8 339 | jdt 340 | 341 | 342 | 343 | org.eclipse.tycho 344 | tycho-compiler-jdt 345 | 0.21.0 346 | 347 | 348 | 349 | 350 | 351 | org.eclipse.m2e 352 | lifecycle-mapping 353 | 1.0.0 354 | 355 | 356 | 357 | 358 | 359 | org.apache.maven.plugins 360 | maven-assembly-plugin 361 | [2.4,) 362 | 363 | single 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | org.apache.maven.plugins 373 | maven-compiler-plugin 374 | [3.1,) 375 | 376 | testCompile 377 | compile 378 | 379 | 380 | 381 | 382 | 383 | 384 | 385 | 386 | 387 | 388 | 389 | 390 | 391 | 392 | 393 | 394 | 396 | build-jar 397 | 398 | false 399 | 400 | 401 | 402 | org.apache.flink 403 | flink-java 404 | ${flink.version} 405 | provided 406 | 407 | 408 | org.apache.flink 409 | flink-streaming-java 410 | ${flink.version} 411 | provided 412 | 413 | 414 | org.apache.flink 415 | flink-clients 416 | ${flink.version} 417 | provided 418 | 419 | 420 | 421 | 422 | 423 | -------------------------------------------------------------------------------- /src/main/documentation/data-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viswanath7/flink-kafka-consumer/8523d835a8bbf59be550a4c2c3a34cd56388c752/src/main/documentation/data-model.png -------------------------------------------------------------------------------- /src/main/documentation/target-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viswanath7/flink-kafka-consumer/8523d835a8bbf59be550a4c2c3a34cd56388c752/src/main/documentation/target-architecture.png -------------------------------------------------------------------------------- /src/main/java/com/example/configuration/ApplicationConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.example.configuration; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.beans.factory.annotation.Value; 6 | import org.springframework.cassandra.core.CqlOperations; 7 | import org.springframework.cassandra.core.CqlTemplate; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.ComponentScan; 10 | import org.springframework.context.annotation.Configuration; 11 | import org.springframework.context.annotation.FilterType; 12 | import org.springframework.context.annotation.PropertySource; 13 | import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; 14 | import org.springframework.data.cassandra.config.CassandraClusterFactoryBean; 15 | import org.springframework.data.cassandra.config.CassandraSessionFactoryBean; 16 | import org.springframework.data.cassandra.config.SchemaAction; 17 | import org.springframework.data.cassandra.config.java.AbstractCassandraConfiguration; 18 | import org.springframework.data.cassandra.convert.CassandraConverter; 19 | import org.springframework.data.cassandra.convert.MappingCassandraConverter; 20 | import org.springframework.data.cassandra.mapping.BasicCassandraMappingContext; 21 | import org.springframework.data.cassandra.mapping.CassandraMappingContext; 22 | import org.springframework.data.cassandra.repository.config.EnableCassandraRepositories; 23 | 24 | @Configuration 25 | @ComponentScan(basePackages = "com.example", 26 | excludeFilters = { 27 | @ComponentScan.Filter(type = FilterType.ANNOTATION, value = Configuration.class), 28 | @ComponentScan.Filter(type = FilterType.REGEX, pattern = { "com.example.configuration.TestConfiguration" }) 29 | })@PropertySource(value = {"classpath:cassandra.properties"}) 30 | @EnableCassandraRepositories("com.example.repository") 31 | public class ApplicationConfiguration extends AbstractCassandraConfiguration { 32 | 33 | private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationConfiguration.class); 34 | 35 | @Value("${cassandra.keyspace}") 36 | private String keySpace; 37 | 38 | @Value("${cassandra.contactpoints}") 39 | private String contactPoints; 40 | 41 | @Value("${cassandra.port}") 42 | private String port; 43 | 44 | @Bean 45 | public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer() { 46 | return new PropertySourcesPlaceholderConfigurer(); 47 | } 48 | 49 | @Override 50 | protected String getKeyspaceName() { 51 | LOGGER.info("Using key-space: {}", keySpace); 52 | return keySpace; 53 | } 54 | 55 | @Override 56 | @Bean 57 | public CassandraClusterFactoryBean cluster() { 58 | final Integer portNumber = Integer.parseInt(port); 59 | final CassandraClusterFactoryBean cluster = new CassandraClusterFactoryBean(); 60 | cluster.setContactPoints(contactPoints); 61 | cluster.setPort(portNumber); 62 | LOGGER.info("Cluster created with contact points [" + contactPoints + "] " + "& portNumber [" + portNumber + "]."); 63 | return cluster; 64 | } 65 | 66 | @Override 67 | @Bean 68 | public CassandraMappingContext cassandraMapping() throws ClassNotFoundException { 69 | return new BasicCassandraMappingContext(); 70 | } 71 | 72 | @Bean 73 | public CassandraMappingContext mappingContext() { 74 | return new BasicCassandraMappingContext(); 75 | } 76 | 77 | @Bean 78 | public CassandraConverter converter() { 79 | return new MappingCassandraConverter(mappingContext()); 80 | } 81 | 82 | @Bean 83 | public CqlOperations CqlTemplate() throws Exception { 84 | return new CqlTemplate(session().getObject()); 85 | } 86 | 87 | @Bean 88 | public CassandraSessionFactoryBean session() throws Exception { 89 | LOGGER.info("Creating cassandra session factory ..."); 90 | CassandraSessionFactoryBean session = new CassandraSessionFactoryBean(); 91 | session.setCluster(cluster().getObject()); 92 | session.setKeyspaceName(getKeyspaceName()); 93 | session.setConverter(converter()); 94 | session.setSchemaAction(SchemaAction.NONE); 95 | LOGGER.info("Cassandra session factory has been successfully created!"); 96 | return session; 97 | } 98 | 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/com/example/converter/MessagingToDomainObjectConverter.java: -------------------------------------------------------------------------------- 1 | package com.example.converter; 2 | 3 | import org.springframework.stereotype.Service; 4 | 5 | import com.example.domain.UserActivityEvent; 6 | import com.example.model.EventByCorrelationId; 7 | import com.example.model.EventByReference; 8 | import com.example.model.EventByType; 9 | import com.example.model.primarykey.CorrelationIdDateTime; 10 | import com.example.model.primarykey.EventTypeDateTime; 11 | import com.example.model.primarykey.ReferenceDateTime; 12 | 13 | @Service 14 | public class MessagingToDomainObjectConverter { 15 | 16 | public EventByCorrelationId toEventByCorrelationId(final UserActivityEvent message) { 17 | EventByCorrelationId eventByCorrelationId = new EventByCorrelationId(); 18 | eventByCorrelationId.setPrimaryKey(new CorrelationIdDateTime(message.getCorrelationId(), message.getDate())); 19 | eventByCorrelationId.setReference(message.getReference()); 20 | eventByCorrelationId.setClientIp(message.getClientIp()); 21 | eventByCorrelationId.setDetails(message.getDetails()); 22 | eventByCorrelationId.setEventType(message.getEventType()); 23 | eventByCorrelationId.setUserAgent(message.getUserAgent()); 24 | eventByCorrelationId.setUserAgentFiltered(message.getUserAgentFiltered()); 25 | return eventByCorrelationId; 26 | } 27 | 28 | public EventByReference toEventByReference(final UserActivityEvent message) { 29 | EventByReference eventByReference = new EventByReference(); 30 | eventByReference.setPrimaryKey(new ReferenceDateTime(message.getReference(), message.getDate())); 31 | eventByReference.setCorrelationId(message.getCorrelationId()); 32 | eventByReference.setClientIp(message.getClientIp()); 33 | eventByReference.setDetails(message.getDetails()); 34 | eventByReference.setEventType(message.getEventType()); 35 | eventByReference.setUserAgent(message.getUserAgent()); 36 | eventByReference.setUserAgentFiltered(message.getUserAgentFiltered()); 37 | return eventByReference; 38 | } 39 | 40 | public EventByType toEventByType(final UserActivityEvent message) { 41 | EventByType eventByType = new EventByType(); 42 | eventByType.setPrimaryKey(new EventTypeDateTime(message.getEventType(), message.getDate())); 43 | eventByType.setCorrelationId(message.getCorrelationId()); 44 | eventByType.setClientIp(message.getClientIp()); 45 | eventByType.setDetails(message.getDetails()); 46 | eventByType.setReference(message.getReference()); 47 | eventByType.setUserAgent(message.getUserAgent()); 48 | eventByType.setUserAgentFiltered(message.getUserAgentFiltered()); 49 | return eventByType; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/example/domain/UserActivityEvent.java: -------------------------------------------------------------------------------- 1 | package com.example.domain; 2 | 3 | 4 | import java.io.Serializable; 5 | import java.time.LocalDateTime; 6 | import java.util.Objects; 7 | 8 | import org.apache.commons.lang3.StringUtils; 9 | import org.apache.commons.lang3.builder.ToStringBuilder; 10 | 11 | import com.fasterxml.jackson.annotation.JsonIgnore; 12 | 13 | 14 | public class UserActivityEvent implements Serializable { 15 | 16 | private Long id; 17 | 18 | private String eventType; 19 | 20 | private LocalDateTime date; 21 | 22 | private String correlationId; 23 | 24 | private String clientIp; 25 | 26 | private String userAgent; 27 | 28 | private String userAgentFiltered; 29 | 30 | private String details; 31 | 32 | private String reference; 33 | 34 | public Long getId() { 35 | return id; 36 | } 37 | 38 | public void setId(final Long id) { 39 | this.id = id; 40 | } 41 | 42 | public LocalDateTime getDate() { 43 | return date; 44 | } 45 | 46 | public void setDate(final LocalDateTime localDateTime) { 47 | this.date = localDateTime; 48 | } 49 | 50 | public String getCorrelationId() { 51 | return correlationId; 52 | } 53 | 54 | public void setCorrelationId(final String correlationId) { 55 | this.correlationId = correlationId; 56 | } 57 | 58 | public String getClientIp() { 59 | return clientIp; 60 | } 61 | 62 | public void setClientIp(final String clientIp) { 63 | this.clientIp = clientIp; 64 | } 65 | 66 | public String getUserAgent() { 67 | return userAgent; 68 | } 69 | 70 | public void setUserAgent(final String userAgent) { 71 | this.userAgent = userAgent; 72 | } 73 | 74 | public String getUserAgentFiltered() { 75 | return userAgentFiltered; 76 | } 77 | 78 | public void setUserAgentFiltered(final String userAgentFiltered) { 79 | this.userAgentFiltered = userAgentFiltered; 80 | } 81 | 82 | public String getDetails() { 83 | return details; 84 | } 85 | 86 | public void setDetails(final String details) { 87 | this.details = details; 88 | } 89 | 90 | public String getReference() { 91 | return reference; 92 | } 93 | 94 | public void setReference(final String reference) { 95 | this.reference = reference; 96 | } 97 | 98 | public String getEventType() { 99 | return eventType; 100 | } 101 | 102 | public void setEventType(final String eventType) { 103 | this.eventType = eventType; 104 | } 105 | 106 | @JsonIgnore 107 | public boolean isValid() { 108 | return eventType != null && !StringUtils.isBlank(correlationId) && date != null; 109 | } 110 | 111 | 112 | public UserActivityEvent(final Long id, final String eventType, final LocalDateTime date, 113 | final String correlationId, final String clientIp, final String userAgent, 114 | final String userAgentFiltered, final String details, final String reference) { 115 | this(); 116 | setId(id); 117 | setEventType(eventType); 118 | setDate(date); 119 | setCorrelationId(correlationId); 120 | setClientIp(clientIp); 121 | setUserAgent(userAgent); 122 | setUserAgentFiltered(userAgentFiltered); 123 | setDetails(details); 124 | setReference(reference); 125 | } 126 | 127 | public UserActivityEvent() {} 128 | 129 | @Override 130 | public boolean equals(final Object o) { 131 | if (this == o) { 132 | return true; 133 | } 134 | if (!(o instanceof UserActivityEvent)) { 135 | return false; 136 | } 137 | final UserActivityEvent event = (UserActivityEvent) o; 138 | return Objects.equals(getId(), event.getId()) && 139 | getEventType() == event.getEventType() && 140 | Objects.equals(getDate(), event.getDate()) && 141 | Objects.equals(getCorrelationId(), event.getCorrelationId()) && 142 | Objects.equals(getClientIp(), event.getClientIp()) && 143 | Objects.equals(getUserAgent(), event.getUserAgent()) && 144 | Objects.equals(getUserAgentFiltered(), event.getUserAgentFiltered()) && 145 | Objects.equals(getDetails(), event.getDetails()) && 146 | Objects.equals(getReference(), event.getReference()); 147 | } 148 | 149 | @Override 150 | public int hashCode() { 151 | return Objects 152 | .hash(getId(), getEventType(), getDate(), getCorrelationId(), getClientIp(), 153 | getUserAgent(), getUserAgentFiltered(), getDetails(), getReference()); 154 | } 155 | 156 | @Override 157 | public String toString() { 158 | return new ToStringBuilder(this) 159 | .append("id", id) 160 | .append("eventType", eventType) 161 | .append("date", date) 162 | .append("correlationId", correlationId) 163 | .append("clientIp", clientIp) 164 | .append("userAgent", userAgent) 165 | .append("userAgentFiltered", userAgentFiltered) 166 | .append("details", details) 167 | .append("reference", reference) 168 | .toString(); 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /src/main/java/com/example/mapper/JSONDeserialiser.java: -------------------------------------------------------------------------------- 1 | package com.example.mapper; 2 | 3 | import java.io.IOException; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.stereotype.Service; 8 | 9 | import com.fasterxml.jackson.databind.ObjectMapper; 10 | import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; 11 | 12 | @Service 13 | public final class JSONDeserialiser { 14 | 15 | private static final Logger LOG = LoggerFactory.getLogger(JSONDeserialiser.class); 16 | 17 | private ObjectMapper mapper; 18 | 19 | public JSONDeserialiser() { 20 | LOG.info("Registering modules so that JSR-310 types are (de)serialized as numbers in Java 8"); 21 | mapper = new ObjectMapper(); 22 | mapper.registerModule(new JavaTimeModule()); 23 | } 24 | 25 | public ObjectMapper getMapper() { 26 | return mapper; 27 | } 28 | 29 | public void setMapper(final ObjectMapper mapper) { 30 | this.mapper = mapper; 31 | mapper.registerModule(new JavaTimeModule()); 32 | } 33 | 34 | /** 35 | * Given a JSON string and its class, returns an object representing the string. 36 | * 37 | * @param json 38 | * @param clazz 39 | * @param 40 | * @return 41 | * @throws IOException 42 | */ 43 | public T convert(final String json, Class clazz) throws IOException { 44 | LOG.debug("Converting json string: \n {} \n to an object", json); 45 | return mapper.readValue(json, clazz); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/example/messaging/CassandraDataSink.java: -------------------------------------------------------------------------------- 1 | package com.example.messaging; 2 | 3 | import org.apache.flink.streaming.api.functions.sink.SinkFunction; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.context.ApplicationContext; 7 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 8 | 9 | import com.example.configuration.ApplicationConfiguration; 10 | import com.example.service.UserActivityEventsPersister; 11 | 12 | /** 13 | * Cassandra database sink to persist json message as rows of tables in cassandra 14 | */ 15 | public class CassandraDataSink implements SinkFunction { 16 | 17 | private static final Logger LOG = LoggerFactory.getLogger(CassandraDataSink.class); 18 | 19 | private static final ApplicationContext applicationContext = new AnnotationConfigApplicationContext(ApplicationConfiguration.class); 20 | 21 | /** 22 | * Function for standard sink behaviour. This function is called for every record. 23 | * 24 | * @param jsonEntity The input json entity that shall be stored in various cassandra table 25 | */ 26 | @Override 27 | public void invoke(final String jsonEntity) throws Exception { 28 | LOG.debug("Persisting JSON entity: \n{}\n via cassandra database sink ...", jsonEntity); 29 | UserActivityEventsPersister userActivityEventsPersister = applicationContext.getBean(UserActivityEventsPersister.class); 30 | userActivityEventsPersister.persist(jsonEntity); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/example/messaging/KafkaConsumer.java: -------------------------------------------------------------------------------- 1 | package com.example.messaging; 2 | 3 | import java.util.Properties; 4 | 5 | import org.apache.flink.api.java.utils.ParameterTool; 6 | import org.apache.flink.streaming.api.datastream.DataStream; 7 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 8 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer082; 9 | import org.apache.flink.streaming.util.serialization.SimpleStringSchema; 10 | 11 | /** 12 | * Kafka consumer: It expects the following parameters to be set 13 | * - "bootstrap.servers" (comma separated list of kafka brokers) 14 | * - "zookeeper.connect" (comma separated list of zookeeper servers) 15 | * - "group.id" the id of the consumer group 16 | * - "topic" the name of the topic to read data from. 17 | * 18 | * One can pass these required parameters using 19 | * "--bootstrap.servers host:port,host1:port1 --zookeeper.connect host:port --topic testTopic" 20 | * 21 | * This is a valid input example: 22 | * --topic test --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myGroup 23 | */ 24 | public class KafkaConsumer { 25 | 26 | private static final StreamExecutionEnvironment executionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment(); 27 | 28 | 29 | //private static JSONDeserialiser jsonMapper = new JSONDeserialiser(); 30 | 31 | public static void main(String[] args) throws Exception { 32 | 33 | // parse user parameters 34 | ParameterTool parameterTool = ParameterTool.fromArgs(args); 35 | 36 | DataStream constructedMessageStream = executionEnvironment.addSource(getDataSource(parameterTool)); 37 | 38 | constructedMessageStream.rebalance().addSink(new CassandraDataSink()); 39 | 40 | //.timeWindowAll(Time.of(1, TimeUnit.SECONDS)) 41 | 42 | 43 | 44 | 45 | // repartition data so that all machines see the messages 46 | // for example when number of kafka partitions < number of flink operators 47 | //.map(value -> jsonMapper.convert(value, UserActivityEvent.class)) 48 | // map function that transforms input elements of type string and returns an output element of type string 49 | //.print(); 50 | // write the contents of the stream to the TaskManager's standard output stream 51 | 52 | 53 | executionEnvironment.execute(); 54 | // Trigger the execution of the program 55 | } 56 | 57 | /** 58 | * Returns a kafka consumer to read from Kafka 0.8.2.x brokers 59 | * 60 | * @param parameterTool 61 | * @return 62 | */ 63 | private static FlinkKafkaConsumer082 getDataSource(final ParameterTool parameterTool) { 64 | return new FlinkKafkaConsumer082<>(getTopicName(parameterTool), getKafkaMessageDeserialiser(), 65 | getKafkaConsumerProperties(parameterTool)); 66 | } 67 | 68 | private static Properties getKafkaConsumerProperties(final ParameterTool parameterTool) { 69 | return parameterTool.getProperties(); 70 | } 71 | 72 | /** 73 | * Gets deserialiser that's used to convert between Kafka's byte messages and Flink's objects 74 | * 75 | * @return 76 | */ 77 | private static SimpleStringSchema getKafkaMessageDeserialiser() { 78 | return new SimpleStringSchema(); 79 | } 80 | 81 | /** 82 | * Returns the name of a topic from which the subscriber should consume messages 83 | * 84 | * @param parameterTool 85 | * @return 86 | */ 87 | private static String getTopicName(final ParameterTool parameterTool) { 88 | return parameterTool.getRequired("topic"); 89 | } 90 | 91 | 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/example/model/Event.java: -------------------------------------------------------------------------------- 1 | package com.example.model; 2 | 3 | 4 | import java.io.Serializable; 5 | import java.time.ZoneId; 6 | 7 | import org.springframework.data.cassandra.mapping.Column; 8 | 9 | public abstract class Event implements Serializable { 10 | 11 | public static final ZoneId UTC = ZoneId.of("UTC"); 12 | 13 | @Column(value = "client_ip") 14 | protected String clientIp; 15 | 16 | @Column(value = "user_agent") 17 | protected String userAgent; 18 | 19 | @Column(value = "user_agent_filtered") 20 | protected String userAgentFiltered; 21 | 22 | @Column(value = "details") 23 | protected String details; 24 | 25 | public String getClientIp() { 26 | return clientIp; 27 | } 28 | 29 | public void setClientIp(final String clientIp) { 30 | this.clientIp = clientIp; 31 | } 32 | 33 | public String getUserAgent() { 34 | return userAgent; 35 | } 36 | 37 | public void setUserAgent(final String userAgent) { 38 | this.userAgent = userAgent; 39 | } 40 | 41 | public String getUserAgentFiltered() { 42 | return userAgentFiltered; 43 | } 44 | 45 | public void setUserAgentFiltered(final String userAgentFiltered) { 46 | this.userAgentFiltered = userAgentFiltered; 47 | } 48 | 49 | public String getDetails() { 50 | return details; 51 | } 52 | 53 | public void setDetails(final String details) { 54 | this.details = details; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/example/model/EventByCorrelationId.java: -------------------------------------------------------------------------------- 1 | package com.example.model; 2 | 3 | 4 | import java.time.LocalDateTime; 5 | import java.time.ZoneId; 6 | import java.util.Date; 7 | 8 | import org.apache.commons.lang3.builder.ToStringBuilder; 9 | import org.springframework.data.cassandra.mapping.Column; 10 | import org.springframework.data.cassandra.mapping.PrimaryKey; 11 | import org.springframework.data.cassandra.mapping.Table; 12 | 13 | import com.example.model.primarykey.CorrelationIdDateTime; 14 | 15 | @Table(value = "events_by_correlation_id") 16 | public class EventByCorrelationId extends Event { 17 | 18 | @Column(value = "event_type") 19 | protected String eventType; 20 | @PrimaryKey 21 | private CorrelationIdDateTime primaryKey; 22 | @Column(value = "reference") 23 | private String reference; 24 | 25 | public EventByCorrelationId(final String eventType, final LocalDateTime date, final String correlationId, 26 | final String clientIp, final String userAgent, 27 | final String userAgentFiltered, final String details, final String reference) { 28 | this(); 29 | setEventType(eventType); 30 | getPrimaryKey().setEventDateTime(Date.from(date.atZone(ZoneId.systemDefault()).toInstant())); 31 | getPrimaryKey().setCorrelationId(correlationId); 32 | setClientIp(clientIp); 33 | setUserAgent(userAgent); 34 | setUserAgentFiltered(userAgentFiltered); 35 | setDetails(details); 36 | setReference(reference); 37 | } 38 | 39 | public EventByCorrelationId() { 40 | } 41 | 42 | public CorrelationIdDateTime getPrimaryKey() { 43 | return primaryKey; 44 | } 45 | 46 | public void setPrimaryKey(final CorrelationIdDateTime primaryKey) { 47 | this.primaryKey = primaryKey; 48 | } 49 | 50 | public String getReference() { 51 | return reference; 52 | } 53 | 54 | public void setReference(final String reference) { 55 | this.reference = reference; 56 | } 57 | 58 | public String getEventType() { 59 | return eventType; 60 | } 61 | 62 | public void setEventType(final String eventType) { 63 | this.eventType = eventType; 64 | } 65 | 66 | @Override 67 | public String toString() { 68 | return new ToStringBuilder(this) 69 | .append("eventType", getEventType()) 70 | .append("eventDateTime", getPrimaryKey().getEventDateTime()) 71 | .append("correlationId", getPrimaryKey().getCorrelationId()) 72 | .append("clientIp", getClientIp()) 73 | .append("userAgent", getUserAgent()) 74 | .append("userAgentFiltered", getUserAgentFiltered()) 75 | .append("details", getDetails()) 76 | .append("reference", getReference()) 77 | .toString(); 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/com/example/model/EventByReference.java: -------------------------------------------------------------------------------- 1 | package com.example.model; 2 | 3 | 4 | import org.apache.commons.lang3.builder.ToStringBuilder; 5 | import org.springframework.data.cassandra.mapping.Column; 6 | import org.springframework.data.cassandra.mapping.PrimaryKey; 7 | import org.springframework.data.cassandra.mapping.Table; 8 | 9 | import com.example.model.primarykey.ReferenceDateTime; 10 | 11 | @Table(value = "events_by_reference_and_year") 12 | public class EventByReference extends Event { 13 | 14 | @Column(value = "event_type") 15 | protected String eventType; 16 | @PrimaryKey 17 | private ReferenceDateTime primaryKey; 18 | @Column(value = "correlation_id") 19 | private String correlationId; 20 | 21 | public ReferenceDateTime getPrimaryKey() { 22 | return primaryKey; 23 | } 24 | 25 | public void setPrimaryKey(final ReferenceDateTime primaryKey) { 26 | this.primaryKey = primaryKey; 27 | } 28 | 29 | public String getCorrelationId() { 30 | return correlationId; 31 | } 32 | 33 | public void setCorrelationId(final String correlationId) { 34 | this.correlationId = correlationId; 35 | } 36 | 37 | public String getEventType() { 38 | return eventType; 39 | } 40 | 41 | public void setEventType(final String eventType) { 42 | this.eventType = eventType; 43 | } 44 | 45 | @Override 46 | public String toString() { 47 | return new ToStringBuilder(this) 48 | .append("eventType", getEventType()) 49 | .append("eventDateTime", getPrimaryKey().getEventDateTime()) 50 | .append("correlationId", getCorrelationId()) 51 | .append("clientIp", getClientIp()) 52 | .append("userAgent", getUserAgent()) 53 | .append("userAgentFiltered", getUserAgentFiltered()) 54 | .append("details", getDetails()) 55 | .append("reference", getPrimaryKey().getReference()) 56 | .toString(); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/example/model/EventByType.java: -------------------------------------------------------------------------------- 1 | package com.example.model; 2 | 3 | import org.apache.commons.lang3.builder.ToStringBuilder; 4 | import org.springframework.data.cassandra.mapping.Column; 5 | import org.springframework.data.cassandra.mapping.PrimaryKey; 6 | import org.springframework.data.cassandra.mapping.Table; 7 | 8 | import com.example.model.primarykey.EventTypeDateTime; 9 | 10 | @Table(value = "events_by_type") 11 | public class EventByType extends Event { 12 | 13 | @PrimaryKey 14 | private EventTypeDateTime primaryKey; 15 | 16 | @Column(value = "reference") 17 | private String reference; 18 | 19 | @Column(value = "correlation_id") 20 | private String correlationId; 21 | 22 | 23 | public EventTypeDateTime getPrimaryKey() { 24 | return primaryKey; 25 | } 26 | 27 | public void setPrimaryKey(final EventTypeDateTime primaryKey) { 28 | this.primaryKey = primaryKey; 29 | } 30 | 31 | public String getReference() { 32 | return reference; 33 | } 34 | 35 | public void setReference(final String reference) { 36 | this.reference = reference; 37 | } 38 | 39 | public String getCorrelationId() { 40 | return correlationId; 41 | } 42 | 43 | public void setCorrelationId(final String correlationId) { 44 | this.correlationId = correlationId; 45 | } 46 | 47 | @Override 48 | public String toString() { 49 | return new ToStringBuilder(this) 50 | .append("eventType", getPrimaryKey().getEventType()) 51 | .append("eventDateTime", getPrimaryKey().getEventDateTime()) 52 | .append("correlationId", getCorrelationId()) 53 | .append("clientIp", getClientIp()) 54 | .append("userAgent", getUserAgent()) 55 | .append("userAgentFiltered", getUserAgentFiltered()) 56 | .append("details", getDetails()) 57 | .append("reference", getReference()) 58 | .toString(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/example/model/builder/EventBuilder.java: -------------------------------------------------------------------------------- 1 | package com.example.model.builder; 2 | 3 | import java.time.LocalDateTime; 4 | 5 | import com.example.model.EventByCorrelationId; 6 | 7 | public class EventBuilder { 8 | 9 | private String eventType; 10 | private LocalDateTime date; 11 | private String correlationId; 12 | private String clientIp; 13 | private String userAgent; 14 | private String userAgentFiltered; 15 | private String details; 16 | private String reference; 17 | 18 | public EventBuilder withEventType(final String eventType) { 19 | this.eventType = eventType; 20 | return this; 21 | } 22 | 23 | public EventBuilder withDate(final LocalDateTime date) { 24 | this.date = date; 25 | return this; 26 | } 27 | 28 | public EventBuilder withCorrelationId(final String correlationId) { 29 | this.correlationId = correlationId; 30 | return this; 31 | } 32 | 33 | public EventBuilder withClientIp(final String clientIp) { 34 | this.clientIp = clientIp; 35 | return this; 36 | } 37 | 38 | public EventBuilder withUserAgent(final String userAgent) { 39 | this.userAgent = userAgent; 40 | return this; 41 | } 42 | 43 | public EventBuilder withUserAgentFiltered(final String userAgentFiltered) { 44 | this.userAgentFiltered = userAgentFiltered; 45 | return this; 46 | } 47 | 48 | public EventBuilder withDetails(final String details) { 49 | this.details = details; 50 | return this; 51 | } 52 | 53 | public EventBuilder withReference(final String reference) { 54 | this.reference = reference; 55 | return this; 56 | } 57 | 58 | public EventByCorrelationId createEvent() { 59 | return new EventByCorrelationId(eventType, date, correlationId, clientIp, userAgent, 60 | userAgentFiltered, details, reference); 61 | } 62 | } -------------------------------------------------------------------------------- /src/main/java/com/example/model/primarykey/CorrelationIdDateTime.java: -------------------------------------------------------------------------------- 1 | package com.example.model.primarykey; 2 | 3 | import java.io.Serializable; 4 | import java.time.LocalDateTime; 5 | import java.util.Date; 6 | import java.util.Objects; 7 | 8 | import org.springframework.cassandra.core.Ordering; 9 | import org.springframework.cassandra.core.PrimaryKeyType; 10 | import org.springframework.data.cassandra.mapping.PrimaryKeyClass; 11 | import org.springframework.data.cassandra.mapping.PrimaryKeyColumn; 12 | 13 | import com.example.model.Event; 14 | 15 | @PrimaryKeyClass 16 | public class CorrelationIdDateTime implements Serializable { 17 | 18 | @PrimaryKeyColumn(name = "correlation_id", ordinal = 0, type = PrimaryKeyType.PARTITIONED) 19 | private String correlationId; 20 | 21 | @PrimaryKeyColumn(name = "event_date_time", ordinal = 1, type = PrimaryKeyType.CLUSTERED, ordering = Ordering.DESCENDING) 22 | private Date eventDateTime; 23 | 24 | public CorrelationIdDateTime() {} 25 | 26 | public CorrelationIdDateTime(final String correlationId, final Date eventDateTime) { 27 | this(); 28 | this.correlationId = correlationId; 29 | this.eventDateTime = eventDateTime; 30 | } 31 | 32 | public CorrelationIdDateTime(final String correlationId, final LocalDateTime eventDateTime) { 33 | this(); 34 | this.correlationId = correlationId; 35 | this.eventDateTime = Date.from(eventDateTime.atZone(Event.UTC).toInstant()); 36 | } 37 | 38 | public String getCorrelationId() { 39 | return correlationId; 40 | } 41 | 42 | public void setCorrelationId(final String correlationId) { 43 | this.correlationId = correlationId; 44 | } 45 | 46 | public Date getEventDateTime() { 47 | return eventDateTime; 48 | } 49 | 50 | public void setEventDateTime(final Date eventDateTime) { 51 | this.eventDateTime = eventDateTime; 52 | } 53 | 54 | @Override 55 | public boolean equals(final Object o) { 56 | if (this == o) { 57 | return true; 58 | } 59 | if (!(o instanceof CorrelationIdDateTime)) { 60 | return false; 61 | } 62 | final CorrelationIdDateTime that = (CorrelationIdDateTime) o; 63 | return Objects.equals(getCorrelationId(), that.getCorrelationId()) && 64 | Objects.equals(getEventDateTime(), that.getEventDateTime()); 65 | } 66 | 67 | @Override 68 | public int hashCode() { 69 | return Objects.hash(getCorrelationId(), getEventDateTime()); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/com/example/model/primarykey/EventTypeDateTime.java: -------------------------------------------------------------------------------- 1 | package com.example.model.primarykey; 2 | 3 | import java.io.Serializable; 4 | import java.time.LocalDateTime; 5 | import java.util.Date; 6 | import java.util.Objects; 7 | 8 | import org.apache.commons.lang3.builder.ToStringBuilder; 9 | import org.springframework.cassandra.core.Ordering; 10 | import org.springframework.cassandra.core.PrimaryKeyType; 11 | import org.springframework.data.cassandra.mapping.PrimaryKeyClass; 12 | import org.springframework.data.cassandra.mapping.PrimaryKeyColumn; 13 | 14 | import com.example.model.Event; 15 | 16 | @PrimaryKeyClass 17 | public class EventTypeDateTime implements Serializable { 18 | 19 | @PrimaryKeyColumn(name = "event_type", ordinal = 0, type = PrimaryKeyType.PARTITIONED) 20 | private String eventType; 21 | 22 | @PrimaryKeyColumn(name = "event_date_time", ordinal = 1, type = PrimaryKeyType.CLUSTERED, ordering = Ordering.DESCENDING) 23 | private Date eventDateTime; 24 | 25 | public EventTypeDateTime() {} 26 | 27 | public EventTypeDateTime(final String eventType, final Date eventDateTime) { 28 | this(); 29 | setEventType(eventType); 30 | setEventDateTime(eventDateTime); 31 | } 32 | 33 | public EventTypeDateTime(final String eventType, final LocalDateTime eventDateTime) { 34 | this(); 35 | setEventType(eventType); 36 | setEventDateTime(Date.from(eventDateTime.atZone(Event.UTC).toInstant())); 37 | } 38 | 39 | public String getEventType() { 40 | return eventType; 41 | } 42 | 43 | public void setEventType(final String eventType) { 44 | this.eventType = eventType; 45 | } 46 | 47 | public Date getEventDateTime() { 48 | return eventDateTime; 49 | } 50 | 51 | public void setEventDateTime(final Date eventDateTime) { 52 | this.eventDateTime = eventDateTime; 53 | } 54 | 55 | @Override 56 | public boolean equals(final Object o) { 57 | if (this == o) { 58 | return true; 59 | } 60 | if (!(o instanceof EventTypeDateTime)) { 61 | return false; 62 | } 63 | final EventTypeDateTime that = (EventTypeDateTime) o; 64 | return Objects.equals(getEventType(), that.getEventType()) && 65 | Objects.equals(getEventDateTime(), that.getEventDateTime()); 66 | } 67 | 68 | @Override 69 | public int hashCode() { 70 | return Objects.hash(getEventType(), getEventDateTime()); 71 | } 72 | 73 | @Override 74 | public String toString() { 75 | return new ToStringBuilder(this) 76 | .append("eventType", eventType) 77 | .append("eventDateTime", eventDateTime) 78 | .toString(); 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/com/example/model/primarykey/ReferenceDateTime.java: -------------------------------------------------------------------------------- 1 | package com.example.model.primarykey; 2 | 3 | import java.io.Serializable; 4 | import java.time.LocalDateTime; 5 | import java.util.Date; 6 | import java.util.Objects; 7 | 8 | import org.apache.commons.lang3.builder.ToStringBuilder; 9 | import org.springframework.cassandra.core.Ordering; 10 | import org.springframework.cassandra.core.PrimaryKeyType; 11 | import org.springframework.data.cassandra.mapping.PrimaryKeyClass; 12 | import org.springframework.data.cassandra.mapping.PrimaryKeyColumn; 13 | 14 | import com.example.model.Event; 15 | 16 | @PrimaryKeyClass 17 | public class ReferenceDateTime implements Serializable { 18 | 19 | @PrimaryKeyColumn(name = "reference", ordinal = 0, type = PrimaryKeyType.PARTITIONED) 20 | private String reference; 21 | 22 | @PrimaryKeyColumn(name = "year", ordinal = 1, type = PrimaryKeyType.PARTITIONED) 23 | private Integer year; 24 | 25 | @PrimaryKeyColumn(name = "event_date_time", ordinal = 2, type = PrimaryKeyType.CLUSTERED, ordering = Ordering.DESCENDING) 26 | private Date eventDateTime; 27 | 28 | public ReferenceDateTime() {} 29 | 30 | public ReferenceDateTime(final String reference, final Integer year, final Date eventDateTime) { 31 | this(); 32 | this.reference = reference; 33 | this.year = year; 34 | this.eventDateTime = eventDateTime; 35 | } 36 | 37 | public ReferenceDateTime(final String reference, final LocalDateTime eventDateTime) { 38 | this(); 39 | this.reference = reference; 40 | this.year = eventDateTime.getYear(); 41 | this.eventDateTime = Date.from(eventDateTime.atZone(Event.UTC).toInstant()); 42 | } 43 | 44 | public String getReference() { 45 | return reference; 46 | } 47 | 48 | public void setReference(final String reference) { 49 | this.reference = reference; 50 | } 51 | 52 | public Integer getYear() { 53 | return year; 54 | } 55 | 56 | public void setYear(final Integer year) { 57 | this.year = year; 58 | } 59 | 60 | public Date getEventDateTime() { 61 | return eventDateTime; 62 | } 63 | 64 | public void setEventDateTime(final Date eventDateTime) { 65 | this.eventDateTime = eventDateTime; 66 | } 67 | 68 | @Override 69 | public boolean equals(final Object o) { 70 | if (this == o) { 71 | return true; 72 | } 73 | if (!(o instanceof ReferenceDateTime)) { 74 | return false; 75 | } 76 | final ReferenceDateTime that = (ReferenceDateTime) o; 77 | return Objects.equals(getReference(), that.getReference()) && 78 | Objects.equals(getYear(), that.getYear()) && 79 | Objects.equals(getEventDateTime(), that.getEventDateTime()); 80 | } 81 | 82 | @Override 83 | public int hashCode() { 84 | return Objects.hash(getReference(), getYear(), getEventDateTime()); 85 | } 86 | 87 | @Override 88 | public String toString() { 89 | return new ToStringBuilder(this) 90 | .append("reference", reference) 91 | .append("year", year) 92 | .append("eventDateTime", eventDateTime) 93 | .toString(); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/com/example/repository/EventsByCorrelationIdRepository.java: -------------------------------------------------------------------------------- 1 | package com.example.repository; 2 | 3 | 4 | import org.springframework.data.cassandra.repository.CassandraRepository; 5 | import org.springframework.data.cassandra.repository.Query; 6 | import org.springframework.stereotype.Repository; 7 | 8 | import com.example.model.EventByCorrelationId; 9 | 10 | @Repository 11 | public interface EventsByCorrelationIdRepository extends CassandraRepository { 12 | 13 | String FIND_EVENTS_FOR_CORRELATION_IDENTIFIER = "SELECT * FROM events_by_correlationId WHERE correlationId = ?0"; 14 | 15 | @Query(FIND_EVENTS_FOR_CORRELATION_IDENTIFIER) 16 | Iterable findByCorrelationId(final String correlationId); 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/example/repository/EventsByReferenceRepository.java: -------------------------------------------------------------------------------- 1 | package com.example.repository; 2 | 3 | 4 | import org.springframework.data.cassandra.repository.CassandraRepository; 5 | import org.springframework.data.cassandra.repository.Query; 6 | import org.springframework.stereotype.Repository; 7 | 8 | import com.example.model.EventByReference; 9 | 10 | @Repository 11 | public interface EventsByReferenceRepository extends CassandraRepository { 12 | 13 | String FIND_EVENTS_FOR_REFERENCE_AND_YEAR = "SELECT * FROM events_by_reference_and_year WHERE reference = ?0 AND year = ?1"; 14 | 15 | @Query(FIND_EVENTS_FOR_REFERENCE_AND_YEAR) 16 | Iterable findByReferenceAndYear(final String reference, final Integer year); 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/example/repository/EventsByTypeRepository.java: -------------------------------------------------------------------------------- 1 | package com.example.repository; 2 | 3 | import org.springframework.data.cassandra.repository.CassandraRepository; 4 | import org.springframework.data.cassandra.repository.Query; 5 | import org.springframework.stereotype.Repository; 6 | 7 | import com.example.model.EventByType; 8 | 9 | @Repository 10 | public interface EventsByTypeRepository extends CassandraRepository { 11 | 12 | String FIND_EVENTS_BY_TYPE = "SELECT * FROM events_by_type WHERE event_type = ?0"; 13 | 14 | @Query(FIND_EVENTS_BY_TYPE) 15 | Iterable findByType(final String eventType); 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/example/service/UserActivityEventsPersister.java: -------------------------------------------------------------------------------- 1 | package com.example.service; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.commons.lang3.StringUtils; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.stereotype.Service; 10 | 11 | import com.example.converter.MessagingToDomainObjectConverter; 12 | import com.example.domain.UserActivityEvent; 13 | import com.example.mapper.JSONDeserialiser; 14 | import com.example.model.EventByCorrelationId; 15 | import com.example.model.EventByReference; 16 | import com.example.model.EventByType; 17 | import com.example.repository.EventsByCorrelationIdRepository; 18 | import com.example.repository.EventsByReferenceRepository; 19 | import com.example.repository.EventsByTypeRepository; 20 | 21 | @Service 22 | public class UserActivityEventsPersister { 23 | 24 | private static final Logger LOG = LoggerFactory.getLogger(UserActivityEventsPersister.class); 25 | 26 | @Autowired 27 | private JSONDeserialiser jsonDeserialiser; 28 | 29 | @Autowired 30 | private MessagingToDomainObjectConverter messagingToDomainObjectConverter; 31 | 32 | @Autowired 33 | private EventsByCorrelationIdRepository eventsByCorrelationIdRepository; 34 | 35 | @Autowired 36 | private EventsByReferenceRepository eventsByReferenceRepository; 37 | 38 | @Autowired 39 | private EventsByTypeRepository eventsByTypeRepository; 40 | 41 | public void persist(final String json) { 42 | LOG.debug("Saving event: {}", json); 43 | if(StringUtils.isBlank(json)) return; 44 | try { 45 | final UserActivityEvent userActivityEvent = jsonDeserialiser.convert(json, UserActivityEvent.class); 46 | saveEventByCorrelationId(userActivityEvent); 47 | saveEventsByReference(userActivityEvent); 48 | saveEventsByType(userActivityEvent); 49 | LOG.info("Successfully saved event: {} ", json); 50 | } catch (final IOException ex) { 51 | LOG.error("Error encountered while trying to save the event: {}", json, ex); 52 | } 53 | } 54 | 55 | //TODO: Make it asynchronous 56 | public void saveEventByCorrelationId(final UserActivityEvent userActivityEvent) { 57 | final EventByCorrelationId eventByCorrelationId = messagingToDomainObjectConverter.toEventByCorrelationId(userActivityEvent); 58 | eventsByCorrelationIdRepository.save(eventByCorrelationId); 59 | } 60 | 61 | //TODO: Make it asynchronous 62 | public void saveEventsByReference(final UserActivityEvent userActivityEvent) { 63 | final EventByReference eventByReference = messagingToDomainObjectConverter.toEventByReference(userActivityEvent); 64 | eventsByReferenceRepository.save(eventByReference); 65 | } 66 | 67 | //TODO: Make it asynchronous 68 | public void saveEventsByType(final UserActivityEvent userActivityEvent) { 69 | final EventByType eventByType = messagingToDomainObjectConverter.toEventByType(userActivityEvent); 70 | eventsByTypeRepository.save(eventByType); 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /src/main/resources/cassandra.properties: -------------------------------------------------------------------------------- 1 | # Cassandra database connection parameters 2 | cassandra.contactpoints=localhost 3 | cassandra.port=9042 4 | cassandra.keyspace=users_activity_db -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n 6 | 7 | log4j.logger.org.springframework=info 8 | log4j.logger.org.cassandraunit=info 9 | log4j.logger.io.netty=error 10 | log4j.logger.me.prettyprint=info 11 | log4j.logger.org.apache=error 12 | log4j.logger.com.datastax=error 13 | log4j.logger.com.example=info -------------------------------------------------------------------------------- /src/test/java/com/example/configuration/TestConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.example.configuration; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.beans.factory.annotation.Value; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.ComponentScan; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.context.annotation.FilterType; 10 | import org.springframework.context.annotation.PropertySource; 11 | import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; 12 | import org.springframework.data.cassandra.config.CassandraClusterFactoryBean; 13 | import org.springframework.data.cassandra.config.java.AbstractCassandraConfiguration; 14 | import org.springframework.data.cassandra.mapping.BasicCassandraMappingContext; 15 | import org.springframework.data.cassandra.mapping.CassandraMappingContext; 16 | import org.springframework.data.cassandra.repository.config.EnableCassandraRepositories; 17 | 18 | @Configuration 19 | @PropertySource(value = { "classpath:cassandra-test.properties" }) 20 | @ComponentScan(basePackages = "com.example", 21 | excludeFilters = { 22 | @ComponentScan.Filter(type = FilterType.ANNOTATION, value = Configuration.class), 23 | @ComponentScan.Filter(type = FilterType.REGEX, pattern = { "com.example.configuration.ApplicationConfiguration" }) 24 | }) 25 | @EnableCassandraRepositories(basePackages = "com.example.repository") 26 | public class TestConfiguration extends AbstractCassandraConfiguration { 27 | private static final Logger LOG = LoggerFactory.getLogger(TestConfiguration.class); 28 | 29 | @Value("${cassandra.keyspace}") 30 | private String keySpace; 31 | 32 | @Value("${cassandra.contactpoints}") 33 | private String contactPoints; 34 | 35 | @Value("${cassandra.port}") 36 | private String port; 37 | 38 | @Bean 39 | public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer() { 40 | return new PropertySourcesPlaceholderConfigurer(); 41 | } 42 | 43 | @Override 44 | protected String getKeyspaceName() { 45 | LOG.info("Using key space: '{}' for test configuration.\n", keySpace); 46 | return keySpace; 47 | } 48 | 49 | @Override 50 | @Bean 51 | public CassandraClusterFactoryBean cluster() { 52 | LOG.info("Creating test cluster created with contact points [" + contactPoints + "] " + "& port [" + port + "]."); 53 | final CassandraClusterFactoryBean cluster = new CassandraClusterFactoryBean(); 54 | cluster.setContactPoints(contactPoints); 55 | cluster.setPort(Integer.parseInt(port)); 56 | return cluster; 57 | } 58 | 59 | @Override 60 | @Bean 61 | public CassandraMappingContext cassandraMapping() throws ClassNotFoundException { 62 | return new BasicCassandraMappingContext(); 63 | } 64 | } -------------------------------------------------------------------------------- /src/test/java/com/example/converter/MessagingToDomainObjectConverterTest.java: -------------------------------------------------------------------------------- 1 | package com.example.converter; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | 7 | public class MessagingToDomainObjectConverterTest { 8 | 9 | @Before 10 | public void setUp() throws Exception { 11 | 12 | } 13 | 14 | @After 15 | public void tearDown() throws Exception { 16 | 17 | } 18 | 19 | @Test 20 | public void should_convert_to_event_by_correlationId() throws Exception { 21 | // TODO: Complete test 22 | } 23 | 24 | @Test 25 | public void should_convert_to_event_by_reference() throws Exception { 26 | // TODO: Complete test 27 | } 28 | } -------------------------------------------------------------------------------- /src/test/java/com/example/mapper/JSONDeserialiserTest.java: -------------------------------------------------------------------------------- 1 | package com.example.mapper; 2 | 3 | import static org.hamcrest.CoreMatchers.equalTo; 4 | import static org.hamcrest.CoreMatchers.notNullValue; 5 | import static org.hamcrest.Matchers.equalToIgnoringCase; 6 | import static org.junit.Assert.assertThat; 7 | 8 | import java.nio.charset.StandardCharsets; 9 | import java.nio.file.Files; 10 | import java.nio.file.Paths; 11 | import java.time.LocalDateTime; 12 | 13 | import org.junit.After; 14 | import org.junit.Before; 15 | import org.junit.Test; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | 19 | import com.example.domain.UserActivityEvent; 20 | 21 | /** 22 | * 23 | */ 24 | public class JSONDeserialiserTest { 25 | //TODO: Convert to spring test class 26 | 27 | private static final Logger LOG = LoggerFactory.getLogger(JSONDeserialiserTest.class); 28 | 29 | public static final String KAFKA_EVENT = "src/test/resources/kafka_event.json"; 30 | 31 | private String json; 32 | private JSONDeserialiser jsonDeserialiser; 33 | 34 | @Before 35 | public void setUp() throws Exception { 36 | json = new String (Files.readAllBytes(Paths.get(KAFKA_EVENT)), StandardCharsets.UTF_8); 37 | jsonDeserialiser = new JSONDeserialiser(); 38 | } 39 | 40 | @After 41 | public void tearDown() throws Exception { 42 | json = null; 43 | jsonDeserialiser = null; 44 | } 45 | 46 | @Test 47 | public void should_convert_json_to_object() throws Exception { 48 | LOG.debug("Checking if json string is properly converted to an object ..."); 49 | final UserActivityEvent userActivityEvent = jsonDeserialiser.convert(json, UserActivityEvent.class); 50 | assertThat(userActivityEvent, notNullValue() ); 51 | assertThat(userActivityEvent.getCorrelationId(), equalToIgnoringCase("2c08b8f1-afbb-455a-a1bd-31f15115d424") ); 52 | assertThat(userActivityEvent.getDate(), equalTo(LocalDateTime.of(2016,2,29,23,45,50,40)) ); 53 | } 54 | } -------------------------------------------------------------------------------- /src/test/java/com/example/repository/CQLScriptIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package com.example.repository; 2 | 3 | import static org.hamcrest.MatcherAssert.assertThat; 4 | import static org.hamcrest.Matchers.is; 5 | 6 | import org.cassandraunit.spring.CassandraDataSet; 7 | import org.cassandraunit.spring.CassandraUnitDependencyInjectionTestExecutionListener; 8 | import org.cassandraunit.spring.CassandraUnitTestExecutionListener; 9 | import org.cassandraunit.spring.EmbeddedCassandra; 10 | import org.junit.Test; 11 | import org.junit.runner.RunWith; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | import org.springframework.test.context.ContextConfiguration; 15 | import org.springframework.test.context.TestExecutionListeners; 16 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 17 | import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; 18 | 19 | import com.datastax.driver.core.Cluster; 20 | import com.datastax.driver.core.Session; 21 | import com.example.configuration.TestConfiguration; 22 | 23 | 24 | @RunWith(SpringJUnit4ClassRunner.class) 25 | @ContextConfiguration(classes = TestConfiguration.class) 26 | @TestExecutionListeners({CassandraUnitDependencyInjectionTestExecutionListener.class, CassandraUnitTestExecutionListener.class, DependencyInjectionTestExecutionListener.class }) 27 | @EmbeddedCassandra(timeout = 10000) 28 | @CassandraDataSet(value = { "users_activity_db.cql" }, keyspace = "cassandra_unit_keyspace") 29 | public class CQLScriptIntegrationTest { 30 | 31 | public static final String CONTACT_POINT = "127.0.0.1"; 32 | public static final int PORT = 9142; 33 | public static final String DEFAULT_CASSANDRA_UNIT_KEYSPACE = "cassandra_unit_keyspace"; 34 | public static final String QUERY_EVENTS_BY_CORRELATION_ID = "SELECT * FROM events_by_correlation_id " + 35 | "WHERE correlation_id = 'cfd66ccc-d857-4e90-b1e5-df98a3d40cd6' "; 36 | public static final String QUERY_EVENTS_BY_REFERENCE = "SELECT * FROM events_by_reference_and_year " + 37 | "WHERE reference = 'wenstfhjwwhucm9zme04szfyq0tdykpbtxlmwlbnywm[' AND year = 2016 AND event_date_time > '2016-01-20 10:26:06' "; 38 | private static final Logger LOG = LoggerFactory.getLogger(CQLScriptIntegrationTest.class); 39 | 40 | @Test 41 | public void should_correctly_execute_cql_script_to_create_column_families() throws Exception { 42 | 43 | LOG.debug("Creating a session to connect to the embedded cassandra database ..."); 44 | Cluster cluster = Cluster.builder().addContactPoints(CONTACT_POINT).withPort(PORT).build(); 45 | Session session = cluster.connect(DEFAULT_CASSANDRA_UNIT_KEYSPACE); 46 | 47 | LOG.debug("Verifying the pre-executed script through execution of select queries ..."); 48 | assertThat(session.execute(QUERY_EVENTS_BY_CORRELATION_ID).iterator().next().getString("correlation_id"), is("cfd66ccc-d857-4e90-b1e5-df98a3d40cd6")); 49 | assertThat(session.execute(QUERY_EVENTS_BY_REFERENCE).iterator().next().getString("event_type"), is("USER_ACCESSED_PROFILE")); 50 | 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/com/example/repository/CassandraRepositoriesIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package com.example.repository; 2 | 3 | import static org.hamcrest.CoreMatchers.notNullValue; 4 | import static org.hamcrest.text.IsEqualIgnoringCase.equalToIgnoringCase; 5 | import static org.junit.Assert.assertThat; 6 | import static org.junit.Assert.assertTrue; 7 | 8 | import java.io.IOException; 9 | import java.nio.charset.StandardCharsets; 10 | import java.nio.file.Files; 11 | import java.nio.file.Paths; 12 | import java.time.LocalDateTime; 13 | import java.time.ZoneId; 14 | import java.util.Date; 15 | import java.util.HashMap; 16 | import java.util.stream.StreamSupport; 17 | 18 | import org.apache.cassandra.exceptions.ConfigurationException; 19 | import org.apache.commons.lang.StringUtils; 20 | import org.apache.thrift.transport.TTransportException; 21 | import org.cassandraunit.spring.CassandraDataSet; 22 | import org.cassandraunit.spring.CassandraUnitDependencyInjectionTestExecutionListener; 23 | import org.cassandraunit.spring.CassandraUnitTestExecutionListener; 24 | import org.cassandraunit.spring.EmbeddedCassandra; 25 | import org.junit.Before; 26 | import org.junit.Test; 27 | import org.junit.runner.RunWith; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | import org.springframework.beans.factory.annotation.Autowired; 31 | import org.springframework.cassandra.core.cql.CqlIdentifier; 32 | import org.springframework.data.cassandra.core.CassandraAdminOperations; 33 | import org.springframework.data.cassandra.core.CassandraOperations; 34 | import org.springframework.test.context.ContextConfiguration; 35 | import org.springframework.test.context.TestExecutionListeners; 36 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 37 | import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; 38 | 39 | import com.datastax.driver.core.querybuilder.QueryBuilder; 40 | import com.datastax.driver.core.querybuilder.Select; 41 | import com.example.configuration.TestConfiguration; 42 | import com.example.converter.MessagingToDomainObjectConverter; 43 | import com.example.domain.UserActivityEvent; 44 | import com.example.mapper.JSONDeserialiser; 45 | import com.example.model.EventByCorrelationId; 46 | import com.example.model.EventByReference; 47 | import com.example.model.EventByType; 48 | 49 | @RunWith(SpringJUnit4ClassRunner.class) 50 | @ContextConfiguration(classes = TestConfiguration.class) 51 | @TestExecutionListeners({CassandraUnitDependencyInjectionTestExecutionListener.class, CassandraUnitTestExecutionListener.class, DependencyInjectionTestExecutionListener.class }) 52 | @EmbeddedCassandra(timeout = 10000) 53 | @CassandraDataSet(value = { "users_activity_db.cql" }, keyspace = "cassandra_unit_keyspace") 54 | public class CassandraRepositoriesIntegrationTest { 55 | public static final String KAFKA_EVENT = "src/test/resources/kafka_event.json"; 56 | public static final String EVENTS_BY_CORRELATION_ID_COLUMN_FAMILY = "events_by_correlation_id"; 57 | public static final String EVENTS_BY_TYPE_COLUMN_FAMILY = "events_by_type"; 58 | public static final String EVENTS_BY_REFERENCE_AND_YEAR_COLUMN_FAMILY = "events_by_reference_and_year"; 59 | public static final String EVENT_REFERENCE = "qtvszw9qbjnswfprsxnfelk2yvdvqt09"; 60 | public static final String EVENT_CORRELATION_ID = "2c08b8f1-afbb-455a-a1bd-31f15115d424"; 61 | public static final String EVENT_TYPE = "LOGIN_INTERCEPTOR_CREATE_SESSION"; 62 | private static final Logger LOG = LoggerFactory.getLogger(CassandraRepositoriesIntegrationTest.class); 63 | 64 | @Autowired 65 | private CassandraAdminOperations adminTemplate; 66 | 67 | @Autowired 68 | private CassandraOperations cassandraTemplate; 69 | 70 | @Autowired 71 | private JSONDeserialiser jsonDeserialiser; 72 | 73 | @Autowired 74 | private MessagingToDomainObjectConverter messagingToDomainObjectConverter; 75 | 76 | @Autowired 77 | private EventsByCorrelationIdRepository eventsByCorrelationIdRepository; 78 | 79 | @Autowired 80 | private EventsByReferenceRepository eventsByReferenceRepository; 81 | 82 | @Autowired 83 | private EventsByTypeRepository eventsByTypeRepository; 84 | 85 | private String json; 86 | 87 | @Before 88 | public void readContent() throws InterruptedException, TTransportException, ConfigurationException, IOException { 89 | LOG.debug("Reading content for column family: {} from the data file: {} ...", EVENTS_BY_CORRELATION_ID_COLUMN_FAMILY, KAFKA_EVENT); 90 | json = new String (Files.readAllBytes(Paths.get(KAFKA_EVENT)), StandardCharsets.UTF_8); 91 | } 92 | 93 | @Test 94 | public void should_save_and_retrieve_event_by_correlation_id() { 95 | 96 | LOG.debug("Testing save operation for 'EventByCorrelationId' followed by a retrieve"); 97 | 98 | LOG.debug("Creating column-family: {} ...", EVENTS_BY_CORRELATION_ID_COLUMN_FAMILY); 99 | adminTemplate.createTable(true, CqlIdentifier.cqlId(EVENTS_BY_CORRELATION_ID_COLUMN_FAMILY), EventByCorrelationId.class, new HashMap<>()); 100 | 101 | final EventByCorrelationId eventByCorrelationId = messagingToDomainObjectConverter.toEventByCorrelationId(getUserActivityEvent(json)); 102 | assertThat( eventByCorrelationId, notNullValue()); 103 | assertThat( eventByCorrelationId.getReference(), equalToIgnoringCase(EVENT_REFERENCE) ); 104 | 105 | eventsByCorrelationIdRepository.save(eventByCorrelationId); 106 | 107 | final Select select = QueryBuilder.select().from(EVENTS_BY_CORRELATION_ID_COLUMN_FAMILY) 108 | .where(QueryBuilder.eq("correlation_id", EVENT_CORRELATION_ID)) 109 | .and(QueryBuilder.gt("event_date_time", convert(LocalDateTime.of(2012, 12, 31, 23, 59)) )).limit(10); 110 | 111 | final EventByCorrelationId event = cassandraTemplate.selectOne(select, EventByCorrelationId.class); 112 | 113 | assertThat(event, notNullValue()); 114 | assertThat(event.getPrimaryKey().getCorrelationId(), equalToIgnoringCase(EVENT_CORRELATION_ID)); 115 | assertThat(event.getReference(), equalToIgnoringCase(EVENT_REFERENCE)); 116 | 117 | LOG.debug("Dropping column family: {} ...", EVENTS_BY_CORRELATION_ID_COLUMN_FAMILY); 118 | adminTemplate.dropTable(CqlIdentifier.cqlId(EVENTS_BY_CORRELATION_ID_COLUMN_FAMILY)); 119 | } 120 | 121 | @Test 122 | public void should_save_and_retrieve_event_by_reference_and_year() { 123 | 124 | LOG.debug("Testing save operation for 'EventByReference' followed by a retrieve"); 125 | 126 | LOG.debug("Creating column-family: {} ...", EVENTS_BY_REFERENCE_AND_YEAR_COLUMN_FAMILY); 127 | adminTemplate.createTable(true, CqlIdentifier.cqlId(EVENTS_BY_REFERENCE_AND_YEAR_COLUMN_FAMILY), EventByReference.class, new HashMap<>()); 128 | 129 | final EventByReference eventByReference = messagingToDomainObjectConverter.toEventByReference(getUserActivityEvent(json)); 130 | assertThat( eventByReference, notNullValue()); 131 | assertThat( eventByReference.getPrimaryKey().getReference(), equalToIgnoringCase(EVENT_REFERENCE) ); 132 | 133 | eventsByReferenceRepository.save(eventByReference); 134 | 135 | final Iterable iterable = eventsByReferenceRepository.findByReferenceAndYear(EVENT_REFERENCE, 2016); 136 | 137 | assertTrue(StreamSupport.stream(iterable.spliterator(), false) 138 | .allMatch(event-> event.getPrimaryKey().getReference().equalsIgnoreCase(EVENT_REFERENCE) && 139 | event.getPrimaryKey().getYear() == 2016)); 140 | 141 | LOG.debug("Dropping column family: {} ...", EVENTS_BY_REFERENCE_AND_YEAR_COLUMN_FAMILY); 142 | adminTemplate.dropTable(CqlIdentifier.cqlId(EVENTS_BY_REFERENCE_AND_YEAR_COLUMN_FAMILY)); 143 | } 144 | 145 | @Test 146 | public void should_save_and_retrieve_event_by_type() { 147 | LOG.debug("Testing save operation for 'EventByType' followed by a retrieve"); 148 | 149 | LOG.debug("Creating column-family: {} ...", EVENTS_BY_TYPE_COLUMN_FAMILY); 150 | adminTemplate.createTable(true, CqlIdentifier.cqlId(EVENTS_BY_TYPE_COLUMN_FAMILY), EventByType.class, new HashMap<>()); 151 | 152 | final EventByType eventByType = messagingToDomainObjectConverter.toEventByType(getUserActivityEvent(json)); 153 | assertThat( eventByType, notNullValue()); 154 | assertThat( eventByType.getPrimaryKey().getEventType(), equalToIgnoringCase(EVENT_TYPE)); 155 | assertThat( eventByType.getReference(), equalToIgnoringCase(EVENT_REFERENCE) ); 156 | 157 | eventsByTypeRepository.save(eventByType); 158 | 159 | final Iterable iterable = eventsByTypeRepository.findByType(EVENT_TYPE); 160 | 161 | assertTrue(StreamSupport.stream(iterable.spliterator(), false) 162 | .allMatch(event-> event.getPrimaryKey().getEventType().equalsIgnoreCase(EVENT_TYPE))); 163 | 164 | LOG.debug("Dropping column family: {} ...", EVENTS_BY_TYPE_COLUMN_FAMILY); 165 | adminTemplate.dropTable(CqlIdentifier.cqlId(EVENTS_BY_TYPE_COLUMN_FAMILY)); 166 | } 167 | 168 | private UserActivityEvent getUserActivityEvent(final String json) { 169 | if(StringUtils.isBlank(json)) return null; 170 | try { 171 | return jsonDeserialiser.convert(json, UserActivityEvent.class); 172 | } catch (final IOException ioEx) { 173 | LOG.error("Error encountered while creating a user activity event from json: {}", json, ioEx); 174 | return null; 175 | } 176 | } 177 | 178 | private Date convert(final LocalDateTime localDateTime) { 179 | return (localDateTime == null)? null : Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); 180 | } 181 | 182 | } 183 | -------------------------------------------------------------------------------- /src/test/resources/cassandra-test.properties: -------------------------------------------------------------------------------- 1 | # Cassandra database connection parameters 2 | cassandra.contactpoints=127.0.0.1 3 | cassandra.port=9142 4 | cassandra.keyspace=cassandra_unit_keyspace -------------------------------------------------------------------------------- /src/test/resources/kafka_event.json: -------------------------------------------------------------------------------- 1 | {"id":1,"eventType":"LOGIN_INTERCEPTOR_CREATE_SESSION","date":[2016,2,29,23,45,50,40],"correlationId":"2c08b8f1-afbb-455a-a1bd-31f15115d424","clientIp":"198.39.100.45","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:39.0) Gecko/20100101 Firefox/39.0","userAgentFiltered":"FIREFOX39-MAC_OS_X","details":"Piet just logged in!","reference":"qtvszw9qbjnswfprsxnfelk2yvdvqt09"} -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | #Set root logger 's level and its appender to an appender called CONSOLE which is defined below. 2 | log4j.rootLogger=DEBUG, CONSOLE 3 | 4 | #Set the behavior of the CONSOLE appender 5 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 6 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.CONSOLE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n 8 | log4j.logger.org.springframework=info 9 | log4j.logger.org.cassandraunit=info 10 | log4j.logger.io.netty=error 11 | log4j.logger.me.prettyprint=info 12 | log4j.logger.org.apache=error 13 | log4j.logger.com.datastax=error 14 | log4j.logger.com.example=debug -------------------------------------------------------------------------------- /src/test/resources/users_activity_db.cql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS events_by_correlation_id 2 | ( 3 | client_ip text, 4 | correlation_id text, 5 | reference text, 6 | event_type text, 7 | details text, 8 | user_agent text, 9 | user_agent_filtered text, 10 | event_date_time timestamp, 11 | PRIMARY KEY (correlation_id, event_date_time) 12 | ) WITH CLUSTERING ORDER BY (event_date_time DESC); 13 | 14 | ALTER TABLE events_by_correlation_id WITH COMPACTION = {'class':'DateTieredCompactionStrategy', 15 | 'timestamp_resolution':'MICROSECONDS', 16 | 'base_time_seconds':'3600', 17 | 'max_sstable_age_days':'730'}; 18 | 19 | CREATE TABLE IF NOT EXISTS events_by_reference_and_year 20 | ( 21 | reference text, 22 | year int, 23 | client_ip text, 24 | correlation_id text, 25 | event_type text, 26 | details text, 27 | user_agent text, 28 | user_agent_filtered text, 29 | event_date_time timestamp, 30 | PRIMARY KEY ((reference, year), event_date_time) 31 | ) WITH CLUSTERING ORDER BY (event_date_time DESC); 32 | 33 | ALTER TABLE events_by_reference_and_year WITH COMPACTION = {'class':'DateTieredCompactionStrategy', 34 | 'timestamp_resolution':'MICROSECONDS', 35 | 'base_time_seconds':'3600', 36 | 'max_sstable_age_days':'730'}; 37 | 38 | CREATE TABLE IF NOT EXISTS events_by_type 39 | ( 40 | client_ip text, 41 | correlation_id text, 42 | reference text, 43 | event_type text, 44 | details text, 45 | user_agent text, 46 | user_agent_filtered text, 47 | event_date_time timestamp, 48 | PRIMARY KEY (event_type, event_date_time) 49 | ) WITH CLUSTERING ORDER BY (event_date_time DESC); 50 | 51 | ALTER TABLE events_by_type WITH COMPACTION = {'class':'DateTieredCompactionStrategy', 52 | 'timestamp_resolution':'MICROSECONDS', 53 | 'base_time_seconds':'3600', 54 | 'max_sstable_age_days':'730'}; 55 | 56 | INSERT INTO events_by_correlation_id (correlation_id, event_date_time, event_type, client_ip, details, reference, user_agent, user_agent_filtered) 57 | VALUES ('cfd66ccc-d857-4e90-b1e5-df98a3d40cd6', '2016-02-10 11:00:50', 'WAYF_DISPLAYED', '198.39.100.45', 'login page is displayed', '', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:39.0) Gecko/20100101 Firefox/39.0','FIREFOX39 - MAC_OS_X') 58 | USING TTL 86400; 59 | 60 | INSERT INTO events_by_reference_and_year (reference, year, event_date_time, event_type, correlation_id, client_ip, details, user_agent, user_agent_filtered) 61 | VALUES ('wenstfhjwwhucm9zme04szfyq0tdykpbtxlmwlbnywm[', 2016, '2016-01-20 10:26:05','LOGIN_INTERCEPTOR_CREATE_SESSION','4ba35527-26c2-4cb8-8ec4-c40b288d62b0','89.255.57.26', 'aegon:7a5d6f13-4b65-4d7b-ba40-cce0a4833d2e', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0','FIREFOX4 - MAC_OS_X') 62 | USING TTL 86400; 63 | 64 | INSERT INTO events_by_reference_and_year (reference, year, event_date_time, event_type, correlation_id, client_ip, details, user_agent, user_agent_filtered) 65 | VALUES ('wenstfhjwwhucm9zme04szfyq0tdykpbtxlmwlbnywm[', 2016, '2016-01-20 10:26:07','USER_ACCESSED_PROFILE','4ba35527-26c2-4cb8-8ec4-c40b288d62b0','89.255.57.26', 'NA', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0','FIREFOX4 - MAC_OS_X') 66 | USING TTL 86400; --------------------------------------------------------------------------------