├── .gitignore ├── README.md ├── config-server-repository ├── config-client-analytics.yml ├── config-client-elastic_query.yml ├── config-client-elastic_query_web.yml ├── config-client-elastic_query_web_2.yml ├── config-client-gateway.yml ├── config-client-kafka_streams.yml ├── config-client-kafka_to_elastic.yml ├── config-client-twitter_to_kafka.yml └── config-client.yml ├── doc ├── AuthClient.png ├── Client_Credentials_flow.jpg ├── Client_Credentials_flow.png ├── SECURITY.md └── generalArchitecture.png ├── docker-compose ├── .env ├── common.yml ├── config │ ├── logback.conf │ └── prometheus.yml ├── data │ ├── db-elastic-query-service.sql │ └── db-keycloak.sql ├── elastic_cluster.yml ├── kafka_cluster.yml ├── monitoring.yml ├── postgres_databases.yml ├── redis_cluster.yml ├── scripts │ ├── check-config-server-started.sh │ ├── check-kafka-topics-created.sh │ ├── check-keycloak-server-started.sh │ └── create-mapping-elasticsearch.sh ├── services.yml └── zipkin.yml ├── modules ├── app-config-data │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com.microservices.config │ │ │ ├── AnalyticsServiceConfigData.java │ │ │ ├── ElasticConfigData.java │ │ │ ├── ElasticQueryConfigData.java │ │ │ ├── ElasticQueryServiceConfigData.java │ │ │ ├── ElasticQueryWebClientConfigData.java │ │ │ ├── GatewayServiceConfigData.java │ │ │ ├── KafkaConfigData.java │ │ │ ├── KafkaConsumerConfigData.java │ │ │ ├── KafkaProducerConfigData.java │ │ │ ├── KafkaStreamsConfigData.java │ │ │ ├── KafkaStreamsServiceConfigData.java │ │ │ ├── RetryConfigData.java │ │ │ ├── TwitterToKafkaServiceConfigData.java │ │ │ └── UserConfigData.java │ │ └── resources │ │ └── logback-common.xml ├── common-config │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── microservices │ │ └── common │ │ └── config │ │ └── RetryConfig.java ├── common-util │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── microservices │ │ └── common │ │ └── util │ │ └── CollectionsUtil.java ├── elastic │ ├── elastic-config │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── microservices │ │ │ └── elastic │ │ │ └── config │ │ │ └── ElasticsearchConfig.java │ ├── elastic-index-client │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── microservices │ │ │ └── elastic │ │ │ └── index │ │ │ └── client │ │ │ ├── repository │ │ │ └── TwitterElasticsearchIndexRepository.java │ │ │ ├── service │ │ │ ├── ElasticIndexClient.java │ │ │ └── impl │ │ │ │ ├── TwitterElasticIndexClient.java │ │ │ │ └── TwitterElasticRepositoryIndexClient.java │ │ │ └── util │ │ │ └── ElasticIndexUtil.java │ ├── elastic-model │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── microservices │ │ │ └── elastic │ │ │ └── model │ │ │ └── index │ │ │ ├── IndexModel.java │ │ │ └── impl │ │ │ └── TwitterIndexModel.java │ ├── elastic-query-client │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── microservices │ │ │ └── elastic │ │ │ └── query │ │ │ └── client │ │ │ ├── exception │ │ │ └── ElasticQueryClientException.java │ │ │ ├── repository │ │ │ ├── TwitterElasticsearchQueryRepository.java │ │ │ └── impl │ │ │ │ └── TwitterElasticRepositoryQueryClient.java │ │ │ ├── service │ │ │ ├── ElasticQueryClient.java │ │ │ └── impl │ │ │ │ └── TwitterElasticQueryClient.java │ │ │ └── util │ │ │ └── ElasticQueryUtil.java │ └── pom.xml ├── kafka │ ├── kafka-admin │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── microservices │ │ │ └── kafka │ │ │ └── admin │ │ │ ├── client │ │ │ └── KafkaAdminClient.java │ │ │ ├── config │ │ │ ├── KafkaAdminConfig.java │ │ │ └── WebClientConfig.java │ │ │ └── exception │ │ │ └── KafkaClientException.java │ ├── kafka-consumer │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── microservices │ │ │ └── kafka │ │ │ └── consumer │ │ │ └── config │ │ │ └── KafkaConsumerConfig.java │ ├── kafka-model │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── com │ │ │ │ └── microservices │ │ │ │ └── kafka │ │ │ │ └── avro │ │ │ │ └── model │ │ │ │ ├── TwitterAnalyticsAvroModel.java │ │ │ │ └── TwitterAvroModel.java │ │ │ └── resources │ │ │ └── avro │ │ │ ├── twitter-analytics.avsc │ │ │ └── twitter.avsc │ ├── kafka-producer │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── microservices │ │ │ └── kafka │ │ │ └── producer │ │ │ └── config │ │ │ ├── KafkaProducerConfig.java │ │ │ └── service │ │ │ ├── KafkaProducer.java │ │ │ └── impl │ │ │ └── TwitterKafkaProducer.java │ └── pom.xml └── mdc-interceptor │ ├── pom.xml │ └── src │ └── main │ └── java │ └── com │ └── microservices │ └── mdc │ ├── Constants.java │ ├── config │ ├── IdGeneratorConfig.java │ └── WebMvcConfig.java │ └── interceptor │ └── MDCHandlerInterceptor.java ├── mvnw ├── mvnw.cmd ├── pom.xml └── services ├── analytics-service ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── microservices │ │ └── analytics │ │ └── service │ │ ├── AnalyticsApplication.java │ │ ├── Constants.java │ │ ├── api │ │ └── AnalyticsController.java │ │ ├── business │ │ ├── AnalyticsService.java │ │ ├── KafkaConsumer.java │ │ └── impl │ │ │ ├── AnalyticsKafkaConsumer.java │ │ │ └── TwitterAnalyticsService.java │ │ ├── config │ │ └── WebSecurityConfig.java │ │ ├── dataaccess │ │ ├── entity │ │ │ ├── AnalyticsEntity.java │ │ │ └── BaseEntity.java │ │ └── repository │ │ │ ├── AnalyticsCustomRepository.java │ │ │ ├── AnalyticsRepository.java │ │ │ └── impl │ │ │ └── AnalyticsRepositoryImpl.java │ │ ├── model │ │ └── AnalyticsResponseModel.java │ │ ├── security │ │ ├── AnalyticsUser.java │ │ ├── AnalyticsUserDetailsService.java │ │ ├── AnalyticsUserJwtConverter.java │ │ └── AudienceValidator.java │ │ └── transformer │ │ ├── AvroToDbEntityModelTransformer.java │ │ └── EntityToResponseModelTransformer.java │ └── resources │ ├── application.yml │ └── logback-spring.xml ├── config-server ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── microservices │ │ └── config │ │ └── server │ │ ├── ConfigServer.java │ │ └── config │ │ └── SecurityConfig.java │ └── resources │ ├── application.yml │ └── logback-spring.xml ├── discovery-service ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── microservices │ │ └── discovery │ │ └── service │ │ └── ServiceRegistrationAndDiscoveryServiceApplication.java │ └── resources │ ├── application-singleserver.yml │ ├── application.yml │ └── logback-spring.xml ├── elastic-query-service ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── microservices │ │ └── query │ │ └── service │ │ ├── Constants.java │ │ ├── ElasticQueryServiceApplication.java │ │ ├── QueryType.java │ │ ├── api │ │ ├── ElasticDocumentController.java │ │ └── error │ │ │ └── handler │ │ │ └── ElasticQueryServiceErrorHandler.java │ │ ├── business │ │ ├── ElasticQueryService.java │ │ ├── QueryUserService.java │ │ └── impl │ │ │ ├── TwitterElasticQueryService.java │ │ │ └── TwitterQueryUserService.java │ │ ├── config │ │ ├── WebClientConfig.java │ │ └── WebSecurityConfig.java │ │ ├── dataaccess │ │ ├── entity │ │ │ └── UserPermission.java │ │ └── repository │ │ │ └── UserPermissionRepository.java │ │ ├── model │ │ ├── ElasticQueryServiceAnalyticsResponseModel.java │ │ ├── ElasticQueryServiceRequestModel.java │ │ ├── ElasticQueryServiceResponseModel.java │ │ ├── ElasticQueryServiceWordCountResponseModel.java │ │ └── assembler │ │ │ └── ElasticQueryServiceResponseModelAssembler.java │ │ ├── security │ │ ├── AudienceValidator.java │ │ ├── PermissionType.java │ │ ├── QueryServicePermissionEvaluator.java │ │ ├── TwitterQueryUser.java │ │ ├── TwitterQueryUserDetailsService.java │ │ └── TwitterQueryUserJwtConverter.java │ │ └── transformer │ │ ├── ElasticToResponseModelTransformer.java │ │ └── UserPermissionsToUserDetailTransformer.java │ └── resources │ ├── application.yml │ └── logback-spring.xml ├── elastic-query-web-client-2 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── microservices │ │ └── elastic │ │ └── query │ │ └── web │ │ └── client │ │ ├── ElasticQueryWebClientApplication.java │ │ ├── api │ │ ├── QueryController.java │ │ └── error │ │ │ └── handler │ │ │ └── ElasticQueryWebClientErrorHandler.java │ │ ├── config │ │ ├── WebClientConfig.java │ │ └── WebSecurityConfig.java │ │ ├── exception │ │ └── ElasticQueryWebClientException.java │ │ ├── model │ │ ├── ElasticQueryWebClientAnalyticsResponseModel.java │ │ ├── ElasticQueryWebClientRequestModel.java │ │ └── ElasticQueryWebClientResponseModel.java │ │ └── service │ │ ├── ElasticQueryWebClient.java │ │ └── impl │ │ └── TwitterElasticQueryWebClient.java │ └── resources │ ├── application.yml │ ├── logback-spring.xml │ └── templates │ ├── error.html │ ├── footer.html │ ├── header.html │ ├── home.html │ └── index.html ├── elastic-query-web-client ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── microservices │ │ └── elastic │ │ └── query │ │ └── web │ │ └── client │ │ ├── ElasticQueryWebClientApplication.java │ │ ├── api │ │ ├── QueryController.java │ │ └── error │ │ │ └── handler │ │ │ └── ElasticQueryWebClientErrorHandler.java │ │ ├── config │ │ ├── WebClientConfig.java │ │ └── WebSecurityConfig.java │ │ ├── exception │ │ └── ElasticQueryWebClientException.java │ │ ├── model │ │ ├── ElasticQueryWebClientAnalyticsResponseModel.java │ │ ├── ElasticQueryWebClientRequestModel.java │ │ └── ElasticQueryWebClientResponseModel.java │ │ └── service │ │ ├── ElasticQueryWebClient.java │ │ └── impl │ │ └── TwitterElasticQueryWebClient.java │ └── resources │ ├── application.yml │ ├── logback-spring.xml │ └── templates │ ├── error.html │ ├── footer.html │ ├── header.html │ ├── home.html │ └── index.html ├── gateway-service ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── microservices │ │ └── gateway │ │ └── service │ │ ├── GatewayServiceApplication.java │ │ ├── config │ │ ├── GatewayConfig.java │ │ └── WebSecurityConfig.java │ │ ├── controller │ │ └── FallbackController.java │ │ └── model │ │ ├── AnalyticsDataFallbackModel.java │ │ └── QueryServiceFallbackModel.java │ └── resources │ ├── application.yml │ └── logback-spring.xml ├── kafka-streams-service ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── microservices │ │ └── kafka │ │ └── streams │ │ └── service │ │ ├── Constants.java │ │ ├── KafkaStreamsServiceApplication.java │ │ ├── api │ │ └── KafkaStreamsController.java │ │ ├── config │ │ ├── KafkaStreamsConfig.java │ │ └── WebSecurityConfig.java │ │ ├── init │ │ ├── StreamsInitializer.java │ │ └── impl │ │ │ └── KafkaStreamsInitializer.java │ │ ├── model │ │ └── KafkaStreamsResponseModel.java │ │ ├── runner │ │ ├── StreamsRunner.java │ │ └── impl │ │ │ └── KafkaStreamsRunner.java │ │ └── security │ │ ├── AudienceValidator.java │ │ ├── KafkaStreamsUser.java │ │ ├── KafkaStreamsUserDetailsService.java │ │ └── KafkaStreamsUserJwtConverter.java │ └── resources │ ├── application.yml │ └── logback-spring.xml ├── kafka-to-elastic-service ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── microservices │ │ └── kafka │ │ └── service │ │ ├── KafkaToElasticServiceApplication.java │ │ ├── consumer │ │ ├── KafkaConsumer.java │ │ └── impl │ │ │ └── TwitterKafkaConsumer.java │ │ └── transformer │ │ └── AvroToElasticModelTransformer.java │ └── resources │ ├── application.yml │ └── logback-spring.xml └── twitter-to-kafka-service ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── microservices │ │ ├── TwitterToKafkaServiceApplication.java │ │ ├── init │ │ ├── StreamInitializer.java │ │ └── impl │ │ │ └── KafkaStreamInitializer.java │ │ ├── listener │ │ └── TwitterKafkaStatusListener.java │ │ ├── runner │ │ ├── StreamRunner.java │ │ └── impl │ │ │ └── MockKafkaStreamRunner.java │ │ └── transformer │ │ └── TwitterStatusToAvroTransformer.java └── resources │ ├── application.yml │ └── logback-spring.xml └── test └── java └── com └── microservices └── TwitterToKafkaServiceApplicationTests.java /.gitignore: -------------------------------------------------------------------------------- 1 | ###################### 2 | # Project Specific 3 | ###################### 4 | **/build/www/** 5 | **/src/test/javascript/coverage/ 6 | **/src/test/javascript/PhantomJS*/ 7 | 8 | ###################### 9 | # Node 10 | ###################### 11 | **/node/ 12 | **/node_tmp/ 13 | **/node_modules/ 14 | **/npm-debug.log.* 15 | 16 | ###################### 17 | # SASS 18 | ###################### 19 | **/sass-cache/ 20 | 21 | ###################### 22 | # Eclipse 23 | ###################### 24 | **/*.pydevproject 25 | **/.project 26 | **/.metadata 27 | **/tmp/ 28 | **/tmp/**/* 29 | **/*.tmp 30 | **/*.bak 31 | **/*.swp 32 | **/*~.nib 33 | **/local.properties 34 | **/.classpath 35 | **/.settings/ 36 | **/.loadpath 37 | **/.factorypath 38 | **/src/main/resources/rebel.xml 39 | 40 | # External tool builders 41 | **/.externalToolBuilders/** 42 | 43 | # Locally stored "Eclipse launch configurations" 44 | **/*.launch 45 | 46 | # CDT-specific 47 | **/.cproject 48 | 49 | # PDT-specific 50 | **/.buildpath 51 | 52 | ###################### 53 | # Intellij 54 | ###################### 55 | **/.idea/ 56 | **/*.iml 57 | **/*.iws 58 | **/*.ipr 59 | **/*.ids 60 | **/*.orig 61 | 62 | ###################### 63 | # Visual Studio Code 64 | ###################### 65 | **/.vscode/ 66 | 67 | ###################### 68 | # Maven 69 | ###################### 70 | **/log/ 71 | **/*/target/** 72 | ###################### 73 | # Gradle 74 | ###################### 75 | **/.gradle/ 76 | **//build/ 77 | 78 | ###################### 79 | # Package Files 80 | ###################### 81 | **/*.jar 82 | **/*.war 83 | **/*.ear 84 | **/*.db 85 | 86 | ###################### 87 | # Windows 88 | ###################### 89 | # Windows image file caches 90 | **/Thumbs.db 91 | 92 | # Folder config file 93 | **/Desktop.ini 94 | 95 | ###################### 96 | # Mac OSX 97 | ###################### 98 | **/.DS_Store 99 | **/.svn 100 | 101 | # Thumbnails 102 | **/._* 103 | 104 | # Files that might appear on external disk 105 | **/.Spotlight-V100 106 | **/.Trashes 107 | 108 | ###################### 109 | # Directories 110 | ###################### 111 | **/bin/ 112 | **/deploy/ 113 | 114 | ###################### 115 | # Logs 116 | ###################### 117 | **/*.log 118 | 119 | ###################### 120 | # Others 121 | ###################### 122 | **/*.class 123 | **/*.*~ 124 | **/*~ 125 | **/.merge_file* 126 | 127 | ###################### 128 | # Gradle Wrapper 129 | ###################### 130 | **/!gradle/wrapper/gradle-wrapper.jar 131 | 132 | ###################### 133 | # Maven Wrapper 134 | ###################### 135 | **/!.mvn/wrapper/maven-wrapper.jar 136 | 137 | ###################### 138 | # ESLint 139 | ###################### 140 | **/.eslintcache 141 | **/.idea/ 142 | 143 | ###################### 144 | # Prometheus 145 | ###################### 146 | **/prometheus 147 | 148 | ###################### 149 | # Prometheus 150 | ###################### 151 | **/grafana -------------------------------------------------------------------------------- /config-server-repository/config-client-analytics.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 8188 3 | servlet: 4 | context-path: / 5 | 6 | analytics-service: 7 | version: v2 8 | custom-audience: analytics-service 9 | 10 | spring: 11 | cloud: 12 | loadbalancer: 13 | ribbon: 14 | enabled: false 15 | security: 16 | oauth2: 17 | resourceserver: 18 | jwt: 19 | issuer-uri: http://localhost:9091/auth/realms/myrealm 20 | jwk-set-uri: http://localhost:9091/auth/realms/myrealm/protocol/openid-connect/certs 21 | jpa: 22 | open-in-view: false 23 | show-sql: true 24 | database-platform: org.hibernate.dialect.PostgreSQL9Dialect 25 | properties: 26 | hibernate: 27 | dialect: org.hibernate.dialect.PostgreSQL9Dialect 28 | use_sql_comments: false 29 | format_sql: true 30 | # that means we want to batch the records by 50 before sending them to the database 31 | jdbc.batch_size: 50 32 | # with order_inserts and updates we configured hibernate to put inserts and updates in order separately to be able to use batching 33 | order_inserts: true 34 | order_updates: true 35 | datasource: 36 | url: jdbc:postgresql://localhost:5432/postgres?currentSchema=analytics&binaryTransfer=true&reWriteBatchedInserts=true 37 | username: postgres 38 | password: postgres 39 | driver-class-name: org.postgresql.Driver 40 | platform: postgres 41 | 42 | springdoc: 43 | api-docs: 44 | path: /api-docs 45 | swagger-ui: 46 | path: /swagger-ui.html 47 | 48 | kafka-config: 49 | bootstrap-servers: localhost:19092, localhost:29092, localhost:39092 50 | schema-registry-url-key: schema.registry.url 51 | schema-registry-url: http://localhost:8081 52 | topic-name: twitter-analytics-topic 53 | topic-names-to-create: 54 | - twitter-analytics-topic 55 | 56 | 57 | security: 58 | paths-to-ignore: /api-docs, /actuator/** 59 | 60 | kafka-consumer-config: 61 | key-deserializer: org.apache.kafka.common.serialization.StringDeserializer 62 | value-deserializer: io.confluent.kafka.serializers.KafkaAvroDeserializer 63 | consumer-group-id: twitter-topic-consumer 64 | auto-offset-reset: earliest 65 | specific-avro-reader-key: specific.avro.reader 66 | specific-avro-reader: true 67 | batch-listener: true 68 | auto-startup: false 69 | concurrency-level: 3 70 | session-timeout-ms: 10000 71 | heartbeat-interval-ms: 3000 72 | max-poll-interval-ms: 300000 73 | max-poll-records: 500 74 | max-partition-fetch-bytes-default: 1048576 75 | max-partition-fetch-bytes-boost-factor: 1 76 | poll-timeout-ms: 150 77 | 78 | retry-config: 79 | initial-interval-ms: 1000 80 | max-interval-ms: 10000 81 | multiplier: 2.0 82 | maxAttempts: 3 83 | sleep-time-ms: 2000 84 | 85 | 86 | management: 87 | endpoints: 88 | web: 89 | base-path: /actuator 90 | exposure.include: health, prometheus 91 | path-mapping.prometheus: metrics 92 | endpoint: 93 | health: 94 | show-details: always 95 | prometheus: 96 | cache.time-to-live: 1ms -------------------------------------------------------------------------------- /config-server-repository/config-client-elastic_query_web.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 8184 3 | servlet: 4 | context-path: /elastic-query-web-client 5 | 6 | elastic-query-web-client: 7 | webclient: 8 | connect-timeout-ms: 10000 9 | read-timeout-ms: 10000 10 | write-timeout-ms: 10000 11 | max-in-memory-size: 10485760 # 10MB 12 | content-type: 'application/vnd.api.v1+json' 13 | accept-type: 'application/vnd.api.v1+json' 14 | base-url: 'http://elastic-query-service/elastic-query-service/documents' 15 | query-by-text: 16 | method: POST 17 | uri: "/get-document-by-text" 18 | accept: ${elastic-query-web-client.webclient.accept-type} 19 | 20 | spring: 21 | cloud: 22 | loadbalancer: 23 | ribbon: 24 | enabled: false 25 | security: 26 | oauth2: 27 | client: 28 | registration: 29 | keycloak: 30 | client-id: 'elastic-query-web-client' 31 | client-secret: '8iYqVnAFHNQK6V9Nldn32nVdWQbl0b5t' 32 | authorization-grant-type: authorization_code 33 | redirect-uri: '{baseUrl}/login/oauth2/code/{registrationId}' 34 | scope: openid 35 | provider: 36 | keycloak: 37 | issuerUri: http://localhost:9091/realms/microservices_realm 38 | 39 | security: 40 | logout-success-url: http://localhost:8184/elastic-query-web-client/ 41 | default-client-registration-id: keycloak 42 | 43 | user-config: 44 | username: test 45 | password: test1234 46 | roles: USER -------------------------------------------------------------------------------- /config-server-repository/config-client-elastic_query_web_2.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 8186 3 | servlet: 4 | context-path: /elastic-query-web-client 5 | 6 | elastic-query-web-client: 7 | webclient: 8 | connect-timeout-ms: 10000 9 | read-timeout-ms: 10000 10 | write-timeout-ms: 10000 11 | max-in-memory-size: 10485760 # 10MB 12 | content-type: 'application/vnd.api.v1+json' 13 | accept-type: 'application/vnd.api.v1+json' 14 | base-url: 'http://elastic-query-service/elastic-query-service/documents' 15 | query-by-text: 16 | method: POST 17 | uri: "/get-document-by-text" 18 | accept: ${elastic-query-web-client.webclient.accept-type} 19 | 20 | 21 | spring: 22 | cloud: 23 | loadbalancer: 24 | ribbon: 25 | enabled: false 26 | security: 27 | oauth2: 28 | client: 29 | registration: 30 | keycloak: 31 | client-id: 'elastic-query-web-client-2' 32 | client-secret: 'j8mbzx15a4zJ78I498YnC5iJuHZWiTSx' 33 | authorization-grant-type: authorization_code 34 | redirect-uri: '{baseUrl}/login/oauth2/code/{registrationId}' 35 | scope: openid 36 | provider: 37 | keycloak: 38 | issuerUri: http://localhost:9091/realms/microservices_realm 39 | 40 | security: 41 | logout-success-url: http://localhost:8184/elastic-query-web-client/ 42 | default-client-registration-id: keycloak 43 | 44 | user-config: 45 | username: test 46 | password: test1234 47 | roles: USER -------------------------------------------------------------------------------- /config-server-repository/config-client-gateway.yml: -------------------------------------------------------------------------------- 1 | gateway-service: 2 | timeout-ms: 3000 3 | failure-rate-threshold: 50 4 | slow-call-rate-threshold: 50 5 | slow-call-duration-threshold: 50 6 | permitted-num-of-calls-in-half-open-state: 10 7 | sliding-window-size: 10 8 | min-number-of-calls: 10 9 | wait-duration-in-open-state: 60000 10 | 11 | server: 12 | port: 9090 13 | 14 | spring: 15 | application: 16 | name: gateway-service 17 | cloud: 18 | loadbalancer: 19 | ribbon: 20 | enabled: false 21 | gateway: 22 | discovery: 23 | locator: 24 | enabled: false 25 | lowerCaseServiceId: true 26 | routes: 27 | - id: elastic-query-service 28 | uri: lb://elastic-query-service 29 | predicates: 30 | - Path=/elastic-query-service/** 31 | filters: 32 | - RewritePath=/elastic-query-service/(?.*), /$\{path} 33 | - name: RequestRateLimiter 34 | args: 35 | redis-rate-limiter.replenishRate: 5 36 | redis-rate-limiter.burstCapacity: 10 37 | key-resolver: "#{@authHeaderResolver}" 38 | - name: CircuitBreaker 39 | args: 40 | name: queryServiceCircuitBreaker 41 | fallbackUri: forward:/fallback/query-fallback 42 | - id: analytics-service 43 | uri: lb://analytics-service 44 | predicates: 45 | - Path=/analytics-service/** 46 | filters: 47 | - RewritePath=/analytics-service/(?.*), /$\{path} 48 | - name: RequestRateLimiter 49 | args: 50 | redis-rate-limiter.replenishRate: 5 51 | redis-rate-limiter.burstCapacity: 10 52 | key-resolver: "#{@authHeaderResolver}" 53 | - name: CircuitBreaker 54 | args: 55 | name: analyticsServiceCircuitBreaker 56 | fallbackUri: forward:/fallback/analytics-fallback 57 | - id: kafka-streams-service 58 | uri: lb://kafka-streams-service 59 | predicates: 60 | - Path=/kafka-streams-service/** 61 | filters: 62 | - RewritePath=/kafka-streams-service/(?.*), /$\{path} 63 | - name: RequestRateLimiter 64 | args: 65 | redis-rate-limiter.replenishRate: 5 66 | redis-rate-limiter.burstCapacity: 10 67 | key-resolver: "#{@authHeaderResolver}" 68 | - name: CircuitBreaker 69 | args: 70 | name: streamsServiceCircuitBreaker 71 | fallbackUri: forward:/fallback/streams-fallback 72 | redis: 73 | host: 127.0.0.1 74 | port: 6379 75 | 76 | management: 77 | endpoints: 78 | web: 79 | base-path: /actuator 80 | exposure.include: health, prometheus, gateway 81 | path-mapping.prometheus: metrics 82 | endpoint: 83 | health: 84 | show-details: always 85 | prometheus: 86 | cache.time-to-live: 1ms 87 | gateway: 88 | enabled: true 89 | -------------------------------------------------------------------------------- /config-server-repository/config-client-kafka_streams.yml: -------------------------------------------------------------------------------- 1 | kafka-streams-service: 2 | custom-audience: kafka-streams-service 3 | 4 | server: 5 | port: 8187 6 | servlet: 7 | context-path: / 8 | 9 | spring: 10 | cloud: 11 | loadbalancer: 12 | ribbon: 13 | enabled: false 14 | security: 15 | oauth2: 16 | resourceserver: 17 | jwt: 18 | issuer-uri: http://localhost:9091/realms/microservices_realm 19 | jwk-set-uri: http://localhost:9091/realms/microservices_realm/protocol/openid-connect/certs 20 | 21 | springdoc: 22 | api-docs: 23 | path: /api-docs 24 | swagger-ui: 25 | path: /swagger-ui.html 26 | 27 | security: 28 | paths-to-ignore: /api-docs, /actuator/** 29 | 30 | 31 | retry-config: 32 | initial-interval-ms: 1000 33 | max-interval-ms: 10000 34 | multiplier: 2.0 35 | maxAttempts: 3 36 | sleep-time-ms: 2000 37 | 38 | kafka-config: 39 | bootstrap-servers: localhost:19092, localhost:29092, localhost:39092 40 | schema-registry-url-key: schema.registry.url 41 | schema-registry-url: http://localhost:8081 42 | topic-names-to-create: 43 | - twitter-topic 44 | - twitter-analytics-topic 45 | 46 | kafka-streams-config: 47 | application-id: kafka-streams-application 48 | input-topic-name: twitter-topic 49 | output-topic-name: twitter-analytics-topic 50 | state-file-location: ${HOME}/kafka-streaming-state 51 | word-count-store-name: word-count-store 52 | 53 | 54 | management: 55 | endpoints: 56 | web: 57 | base-path: /actuator 58 | exposure.include: health, prometheus 59 | path-mapping.prometheus: metrics 60 | endpoint: 61 | health: 62 | show-details: always 63 | prometheus: 64 | cache.time-to-live: 1ms -------------------------------------------------------------------------------- /config-server-repository/config-client-kafka_to_elastic.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 8182 3 | 4 | kafka-config: 5 | bootstrap-servers: localhost:19092, localhost:29092, localhost:39092 6 | schema-registry-url-key: schema.registry.url 7 | schema-registry-url: http://localhost:8081 8 | topic-name: twitter-topic 9 | topic-names-to-create: 10 | - twitter-topic 11 | num-of-partitions: 3 12 | replication-factor: 3 13 | 14 | retry-config: 15 | initial-interval-ms: 1000 16 | max-interval-ms: 10000 17 | multiplier: 2.0 18 | maxAttempts: 3 19 | sleep-time-ms: 2000 20 | 21 | kafka-consumer-config: 22 | # Since we read serialized data from the topic, the consumer has to know how to deserialize it. 23 | key-deserializer: org.apache.kafka.common.serialization.LongDeserializer 24 | value-deserializer: io.confluent.kafka.serializers.KafkaAvroDeserializer 25 | # Sets a unique id for a consumer to allow using offsets (so we don't start from beginning everytime) 26 | consumer-group-id: twitter-topic-consumer 27 | # We say that we want to start reading from the beginning of partition. 28 | auto-offset-reset: earliest 29 | # we work with avro type so we have to set this properties to true. 30 | specific-avro-reader-key: specific.avro.reader 31 | specific-avro-reader: true 32 | # This property allows to consume data in batches 33 | batch-listener: true 34 | # We want to start listening from topic after checking the topic is already there 35 | auto-startup: false 36 | # Sets the number of threads to work on consuming, required for maximum concurrency. 37 | # E.g. 1 topic, 4 partitions -> Set as 4 38 | concurrency-level: 3 39 | # It says that in this time the broker has to receive at least one heartbeat from the consumer, otherwise consider it broken 40 | session-timeout-ms: 10000 41 | # Specify the frequence of heartbeat from the consumer to the broker. 42 | heartbeat-interval-ms: 3000 43 | # This property is for user threads. If message processing logic is too heavy to cause larger than this time. 44 | # Coordinator explicitly have the consumer leave the group and also triggers 45 | # a new round of rebalance 46 | max-poll-interval-ms: 300000 47 | # Set the maximum records to fetch in each poll 48 | max-poll-records: 500 49 | # maximum bytes to fetch in each poll 50 | max-partition-fetch-bytes-default: 1048576 51 | max-partition-fetch-bytes-boost-factor: 1 52 | # how long we will wait until at least one record is available. 53 | # it will block in the poll method on kafka consumer nad wait this time for the new records. 54 | # too high will block too much and too low will cost CPU stall = CPU doesn't make progress although it is running. 55 | poll-timeout-ms: 150 56 | 57 | elastic-config: 58 | index-name: twitter-index 59 | connection-url: http://localhost:9200 60 | connect-timeout-ms: 5000 61 | socket-timeout-ms: 30000 62 | is-repository: true -------------------------------------------------------------------------------- /config-server-repository/config-client-twitter_to_kafka.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 8181 3 | 4 | twitter-to-kafka-service: 5 | twitter-keywords: 6 | - Java 7 | - Microservices 8 | - Spring 9 | - Kafka 10 | - Elasticsearch 11 | - Maradona 12 | - Naples 13 | enable-mock-tweets: true 14 | mock-min-tweet-length: 5 15 | mock-max-tweet-length: 15 16 | mock-sleep-ms: 1000 17 | 18 | retry-config: 19 | initial-interval-ms: 1000 20 | max-interval-ms: 10000 21 | multiplier: 2.0 22 | maxAttempts: 3 23 | sleep-time-ms: 2000 24 | 25 | kafka-config: 26 | bootstrap-servers: localhost:19092, localhost:29092, localhost:39092 27 | schema-registry-url-key: schema.registry.url 28 | schema-registry-url: http://localhost:8081 29 | topic-name: twitter-topic 30 | topic-names-to-create: 31 | - twitter-topic 32 | - twitter-analytics-topic 33 | num-of-partitions: 3 34 | replication-factor: 3 35 | 36 | kafka-producer-config: 37 | # key will be a long 38 | key-serializer-class: org.apache.kafka.common.serialization.LongSerializer 39 | # value will be a kafkaAvroSerializer 40 | value-serializer-class: io.confluent.kafka.serializers.KafkaAvroSerializer 41 | # Data will be compressed with the specific snappy (compression library from Google) 42 | compression-type: snappy 43 | # We want to get acknowledge from all the broker replicas to be more resilient 44 | acks: all 45 | batch-size: 16384 46 | # Tune for higher througput, maximum of tune is + 100 from default value 47 | batch-size-boost-factor: 100 48 | # 5ms add a delay on producer, useful in case of light load 49 | linger-ms: 5 50 | # After 6 seconds if not ack comes producer throw timeoutError 51 | request-timeout-ms: 60000 52 | # Retry max 5 times in case of error 53 | retry-count: 5 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /config-server-repository/config-client.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | config: 4 | allow-override: true 5 | override-none: true 6 | -------------------------------------------------------------------------------- /doc/AuthClient.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armando1514/Event-Driven-Microservices/0f0cba49a8824873d08b92869200421c0584db7f/doc/AuthClient.png -------------------------------------------------------------------------------- /doc/Client_Credentials_flow.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armando1514/Event-Driven-Microservices/0f0cba49a8824873d08b92869200421c0584db7f/doc/Client_Credentials_flow.jpg -------------------------------------------------------------------------------- /doc/Client_Credentials_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armando1514/Event-Driven-Microservices/0f0cba49a8824873d08b92869200421c0584db7f/doc/Client_Credentials_flow.png -------------------------------------------------------------------------------- /doc/generalArchitecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armando1514/Event-Driven-Microservices/0f0cba49a8824873d08b92869200421c0584db7f/doc/generalArchitecture.png -------------------------------------------------------------------------------- /docker-compose/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PATH_SEPARATOR=: 2 | COMPOSE_FILE=common.yml:zipkin.yml:monitoring.yml:redis_cluster.yml:postgres_databases.yml:kafka_cluster.yml:elastic_cluster.yml:services.yml 3 | KAFKA_VERSION=5.4.9 4 | ELASTIC_VERSION=7.7.1 5 | REDIS_VERSION=7.0.5 6 | KEYCLOAK_VERSION=19.0.1 7 | POSTGRES_VERSION=14.5 8 | GRAFANA_VERSION=9.0.9 9 | ZIPKIN_VERSION=2.22 10 | PROMETHEUS_VERSION=v2.37.1 11 | SERVICE_VERSION=0.0.1-SNAPSHOT 12 | GLOBAL_NETWORK=application 13 | GROUP_ID=com.microservices -------------------------------------------------------------------------------- /docker-compose/common.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | networks: 4 | application: 5 | driver: bridge -------------------------------------------------------------------------------- /docker-compose/config/logback.conf: -------------------------------------------------------------------------------- 1 | input { 2 | file { 3 | path => "/logs/*.log" 4 | codec => "json" 5 | type => "logback" 6 | } 7 | } 8 | 9 | output { 10 | if [type]=="logback" { 11 | elasticsearch { 12 | hosts => [ "http://elastic-1:9200" ] 13 | index => "logback-twitter-%{+YYYY.MM.dd}" 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /docker-compose/config/prometheus.yml: -------------------------------------------------------------------------------- 1 | scrape_configs: 2 | - job_name: 'prometheus' 3 | scrape_interval: 1m 4 | static_configs: 5 | - targets: ['localhost:9090'] 6 | - job_name: 'grafana' 7 | scrape_interval: 1m 8 | metrics_path: '/metrics' 9 | static_configs: 10 | - targets: ['grafana:3000'] 11 | - job_name: ' gateway-service-1' 12 | scrape_interval: 1m 13 | metrics_path: '/actuator/metrics' 14 | static_configs: 15 | - targets: ['gateway-service-1:9092'] 16 | - job_name: 'elastic-query-service-1' 17 | scrape_interval: 1m 18 | metrics_path: '/actuator/metrics' 19 | static_configs: 20 | - targets: ['elastic-query-service-1:8183'] 21 | - job_name: 'elastic-query-service-2' 22 | scrape_interval: 1m 23 | metrics_path: '/actuator/metrics' 24 | static_configs: 25 | - targets: [ 'elastic-query-service-2:8185' ] 26 | - job_name: 'kafka-streams-service' 27 | scrape_interval: 1m 28 | metrics_path: '/actuator/metrics' 29 | static_configs: 30 | - targets: ['kafka-streams-service:8187'] 31 | - job_name: 'analytics-service' 32 | scrape_interval: 1m 33 | metrics_path: '/actuator/metrics' 34 | static_configs: 35 | - targets: ['analytics-service:8188'] -------------------------------------------------------------------------------- /docker-compose/elastic_cluster.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | elastic-1: 4 | image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-latest} 5 | hostname: elastic-1 6 | environment: 7 | - node.name=elastic-1 8 | - cluster.name=es-twitter-cluster 9 | - discovery.seed_hosts=elastic-2,elastic-3 10 | - cluster.initial_master_nodes=elastic-1,elastic-2,elastic-3 11 | - bootstrap.memory_lock=true 12 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 13 | ulimits: 14 | memlock: 15 | soft: -1 16 | hard: -1 17 | volumes: 18 | - data01_7_15_2:/usr/share/elasticsearch/data 19 | ports: 20 | - 9200:9200 21 | networks: 22 | - ${GLOBAL_NETWORK:-services} 23 | elastic-2: 24 | image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION} 25 | environment: 26 | - node.name=elastic-2 27 | - cluster.name=es-twitter-cluster 28 | - discovery.seed_hosts=elastic-1,elastic-3 29 | - cluster.initial_master_nodes=elastic-1,elastic-2,elastic-3 30 | - bootstrap.memory_lock=true 31 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 32 | ulimits: 33 | memlock: 34 | soft: -1 35 | hard: -1 36 | volumes: 37 | - data02_7_15_2:/usr/share/elasticsearch/data 38 | networks: 39 | - ${GLOBAL_NETWORK:-services} 40 | elastic-3: 41 | image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION} 42 | environment: 43 | - node.name=elastic-3 44 | - cluster.name=es-twitter-cluster 45 | - discovery.seed_hosts=elastic-1,elastic-2 46 | - cluster.initial_master_nodes=elastic-1,elastic-2,elastic-3 47 | - bootstrap.memory_lock=true 48 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 49 | ulimits: 50 | memlock: 51 | soft: -1 52 | hard: -1 53 | volumes: 54 | - data03_7_15_2:/usr/share/elasticsearch/data 55 | networks: 56 | - ${GLOBAL_NETWORK:-elastic} 57 | kibana: 58 | image: docker.elastic.co/kibana/kibana:${ELASTIC_VERSION} 59 | ports: 60 | - 5601:5601 61 | depends_on: 62 | - elastic-1 63 | - elastic-2 64 | - elastic-3 65 | environment: 66 | ELASTICSEARCH_HOSTS: http://elastic-1:9200 67 | networks: 68 | - ${GLOBAL_NETWORK:-elastic} 69 | logstash: 70 | image: docker.elastic.co/logstash/logstash:${ELASTIC_VERSION} 71 | command: logstash -f /etc/logstash/conf.d/logback.conf 72 | ports: 73 | - "9600:9600" 74 | depends_on: 75 | - elastic-1 76 | - elastic-2 77 | - elastic-3 78 | volumes: 79 | - ./config/logback.conf:/etc/logstash/conf.d/logback.conf 80 | - ./docker-logs/logstash:/logs 81 | environment: 82 | LS_JAVA_OPTS: "-Xms256m -Xmx512m" 83 | CLUSTER_NAME: es-twitter-cluster 84 | XPACK_MONITORING_ELASTICSEARCH_HOSTS: http://elastic-1:9200 85 | networks: 86 | - ${GLOBAL_NETWORK:-services} 87 | 88 | volumes: 89 | data01_7_15_2: 90 | driver: local 91 | data02_7_15_2: 92 | driver: local 93 | data03_7_15_2: 94 | driver: local -------------------------------------------------------------------------------- /docker-compose/monitoring.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | grafana: 4 | image: grafana/grafana:${GRAFANA_VERSION:-latest} 5 | hostname: grafana 6 | ports: 7 | - 3000:3000 8 | volumes: 9 | - ./grafana:/var/lib/grafana 10 | environment: 11 | - GF_SECURITY_ADMIN_USER=admin 12 | - GF_SECURITY_ADMIN_PASSWORD=admin 13 | networks: 14 | - ${GLOBAL_NETWORK:-services} 15 | prometheus: 16 | image: prom/prometheus:${PROMETHEUS_VERSION:-latest} 17 | hostname: prometheus 18 | ports: 19 | - 9098:9090 20 | volumes: 21 | - ./config/prometheus.yml:/etc/prometheus/prometheus.yml 22 | - ./prometheus:/prometheus 23 | networks: 24 | - ${GLOBAL_NETWORK:-services} -------------------------------------------------------------------------------- /docker-compose/postgres_databases.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | postgres_db_keycloak: 4 | image: postgres:${POSTGRES_VERSION:-latest} 5 | environment: 6 | - "POSTGRES_USER=postgres" 7 | - "POSTGRES_PASSWORD=postgres" 8 | - "POSTGRES_DB=keycloak" 9 | ports: 10 | - '5432:5432' 11 | volumes: 12 | - "./data/db-keycloak.sql:/docker-entrypoint-initdb.d/db-keycloak.sql" 13 | - "postgres_storage_1:/var/lib/postgresql/data/" 14 | networks: 15 | - ${GLOBAL_NETWORK:-services} 16 | postgres_db_elastic_query_service: 17 | image: postgres:${POSTGRES_VERSION:-latest} 18 | environment: 19 | - "POSTGRES_USER=postgres" 20 | - "POSTGRES_PASSWORD=postgres" 21 | - "POSTGRES_DB=postgres" 22 | ports: 23 | - '5433:5432' 24 | volumes: 25 | - "./data/db-elastic-query-service.sql:/docker-entrypoint-initdb.d/db-elastic-query-service.sql" 26 | - "postgres_storage_2:/var/lib/postgresql/data/" 27 | networks: 28 | - ${GLOBAL_NETWORK:-services} 29 | 30 | volumes: 31 | postgres_storage_1: 32 | driver: local 33 | postgres_storage_2: 34 | driver: local 35 | -------------------------------------------------------------------------------- /docker-compose/redis_cluster.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | redis-master: 4 | image: redis:${REDIS_VERSION:-latest} 5 | hostname: redis 6 | command: ["redis-server", "--appendonly", "yes"] 7 | ports: 8 | - "6379:6379" 9 | networks: 10 | - ${GLOBAL_NETWORK:-services} 11 | redis-slave: 12 | image: redis:${REDIS_VERSION:-latest} 13 | command: ["redis-server", "--slaveof", "redis-master", "6379"] 14 | ports: 15 | - "6380:6379" 16 | depends_on: 17 | - redis-master 18 | networks: 19 | - ${GLOBAL_NETWORK:-services} -------------------------------------------------------------------------------- /docker-compose/scripts/check-config-server-started.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # check-config-server-started.sh 3 | 4 | apt-get update -y 5 | 6 | yes | apt-get install curl 7 | 8 | curlResult=$(curl -s -o /dev/null -I -w "%{http_code}" http://config-server-1:8888/actuator/health) 9 | 10 | echo "result status code:" "$curlResult" 11 | 12 | while [[ ! $curlResult == "200" ]]; do 13 | >&2 echo "Config server is not up yet!" 14 | sleep 2 15 | curlResult=$(curl -s -o /dev/null -I -w "%{http_code}" http://config-server-1:8888/actuator/health) 16 | done 17 | 18 | check-keycloak-server-started.sh -------------------------------------------------------------------------------- /docker-compose/scripts/check-kafka-topics-created.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # check-kafka-topics-created.sh 3 | 4 | apt-get update -y 5 | 6 | yes | apt-get install kafkacat 7 | 8 | kafkacatResult=$(kafkacat -L -b kafka-broker-1:9092) 9 | 10 | echo "kafkacat result:" $kafkacatResult 11 | 12 | while [[ ! $kafkacatResult == *"twitter-topic"* ]]; do 13 | >&2 echo "Kafka topic has not been created yet!" 14 | sleep 2 15 | kafkacatResult=$(kafkacat -L -b kafka-broker-1:9092) 16 | done 17 | 18 | 19 | /usr/local/bin/create-mapping-elasticsearch.sh 20 | -------------------------------------------------------------------------------- /docker-compose/scripts/check-keycloak-server-started.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # check-keycloak-server-started.sh 3 | curlResult=$(curl -s -o /dev/null -I -w "%{http_code}" http://keycloak-authorization-server:9091/realms/microservices_realm) 4 | 5 | echo "result status code:" "$curlResult" 6 | 7 | while [[ ! $curlResult == "200" ]]; do 8 | >&2 echo "Keycloak server is not up yet!" 9 | sleep 2 10 | curlResult=$(curl -s -o /dev/null -I -w "%{http_code}" http://keycloak-authorization-server:9091/realms/microservices_realm) 11 | done 12 | 13 | /cnb/process/web -------------------------------------------------------------------------------- /docker-compose/zipkin.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | storage: 4 | image: openzipkin/zipkin-mysql:${ZIPKIN_VERSION-latest} 5 | container_name: mysql 6 | networks: 7 | - ${GLOBAL_NETWORK:-services} 8 | zipkin: 9 | image: openzipkin/zipkin:${ZIPKIN_VERSION-latest} 10 | hostname: zipkin 11 | container_name: zipkin 12 | environment: 13 | - STORAGE_TYPE=mysql 14 | - MYSQL_HOST=mysql 15 | - MYSQL_USER=zipkin 16 | - MYSQL_PASS=zipkin 17 | - JAVA_OPTS=-Dlogging.level.zipkin2=DEBUG 18 | - KAFKA_BOOTSTRAP_SERVERS=kafka-broker-1:9092, kafka-broker-2:9092, kafka-broker-3:9092 19 | ports: 20 | - 9411:9411 21 | depends_on: 22 | - storage 23 | networks: 24 | - ${GLOBAL_NETWORK:-services} 25 | -------------------------------------------------------------------------------- /modules/app-config-data/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../pom.xml 10 | 11 | 4.0.0 12 | 13 | config 14 | 15 | app-config-data 16 | 17 | 18 | UTF-8 19 | 1.7 20 | 1.7 21 | 22 | 23 | 24 | 25 | org.projectlombok 26 | lombok 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/AnalyticsServiceConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | import lombok.Data; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | @Data 8 | @Configuration 9 | @ConfigurationProperties(prefix = "analytics-service") 10 | public class AnalyticsServiceConfigData { 11 | private String version; 12 | private String customAudience; 13 | } -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/ElasticConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | import lombok.Data; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | @Data 8 | @Configuration 9 | @ConfigurationProperties(prefix = "elastic-config") 10 | public class ElasticConfigData { 11 | private String indexName; 12 | private String connectionUrl; 13 | private Integer connectTimeoutMs; 14 | private Integer socketTimeoutMs; 15 | } 16 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/ElasticQueryConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | import lombok.Data; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | @Data 8 | @Configuration 9 | @ConfigurationProperties(prefix = "elastic-query-config") 10 | public class ElasticQueryConfigData { 11 | private String textField; 12 | } 13 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/ElasticQueryServiceConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | import lombok.Data; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | @Data 8 | @Configuration 9 | @ConfigurationProperties(prefix = "elastic-query-service") 10 | public class ElasticQueryServiceConfigData { 11 | private String version; 12 | private String customAudience; 13 | private Long backPressureDelayMs; 14 | private WebClient webClient; 15 | private Query queryFromKafkaStateStore; 16 | private Query queryFromAnalyticsDatabase; 17 | 18 | @Data 19 | public static class WebClient { 20 | private Integer connectTimeoutMs; 21 | private Integer readTimeoutMs; 22 | private Integer writeTimeoutMs; 23 | private Integer maxInMemorySize; 24 | private String contentType; 25 | private String acceptType; 26 | private String queryType; 27 | } 28 | 29 | @Data 30 | public static class Query { 31 | private String method; 32 | private String accept; 33 | private String uri; 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/ElasticQueryWebClientConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | import lombok.Data; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | import java.util.List; 8 | 9 | 10 | @Data 11 | @Configuration 12 | @ConfigurationProperties(prefix = "elastic-query-web-client") 13 | public class ElasticQueryWebClientConfigData { 14 | private WebClient webClient; 15 | 16 | private Query queryByText; 17 | 18 | 19 | @Data 20 | public static class WebClient { 21 | private Integer connectTimeoutMs; 22 | private Integer readTimeoutMs; 23 | private Integer writeTimeoutMs; 24 | private Integer maxInMemorySize; 25 | private String contentType; 26 | private String acceptType; 27 | private String baseUrl; 28 | private String serviceId; 29 | private List instances; 30 | } 31 | 32 | @Data 33 | public static class Query { 34 | private String method; 35 | private String accept; 36 | private String uri; 37 | } 38 | 39 | @Data 40 | public static class Instance{ 41 | private String id; 42 | private String host; 43 | private Integer port; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/GatewayServiceConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | import lombok.Data; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | @Data 8 | @Configuration 9 | @ConfigurationProperties(prefix = "gateway-service") 10 | public class GatewayServiceConfigData { 11 | private Long timeoutMs; 12 | private Float failureRateThreshold; 13 | private Float slowCallRateThreshold; 14 | private Long slowCallDurationThreshold; 15 | private Integer permittedNumOfCallsInHalfOpenState; 16 | private Integer slidingWindowSize; 17 | private Integer minNumberOfCalls; 18 | private Long waitDurationInOpenState; 19 | 20 | } 21 | 22 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/KafkaConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | 4 | import lombok.Data; 5 | import org.springframework.boot.context.properties.ConfigurationProperties; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | import java.util.List; 9 | 10 | @Data 11 | @Configuration 12 | @ConfigurationProperties(prefix = "kafka-config") 13 | public class KafkaConfigData { 14 | private String bootstrapServers; 15 | private String schemaRegistryUrlKey; 16 | private String schemaRegistryUrl; 17 | private String topicName; 18 | private List topicNamesToCreate; 19 | private Integer numOfPartitions; 20 | private Short replicationFactor; 21 | } 22 | 23 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/KafkaConsumerConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | 4 | import lombok.Data; 5 | import org.springframework.boot.context.properties.ConfigurationProperties; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | @Data 9 | @Configuration 10 | @ConfigurationProperties(prefix = "kafka-consumer-config") 11 | public class KafkaConsumerConfigData { 12 | private String keyDeserializer; 13 | private String valueDeserializer; 14 | private String consumerGroupId; 15 | private String autoOffsetReset; 16 | private String specificAvroReaderKey; 17 | private String specificAvroReader; 18 | private Boolean batchListener; 19 | private Boolean autoStartup; 20 | private Integer concurrencyLevel; 21 | private Integer sessionTimeoutMs; 22 | private Integer heartbeatIntervalMs; 23 | private Integer maxPollIntervalMs; 24 | private Integer maxPollRecords; 25 | private Integer maxPartitionFetchBytesDefault; 26 | private Integer maxPartitionFetchBytesBoostFactor; 27 | private Long pollTimeoutMs; 28 | } 29 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/KafkaProducerConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | 4 | import lombok.Data; 5 | import org.springframework.boot.context.properties.ConfigurationProperties; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | 9 | 10 | @Data 11 | @Configuration 12 | @ConfigurationProperties(prefix = "kafka-producer-config") 13 | public class KafkaProducerConfigData { 14 | private String keySerializerClass; 15 | private String valueSerializerClass; 16 | private String compressionType; 17 | private String acks; 18 | private Integer batchSize; 19 | private Integer batchSizeBoostFactor; 20 | private Integer lingerMs; 21 | private Integer requestTimeoutMs; 22 | private Integer retryCount; 23 | } 24 | 25 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/KafkaStreamsConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | import lombok.Data; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | @Data 8 | @Configuration 9 | @ConfigurationProperties(prefix = "kafka-streams-config") 10 | public class KafkaStreamsConfigData { 11 | private String applicationID; 12 | private String inputTopicName; 13 | private String outputTopicName; 14 | private String stateFileLocation; 15 | private String wordCountStoreName; 16 | } 17 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/KafkaStreamsServiceConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | 4 | import lombok.Data; 5 | import org.springframework.boot.context.properties.ConfigurationProperties; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | @Data 9 | @Configuration 10 | @ConfigurationProperties(prefix = "kafka-streams-service") 11 | public class KafkaStreamsServiceConfigData { 12 | private String version; 13 | private String customAudience; 14 | } 15 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/RetryConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | 4 | import lombok.Data; 5 | import org.springframework.boot.context.properties.ConfigurationProperties; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | @Data 9 | @Configuration 10 | @ConfigurationProperties(prefix = "retry-config") 11 | public class RetryConfigData { 12 | 13 | private Long initialIntervalMs; 14 | private Long maxIntervalMs; 15 | private Double multiplier; 16 | private Integer maxAttempts; 17 | private Long sleepTimeMs; 18 | } 19 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/TwitterToKafkaServiceConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | 4 | import lombok.Data; 5 | import org.springframework.boot.context.properties.ConfigurationProperties; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | import java.util.List; 9 | 10 | @Data 11 | @Configuration 12 | @ConfigurationProperties(prefix = "twitter-to-kafka-service") 13 | public class TwitterToKafkaServiceConfigData { 14 | private List twitterKeywords; 15 | private String welcomeMessage; 16 | private Boolean enableMockTweets; 17 | private Long mockSleepMs; 18 | private Integer mockMinTweetLength; 19 | private Integer mockMaxTweetLength; 20 | } 21 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/java/com.microservices.config/UserConfigData.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config; 2 | 3 | import lombok.Data; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | @Data 8 | @Configuration 9 | @ConfigurationProperties(prefix = "user-config") 10 | public class UserConfigData { 11 | private String username; 12 | private String password; 13 | private String[] roles; 14 | } 15 | -------------------------------------------------------------------------------- /modules/app-config-data/src/main/resources/logback-common.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{yyyy-MM-dd HH:mm:ss} [%thread] [%X{X-B3-TraceId:-},%X{X-B3-SpanId:-}] %-5level %X{correlationID} %logger{36} - %msg%n 8 | 9 | 10 | 11 | 12 | 14 | ${DEV_HOME}/${APP_NAME}.log 15 | 16 | 17 | %d{yyyy-MM-dd HH:mm:ss} [%thread] [%X{X-B3-TraceId:-},%X{X-B3-SpanId:-}] %-5level %X{correlationID} %logger{36} - %msg%n 18 | 19 | 20 | 21 | 22 | ${DEV_HOME}/archived/${APP_NAME}-log.%d{yyyy-MM-dd}.%i.log 23 | 24 | 10MB 25 | 60 26 | 20GB 27 | 28 | 29 | 30 | 31 | ${DEV_HOME}/logstash/${APP_NAME}.log 32 | 33 | 34 | ${DEV_HOME}/logstash/archived/${APP_NAME}-log.%d{yyyy-MM-dd}.%i.log 35 | 36 | 10MB 37 | 60 38 | 20GB 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /modules/common-config/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../pom.xml 10 | 11 | 4.0.0 12 | 13 | common-config 14 | 15 | 16 | 17 | com.microservices 18 | config 19 | 20 | 21 | org.springframework.retry 22 | spring-retry 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-aop 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /modules/common-config/src/main/java/com/microservices/common/config/RetryConfig.java: -------------------------------------------------------------------------------- 1 | package com.microservices.common.config; 2 | 3 | import com.microservices.config.RetryConfigData; 4 | import org.springframework.context.annotation.Bean; 5 | import org.springframework.context.annotation.Configuration; 6 | import org.springframework.retry.backoff.ExponentialBackOffPolicy; 7 | import org.springframework.retry.policy.SimpleRetryPolicy; 8 | import org.springframework.retry.support.RetryTemplate; 9 | @Configuration 10 | public class RetryConfig { 11 | 12 | private final RetryConfigData retryConfigData; 13 | 14 | public RetryConfig(RetryConfigData configData) { 15 | this.retryConfigData = configData; 16 | } 17 | 18 | @Bean 19 | public RetryTemplate retryTemplate() { 20 | RetryTemplate retryTemplate = new RetryTemplate(); 21 | 22 | ExponentialBackOffPolicy exponentialBackOffPolicy = new ExponentialBackOffPolicy(); 23 | exponentialBackOffPolicy.setInitialInterval(retryConfigData.getInitialIntervalMs()); 24 | exponentialBackOffPolicy.setMaxInterval(retryConfigData.getMaxIntervalMs()); 25 | exponentialBackOffPolicy.setMultiplier(retryConfigData.getMultiplier()); 26 | 27 | retryTemplate.setBackOffPolicy(exponentialBackOffPolicy); 28 | 29 | SimpleRetryPolicy simpleRetryPolicy = new SimpleRetryPolicy(); 30 | simpleRetryPolicy.setMaxAttempts(retryConfigData.getMaxAttempts()); 31 | 32 | retryTemplate.setRetryPolicy(simpleRetryPolicy); 33 | 34 | return retryTemplate; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /modules/common-util/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../pom.xml 10 | 11 | 4.0.0 12 | 13 | common-util 14 | 15 | common-util 16 | 17 | 18 | -------------------------------------------------------------------------------- /modules/common-util/src/main/java/com/microservices/common/util/CollectionsUtil.java: -------------------------------------------------------------------------------- 1 | package com.microservices.common.util; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | public class CollectionsUtil { 7 | 8 | private CollectionsUtil(){ 9 | 10 | } 11 | 12 | private static class CollectionsUtilHolder { 13 | static final CollectionsUtil INSTANCE = new CollectionsUtil(); 14 | } 15 | 16 | public static CollectionsUtil getInstance(){ 17 | return CollectionsUtilHolder.INSTANCE; 18 | } 19 | 20 | public List getListFromIterable(Iterable iterable){ 21 | List list = new ArrayList<>(); 22 | iterable.forEach(list::add); 23 | return list; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /modules/elastic/elastic-config/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../../pom.xml 10 | 11 | 4.0.0 12 | 13 | elastic-config 14 | 15 | elastic-config 16 | 17 | 18 | UTF-8 19 | 1.7 20 | 1.7 21 | 22 | 23 | 24 | 25 | com.microservices 26 | config 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-data-elasticsearch 31 | 32 | 33 | org.springframework 34 | spring-web 35 | 36 | 37 | org.elasticsearch 38 | elasticsearch 39 | 40 | 41 | org.elasticsearch.client 42 | elasticsearch-rest-high-level-client 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /modules/elastic/elastic-config/src/main/java/com/microservices/elastic/config/ElasticsearchConfig.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.config; 2 | 3 | 4 | import com.microservices.config.ElasticConfigData; 5 | import org.apache.http.HttpHost; 6 | import org.elasticsearch.client.RestClient; 7 | import org.elasticsearch.client.RestHighLevelClient; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.Configuration; 10 | import org.springframework.data.elasticsearch.config.AbstractElasticsearchConfiguration; 11 | import org.springframework.data.elasticsearch.core.ElasticsearchOperations; 12 | import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; 13 | import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; 14 | import org.springframework.web.util.UriComponents; 15 | import org.springframework.web.util.UriComponentsBuilder; 16 | 17 | import java.util.Objects; 18 | 19 | @Configuration 20 | @EnableElasticsearchRepositories(basePackages = "com.microservices.elastic") 21 | public class ElasticsearchConfig extends AbstractElasticsearchConfiguration { 22 | 23 | private final ElasticConfigData elasticConfigData; 24 | 25 | public ElasticsearchConfig(ElasticConfigData configData) { 26 | this.elasticConfigData = configData; 27 | } 28 | 29 | @Override 30 | @Bean 31 | public RestHighLevelClient elasticsearchClient() { 32 | UriComponents serverUri = UriComponentsBuilder.fromHttpUrl(elasticConfigData.getConnectionUrl()).build(); 33 | return new RestHighLevelClient( 34 | RestClient.builder(new HttpHost( 35 | Objects.requireNonNull(serverUri.getHost()), 36 | serverUri.getPort(), 37 | serverUri.getScheme() 38 | )).setRequestConfigCallback( 39 | requestConfigBuilder -> 40 | requestConfigBuilder 41 | .setConnectTimeout(elasticConfigData.getConnectTimeoutMs()) 42 | .setSocketTimeout(elasticConfigData.getSocketTimeoutMs()) 43 | ) 44 | ); 45 | } 46 | 47 | @Bean 48 | public ElasticsearchOperations elasticsearchOperations() { 49 | return new ElasticsearchRestTemplate(elasticsearchClient()); 50 | } 51 | 52 | } -------------------------------------------------------------------------------- /modules/elastic/elastic-index-client/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../../pom.xml 10 | 11 | 4.0.0 12 | 13 | elastic-index-client 14 | 15 | elastic-index-client 16 | 17 | 18 | UTF-8 19 | 1.7 20 | 1.7 21 | 22 | 23 | 24 | 25 | com.microservices 26 | config 27 | 28 | 29 | com.microservices 30 | elastic-model 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /modules/elastic/elastic-index-client/src/main/java/com/microservices/elastic/index/client/repository/TwitterElasticsearchIndexRepository.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.index.client.repository; 2 | 3 | import com.microservices.elastic.model.index.impl.TwitterIndexModel; 4 | import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; 5 | import org.springframework.stereotype.Repository; 6 | 7 | @Repository 8 | public interface TwitterElasticsearchIndexRepository extends ElasticsearchRepository { 9 | } 10 | 11 | -------------------------------------------------------------------------------- /modules/elastic/elastic-index-client/src/main/java/com/microservices/elastic/index/client/service/ElasticIndexClient.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.index.client.service; 2 | 3 | import com.microservices.elastic.model.index.IndexModel; 4 | 5 | import java.util.List; 6 | 7 | public interface ElasticIndexClient { 8 | List save(List documents); 9 | } 10 | -------------------------------------------------------------------------------- /modules/elastic/elastic-index-client/src/main/java/com/microservices/elastic/index/client/service/impl/TwitterElasticIndexClient.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.index.client.service.impl; 2 | 3 | import com.microservices.config.ElasticConfigData; 4 | import com.microservices.elastic.index.client.service.ElasticIndexClient; 5 | import com.microservices.elastic.index.client.util.ElasticIndexUtil; 6 | import com.microservices.elastic.model.index.impl.TwitterIndexModel; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 10 | import org.springframework.data.elasticsearch.core.ElasticsearchOperations; 11 | import org.springframework.data.elasticsearch.core.IndexedObjectInformation; 12 | import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; 13 | import org.springframework.data.elasticsearch.core.query.IndexQuery; 14 | import org.springframework.stereotype.Service; 15 | 16 | import java.util.List; 17 | import java.util.stream.Collectors; 18 | 19 | // DOES THE SAME OF TWITTERELASTICREPOSITORYINDEXCLIENT CLASS BUT WITH MORE CONTROL ON THE QUERY BUILD BECAUSE WE SAY AT LOW LEVEL HOW TO MAP THE IDs WITH THE UTIL CLASS ELASTICINDEXUTIL. 20 | /* 21 | a.) ElasticsearchRepository provides convenient methods like save, find etc. 22 | 23 | b.) ElasticsearchOperations gives chance of running low level queries like elasticsearch bool, must etc. 24 | 25 | c.) ElasticsearchOperations requires to convert the input object to query objects 26 | */ 27 | @Service 28 | @ConditionalOnProperty(name = "elastic-config.is-repository", havingValue = "false") 29 | public class TwitterElasticIndexClient implements ElasticIndexClient { 30 | 31 | private static final Logger LOG = LoggerFactory.getLogger(TwitterElasticIndexClient.class); 32 | 33 | private final ElasticConfigData elasticConfigData; 34 | 35 | private final ElasticsearchOperations elasticsearchOperations; 36 | 37 | private final ElasticIndexUtil elasticIndexUtil; 38 | 39 | public TwitterElasticIndexClient(ElasticConfigData configData, 40 | ElasticsearchOperations elasticOperations, 41 | ElasticIndexUtil indexUtil) { 42 | this.elasticConfigData = configData; 43 | this.elasticsearchOperations = elasticOperations; 44 | this.elasticIndexUtil = indexUtil; 45 | } 46 | 47 | @Override 48 | public List save(List documents) { 49 | List indexQueries = elasticIndexUtil.getIndexQueries(documents); 50 | List documentIds = elasticsearchOperations.bulkIndex( 51 | indexQueries, 52 | IndexCoordinates.of(elasticConfigData.getIndexName()) 53 | ).stream().map(IndexedObjectInformation::getId).collect(Collectors.toList()); 54 | LOG.info("Documents indexed successfully with type: {} and ids: {}", TwitterIndexModel.class.getName(), 55 | documentIds); 56 | return documentIds; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /modules/elastic/elastic-index-client/src/main/java/com/microservices/elastic/index/client/service/impl/TwitterElasticRepositoryIndexClient.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.index.client.service.impl; 2 | import com.microservices.elastic.index.client.repository.TwitterElasticsearchIndexRepository; 3 | import com.microservices.elastic.index.client.service.ElasticIndexClient; 4 | import com.microservices.elastic.model.index.impl.TwitterIndexModel; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 8 | import org.springframework.stereotype.Service; 9 | 10 | import java.util.List; 11 | import java.util.stream.Collectors; 12 | // DOES THE SAME OF TWITTERELASTICINDEXCLIENT CLASS BUT WITH LESS CONTROL ON THE QUERY BUILD 13 | /* 14 | a.) ElasticsearchRepository provides convenient methods like save, find etc. 15 | 16 | b.) ElasticsearchOperations gives chance of running low level queries like elasticsearch bool, must etc. 17 | 18 | c.) ElasticsearchOperations requires to convert the input object to query objects 19 | */ 20 | 21 | @Service 22 | @ConditionalOnProperty(name = "elastic-config.is-repository", havingValue = "true", matchIfMissing = true) 23 | public class TwitterElasticRepositoryIndexClient implements ElasticIndexClient { 24 | 25 | private static final Logger LOG = LoggerFactory.getLogger(TwitterElasticRepositoryIndexClient.class); 26 | 27 | private final TwitterElasticsearchIndexRepository twitterElasticsearchIndexRepository; 28 | 29 | public TwitterElasticRepositoryIndexClient(TwitterElasticsearchIndexRepository indexRepository) { 30 | this.twitterElasticsearchIndexRepository = indexRepository; 31 | } 32 | 33 | @Override 34 | public List save(List documents) { 35 | List repositoryResponse = 36 | (List) twitterElasticsearchIndexRepository.saveAll(documents); 37 | List ids = repositoryResponse.stream().map(TwitterIndexModel::getId).collect(Collectors.toList()); 38 | LOG.info("Documents indexed successfully with type: {} and ids: {}", TwitterIndexModel.class.getName(), ids); 39 | return ids; 40 | } 41 | } -------------------------------------------------------------------------------- /modules/elastic/elastic-index-client/src/main/java/com/microservices/elastic/index/client/util/ElasticIndexUtil.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.index.client.util; 2 | 3 | 4 | import com.microservices.elastic.model.index.IndexModel; 5 | import org.springframework.data.elasticsearch.core.query.IndexQuery; 6 | import org.springframework.data.elasticsearch.core.query.IndexQueryBuilder; 7 | import org.springframework.stereotype.Component; 8 | 9 | import java.util.List; 10 | import java.util.stream.Collectors; 11 | 12 | @Component 13 | public class ElasticIndexUtil { 14 | 15 | public List getIndexQueries(List documents) { 16 | return documents.stream() 17 | .map(document -> new IndexQueryBuilder() 18 | .withId(document.getId()) 19 | .withObject(document) 20 | .build() 21 | ).collect(Collectors.toList()); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /modules/elastic/elastic-model/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../../pom.xml 10 | 11 | 4.0.0 12 | 13 | elastic-model 14 | 15 | elastic-model 16 | 17 | 18 | UTF-8 19 | 1.7 20 | 1.7 21 | 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-data-elasticsearch 27 | 28 | 29 | org.projectlombok 30 | lombok 31 | ${project-lombok.version} 32 | 33 | 34 | org.projectlombok 35 | lombok 36 | compile 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /modules/elastic/elastic-model/src/main/java/com/microservices/elastic/model/index/IndexModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.model.index; 2 | 3 | public interface IndexModel { 4 | String getId(); 5 | } 6 | -------------------------------------------------------------------------------- /modules/elastic/elastic-model/src/main/java/com/microservices/elastic/model/index/impl/TwitterIndexModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.model.index.impl; 2 | 3 | import com.fasterxml.jackson.annotation.JsonFormat; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.microservices.elastic.model.index.IndexModel; 6 | import lombok.Builder; 7 | import lombok.Data; 8 | import org.springframework.data.elasticsearch.annotations.Document; 9 | import org.springframework.data.elasticsearch.annotations.Field; 10 | import org.springframework.data.elasticsearch.annotations.FieldType; 11 | 12 | import java.time.ZonedDateTime; 13 | 14 | @Data 15 | @Builder 16 | @Document(indexName = "#{@elasticConfigData.indexName}") 17 | public class TwitterIndexModel implements IndexModel { 18 | 19 | @JsonProperty 20 | private String id; 21 | @JsonProperty 22 | private Long userId; 23 | @JsonProperty 24 | private String text; 25 | 26 | @Field(type = FieldType.Date, format = {}, pattern = "uuuu-MM-dd'T'HH:mm:ssZZ") 27 | @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "uuuu-MM-dd'T'HH:mm:ssZZ") 28 | @JsonProperty 29 | private ZonedDateTime createdAt; 30 | } -------------------------------------------------------------------------------- /modules/elastic/elastic-query-client/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../../pom.xml 10 | 11 | 4.0.0 12 | 13 | elastic-query-client 14 | 15 | elastic-query-client 16 | 17 | 18 | 19 | 20 | org.springframework.boot 21 | spring-boot-starter-validation 22 | 23 | 24 | com.microservices 25 | config 26 | 27 | 28 | com.microservices 29 | common-util 30 | 31 | 32 | com.microservices 33 | elastic-config 34 | 35 | 36 | com.microservices 37 | elastic-model 38 | 39 | 40 | org.springframework 41 | spring-context 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /modules/elastic/elastic-query-client/src/main/java/com/microservices/elastic/query/client/exception/ElasticQueryClientException.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.client.exception; 2 | 3 | public class ElasticQueryClientException extends RuntimeException { 4 | 5 | public ElasticQueryClientException() { 6 | super(); 7 | } 8 | 9 | public ElasticQueryClientException(String message) { 10 | super(message); 11 | } 12 | 13 | public ElasticQueryClientException(String message, Throwable t) { 14 | super(message, t); 15 | 16 | } 17 | } -------------------------------------------------------------------------------- /modules/elastic/elastic-query-client/src/main/java/com/microservices/elastic/query/client/repository/TwitterElasticsearchQueryRepository.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.client.repository; 2 | 3 | import com.microservices.elastic.model.index.impl.TwitterIndexModel; 4 | import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; 5 | import org.springframework.stereotype.Repository; 6 | 7 | import java.util.List; 8 | 9 | @Repository 10 | public interface TwitterElasticsearchQueryRepository extends ElasticsearchRepository { 11 | 12 | List findByText(String text); 13 | } 14 | -------------------------------------------------------------------------------- /modules/elastic/elastic-query-client/src/main/java/com/microservices/elastic/query/client/repository/impl/TwitterElasticRepositoryQueryClient.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.client.repository.impl; 2 | 3 | import com.microservices.common.util.CollectionsUtil; 4 | import com.microservices.elastic.model.index.impl.TwitterIndexModel; 5 | import com.microservices.elastic.query.client.exception.ElasticQueryClientException; 6 | import com.microservices.elastic.query.client.repository.TwitterElasticsearchQueryRepository; 7 | import com.microservices.elastic.query.client.service.ElasticQueryClient; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.context.annotation.Primary; 11 | import org.springframework.stereotype.Service; 12 | 13 | import java.util.List; 14 | import java.util.Optional; 15 | /* 16 | Repository is a high level client that offers predefined methods to query against the fields on the elasticsearch. 17 | But it cannot be used to send complex queries to elasticsearch. 18 | You have less control with the repository implementation compared to ElasticQueryClient. 19 | */ 20 | 21 | @Primary 22 | @Service 23 | public class TwitterElasticRepositoryQueryClient implements ElasticQueryClient { 24 | private static final Logger LOG = LoggerFactory.getLogger(TwitterElasticRepositoryQueryClient.class); 25 | 26 | private final TwitterElasticsearchQueryRepository twitterElasticsearchQueryRepository; 27 | 28 | public TwitterElasticRepositoryQueryClient(TwitterElasticsearchQueryRepository repository) { 29 | this.twitterElasticsearchQueryRepository = repository; 30 | } 31 | 32 | @Override 33 | public TwitterIndexModel getIndexModelById(String id) { 34 | Optional searchResult = twitterElasticsearchQueryRepository.findById(id); 35 | LOG.info("Document with id {} retrieved successfully", 36 | searchResult.orElseThrow(() -> 37 | new ElasticQueryClientException("No document found at elasticsearch with id " + id)).getId()); 38 | return searchResult.get(); 39 | } 40 | 41 | @Override 42 | public List getIndexModelByText(String text) { 43 | List searchResult = twitterElasticsearchQueryRepository.findByText(text); 44 | LOG.info("{} of documents with text {} retrieved successfully", searchResult.size(), text); 45 | return searchResult; 46 | } 47 | 48 | @Override 49 | public List getAllIndexModels() { 50 | List searchResult = 51 | CollectionsUtil.getInstance().getListFromIterable(twitterElasticsearchQueryRepository.findAll()); 52 | LOG.info("{} number of documents retrieved successfully", searchResult.size()); 53 | return searchResult; 54 | } 55 | } 56 | 57 | -------------------------------------------------------------------------------- /modules/elastic/elastic-query-client/src/main/java/com/microservices/elastic/query/client/service/ElasticQueryClient.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.client.service; 2 | 3 | import com.microservices.elastic.model.index.IndexModel; 4 | 5 | import java.util.List; 6 | 7 | public interface ElasticQueryClient { 8 | 9 | T getIndexModelById(String id); 10 | 11 | List getIndexModelByText(String text); 12 | 13 | List getAllIndexModels(); 14 | } 15 | -------------------------------------------------------------------------------- /modules/elastic/elastic-query-client/src/main/java/com/microservices/elastic/query/client/util/ElasticQueryUtil.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.client.util; 2 | 3 | import com.microservices.elastic.model.index.IndexModel; 4 | import org.elasticsearch.index.query.BoolQueryBuilder; 5 | import org.elasticsearch.index.query.QueryBuilders; 6 | import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; 7 | import org.springframework.data.elasticsearch.core.query.Query; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.util.Collections; 11 | 12 | @Component 13 | public class ElasticQueryUtil { 14 | 15 | public Query getSearchQueryById(String id) { 16 | return new NativeSearchQueryBuilder() 17 | .withIds(Collections.singleton(id)) 18 | .build(); 19 | } 20 | 21 | public Query getSearchQueryByFieldText(String field, String text) { 22 | return new NativeSearchQueryBuilder() 23 | .withQuery(new BoolQueryBuilder() 24 | .must(QueryBuilders.matchQuery( 25 | field, 26 | text 27 | ))) 28 | .build(); 29 | 30 | } 31 | 32 | public Query getSearchQueryForAll() { 33 | return new NativeSearchQueryBuilder() 34 | .withQuery(new BoolQueryBuilder() 35 | .must( 36 | QueryBuilders.matchAllQuery() 37 | )) 38 | .build(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /modules/elastic/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../pom.xml 10 | 11 | 4.0.0 12 | 13 | elastic 14 | pom 15 | 16 | elastic-model 17 | elastic-config 18 | elastic-index-client 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /modules/kafka/kafka-admin/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../../pom.xml 10 | 11 | 4.0.0 12 | 13 | kafka-admin 14 | 15 | kafka-admin 16 | 17 | 18 | 19 | 20 | com.microservices 21 | config 22 | 23 | 24 | com.microservices 25 | common-config 26 | 27 | 28 | org.springframework.kafka 29 | spring-kafka 30 | 31 | 32 | org.springframework.boot 33 | spring-boot-starter-webflux 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /modules/kafka/kafka-admin/src/main/java/com/microservices/kafka/admin/config/KafkaAdminConfig.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.admin.config; 2 | 3 | 4 | import com.microservices.config.KafkaConfigData; 5 | import org.apache.kafka.clients.CommonClientConfigs; 6 | import org.apache.kafka.clients.admin.AdminClient; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.retry.annotation.EnableRetry; 10 | 11 | import java.util.Map; 12 | 13 | @EnableRetry 14 | @Configuration 15 | public class KafkaAdminConfig { 16 | 17 | private final KafkaConfigData kafkaConfigData; 18 | 19 | public KafkaAdminConfig(KafkaConfigData configData) { 20 | this.kafkaConfigData = configData; 21 | } 22 | 23 | @Bean 24 | public AdminClient adminClient() { 25 | return AdminClient.create(Map.of(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, 26 | kafkaConfigData.getBootstrapServers())); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /modules/kafka/kafka-admin/src/main/java/com/microservices/kafka/admin/config/WebClientConfig.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.admin.config; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.web.reactive.function.client.WebClient; 6 | 7 | @Configuration 8 | public class WebClientConfig { 9 | 10 | @Bean 11 | WebClient webClient() { 12 | return WebClient.builder().build(); 13 | } 14 | } 15 | 16 | -------------------------------------------------------------------------------- /modules/kafka/kafka-admin/src/main/java/com/microservices/kafka/admin/exception/KafkaClientException.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.admin.exception; 2 | 3 | public class KafkaClientException extends RuntimeException{ 4 | 5 | public KafkaClientException(){} 6 | 7 | public KafkaClientException(String message){ 8 | super(message); 9 | } 10 | 11 | public KafkaClientException(String message, Throwable cause) { 12 | super(message, cause); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /modules/kafka/kafka-consumer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../../pom.xml 10 | 11 | 4.0.0 12 | 13 | kafka-consumer 14 | 15 | 16 | 17 | com.microservices 18 | config 19 | 20 | 21 | org.springframework.kafka 22 | spring-kafka 23 | 24 | 25 | org.apache.avro 26 | avro 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /modules/kafka/kafka-model/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../../pom.xml 10 | 11 | 4.0.0 12 | 13 | kafka-model 14 | 15 | 16 | 17 | org.apache.avro 18 | avro 19 | 20 | 21 | 22 | 23 | 24 | 25 | org.apache.avro 26 | avro-maven-plugin 27 | ${avro.version} 28 | 29 | String 30 | 31 | 32 | 33 | generate-sources 34 | 35 | schema 36 | 37 | 38 | src/main/resources/avro 39 | src/main/java 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /modules/kafka/kafka-model/src/main/resources/avro/twitter-analytics.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "com.microservices.kafka.avro.model", 2 | "type": "record", 3 | "name": "TwitterAnalyticsAvroModel", 4 | "fields": [ 5 | {"name": "word", "type": ["null", "string"]}, 6 | {"name": "wordCount", "type": ["null", "long"]}, 7 | {"name": "createdAt", "type": ["null", "long"], "logicalType": ["null", "date"]} 8 | ] 9 | } -------------------------------------------------------------------------------- /modules/kafka/kafka-model/src/main/resources/avro/twitter.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.microservices.kafka.avro.model", 3 | "type": "record", 4 | "name": "TwitterAvroModel", 5 | "fields": [ 6 | {"name": "userId", "type": "long"}, 7 | {"name": "id", "type": "long"}, 8 | {"name": "text", "type": ["null", "string"]}, 9 | {"name": "createdAt", "type": ["null", "long"], "logicalType": ["null", "date"]} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /modules/kafka/kafka-producer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../../pom.xml 10 | 11 | 4.0.0 12 | 13 | kafka-producer 14 | 15 | kafka-producer 16 | 17 | 18 | 19 | 20 | com.microservices 21 | config 22 | 23 | 24 | com.microservices 25 | kafka-model 26 | 27 | 28 | org.springframework.kafka 29 | spring-kafka 30 | 31 | 32 | io.confluent 33 | kafka-avro-serializer 34 | 35 | 36 | javax.annotation 37 | javax.annotation-api 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /modules/kafka/kafka-producer/src/main/java/com/microservices/kafka/producer/config/KafkaProducerConfig.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.producer.config; 2 | 3 | import com.microservices.config.KafkaConfigData; 4 | import com.microservices.config.KafkaProducerConfigData; 5 | import org.apache.avro.specific.SpecificRecordBase; 6 | import org.apache.kafka.clients.producer.ProducerConfig; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.kafka.core.DefaultKafkaProducerFactory; 10 | import org.springframework.kafka.core.KafkaTemplate; 11 | import org.springframework.kafka.core.ProducerFactory; 12 | 13 | import java.io.Serializable; 14 | import java.util.HashMap; 15 | import java.util.Map; 16 | 17 | @Configuration 18 | public class KafkaProducerConfig { 19 | 20 | private final KafkaConfigData kafkaConfigData; 21 | 22 | private final KafkaProducerConfigData kafkaProducerConfigData; 23 | 24 | public KafkaProducerConfig(KafkaConfigData configData, KafkaProducerConfigData producerConfigData) { 25 | this.kafkaConfigData = configData; 26 | this.kafkaProducerConfigData = producerConfigData; 27 | } 28 | 29 | @Bean 30 | public Map producerConfig() { 31 | Map props = new HashMap<>(); 32 | props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaConfigData.getBootstrapServers()); 33 | props.put(kafkaConfigData.getSchemaRegistryUrlKey(), kafkaConfigData.getSchemaRegistryUrl()); 34 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, kafkaProducerConfigData.getKeySerializerClass()); 35 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, kafkaProducerConfigData.getValueSerializerClass()); 36 | props.put(ProducerConfig.BATCH_SIZE_CONFIG, kafkaProducerConfigData.getBatchSize() * 37 | kafkaProducerConfigData.getBatchSizeBoostFactor()); 38 | props.put(ProducerConfig.LINGER_MS_CONFIG, kafkaProducerConfigData.getLingerMs()); 39 | props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, kafkaProducerConfigData.getCompressionType()); 40 | props.put(ProducerConfig.ACKS_CONFIG, kafkaProducerConfigData.getAcks()); 41 | props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, kafkaProducerConfigData.getRequestTimeoutMs()); 42 | props.put(ProducerConfig.RETRIES_CONFIG, kafkaProducerConfigData.getRetryCount()); 43 | return props; 44 | } 45 | 46 | @Bean 47 | public ProducerFactory producerFactory() { 48 | return new DefaultKafkaProducerFactory<>(producerConfig()); 49 | } 50 | 51 | @Bean 52 | public KafkaTemplate kafkaTemplate() { 53 | return new KafkaTemplate<>(producerFactory()); 54 | } 55 | } 56 | 57 | -------------------------------------------------------------------------------- /modules/kafka/kafka-producer/src/main/java/com/microservices/kafka/producer/config/service/KafkaProducer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.producer.config.service; 2 | 3 | import org.apache.avro.specific.SpecificRecordBase; 4 | 5 | import java.io.Serializable; 6 | 7 | public interface KafkaProducer { 8 | void send(String topicName, K key, V message); 9 | } 10 | -------------------------------------------------------------------------------- /modules/kafka/kafka-producer/src/main/java/com/microservices/kafka/producer/config/service/impl/TwitterKafkaProducer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.producer.config.service.impl; 2 | 3 | import org.apache.kafka.clients.producer.RecordMetadata; 4 | 5 | import com.microservices.kafka.avro.model.TwitterAvroModel; 6 | import com.microservices.kafka.producer.config.service.KafkaProducer; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.kafka.core.KafkaTemplate; 10 | import org.springframework.kafka.support.SendResult; 11 | import org.springframework.stereotype.Service; 12 | import org.springframework.util.concurrent.ListenableFuture; 13 | import org.springframework.util.concurrent.ListenableFutureCallback; 14 | 15 | import javax.annotation.PreDestroy; 16 | 17 | @Service 18 | public class TwitterKafkaProducer implements KafkaProducer { 19 | 20 | private static final Logger LOG = LoggerFactory.getLogger(TwitterKafkaProducer.class); 21 | 22 | private KafkaTemplate kafkaTemplate; 23 | 24 | public TwitterKafkaProducer(KafkaTemplate template) { 25 | this.kafkaTemplate = template; 26 | } 27 | 28 | @Override 29 | public void send(String topicName, Long key, TwitterAvroModel message) { 30 | LOG.info("Sending message='{}' to topic='{}'", message, topicName); 31 | ListenableFuture> kafkaResultFuture = 32 | kafkaTemplate.send(topicName, key, message); 33 | addCallback(topicName, message, kafkaResultFuture); 34 | } 35 | 36 | @PreDestroy 37 | public void close() { 38 | if (kafkaTemplate != null) { 39 | LOG.info("Closing kafka producer!"); 40 | kafkaTemplate.destroy(); 41 | } 42 | } 43 | 44 | private void addCallback(String topicName, TwitterAvroModel message, 45 | ListenableFuture> kafkaResultFuture) { 46 | kafkaResultFuture.addCallback(new ListenableFutureCallback<>() { 47 | @Override 48 | public void onFailure(Throwable throwable) { 49 | LOG.error("Error while sending message {} to topic {}", message.toString(), topicName, throwable); 50 | } 51 | 52 | @Override 53 | public void onSuccess(SendResult result) { 54 | RecordMetadata metadata = result.getRecordMetadata(); 55 | LOG.debug("Received new metadata. Topic: {}; Partition {}; Offset {}; Timestamp {}, at time {}", 56 | metadata.topic(), 57 | metadata.partition(), 58 | metadata.offset(), 59 | metadata.timestamp(), 60 | System.nanoTime()); 61 | } 62 | }); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /modules/kafka/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../pom.xml 10 | 11 | 4.0.0 12 | 13 | kafka 14 | pom 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /modules/mdc-interceptor/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../pom.xml 10 | 11 | 4.0.0 12 | 0.0.1-SNAPSHOT 13 | 14 | mdc-interceptor 15 | 16 | 17 | 18 | org.springframework 19 | spring-context 20 | compile 21 | 22 | 23 | org.springframework 24 | spring-webmvc 25 | compile 26 | 27 | 28 | javax.servlet 29 | javax.servlet-api 30 | compile 31 | 32 | 33 | org.slf4j 34 | slf4j-api 35 | compile 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /modules/mdc-interceptor/src/main/java/com/microservices/mdc/Constants.java: -------------------------------------------------------------------------------- 1 | package com.microservices.mdc; 2 | 3 | public class Constants { 4 | public static final String CORRELATION_ID_HEADER = "X-Correlation-ID"; 5 | public static final String CORRELATION_ID_KEY = "correlationID"; 6 | } 7 | -------------------------------------------------------------------------------- /modules/mdc-interceptor/src/main/java/com/microservices/mdc/config/IdGeneratorConfig.java: -------------------------------------------------------------------------------- 1 | package com.microservices.mdc.config; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.util.IdGenerator; 6 | import org.springframework.util.JdkIdGenerator; 7 | 8 | @Configuration 9 | public class IdGeneratorConfig { 10 | 11 | @Bean 12 | public IdGenerator idGenerator() { 13 | return new JdkIdGenerator(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /modules/mdc-interceptor/src/main/java/com/microservices/mdc/config/WebMvcConfig.java: -------------------------------------------------------------------------------- 1 | package com.microservices.mdc.config; 2 | 3 | import com.microservices.mdc.interceptor.MDCHandlerInterceptor; 4 | import org.springframework.stereotype.Component; 5 | import org.springframework.web.servlet.config.annotation.InterceptorRegistry; 6 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; 7 | 8 | @Component 9 | public class WebMvcConfig implements WebMvcConfigurer { 10 | 11 | private final MDCHandlerInterceptor mdcHandlerInterceptor; 12 | 13 | public WebMvcConfig(MDCHandlerInterceptor handlerInterceptor) { 14 | this.mdcHandlerInterceptor = handlerInterceptor; 15 | } 16 | 17 | @Override 18 | public void addInterceptors(InterceptorRegistry registry) { 19 | registry.addInterceptor(mdcHandlerInterceptor); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /modules/mdc-interceptor/src/main/java/com/microservices/mdc/interceptor/MDCHandlerInterceptor.java: -------------------------------------------------------------------------------- 1 | package com.microservices.mdc.interceptor; 2 | 3 | import org.slf4j.MDC; 4 | import org.springframework.stereotype.Component; 5 | import org.springframework.util.IdGenerator; 6 | import org.springframework.util.StringUtils; 7 | import org.springframework.web.servlet.HandlerInterceptor; 8 | 9 | import javax.servlet.http.HttpServletRequest; 10 | import javax.servlet.http.HttpServletResponse; 11 | 12 | import static com.microservices.mdc.Constants.CORRELATION_ID_HEADER; 13 | import static com.microservices.mdc.Constants.CORRELATION_ID_KEY; 14 | 15 | @Component 16 | public class MDCHandlerInterceptor implements HandlerInterceptor { 17 | 18 | private final IdGenerator idGenerator; 19 | 20 | public MDCHandlerInterceptor(IdGenerator generator) { 21 | this.idGenerator = generator; 22 | } 23 | 24 | @Override 25 | public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { 26 | String correlationId = request.getHeader(CORRELATION_ID_HEADER); 27 | if (StringUtils.hasLength(correlationId)) { 28 | MDC.put(CORRELATION_ID_KEY, correlationId); 29 | } else { 30 | MDC.put(CORRELATION_ID_KEY, getNewCorrelationId()); 31 | } 32 | return true; 33 | } 34 | 35 | @Override 36 | public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) { 37 | MDC.remove(CORRELATION_ID_KEY); 38 | } 39 | 40 | private String getNewCorrelationId() { 41 | return idGenerator.generateId().toString(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/AnalyticsApplication.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.cloud.client.discovery.EnableDiscoveryClient; 6 | import org.springframework.context.annotation.ComponentScan; 7 | 8 | @EnableDiscoveryClient 9 | @SpringBootApplication 10 | @ComponentScan(basePackages = {"com.microservices"}) 11 | public class AnalyticsApplication { 12 | 13 | public static void main(String[] args) { 14 | SpringApplication.run(AnalyticsApplication.class, args); 15 | } 16 | } -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/Constants.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service; 2 | 3 | public class Constants { 4 | public static final String NA = "N/A"; 5 | } 6 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/api/AnalyticsController.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.api; 2 | 3 | import com.microservices.analytics.service.business.AnalyticsService; 4 | import com.microservices.analytics.service.model.AnalyticsResponseModel; 5 | import io.swagger.v3.oas.annotations.Operation; 6 | import io.swagger.v3.oas.annotations.media.Content; 7 | import io.swagger.v3.oas.annotations.media.Schema; 8 | import io.swagger.v3.oas.annotations.responses.ApiResponse; 9 | import io.swagger.v3.oas.annotations.responses.ApiResponses; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.springframework.http.ResponseEntity; 13 | import org.springframework.security.access.prepost.PreAuthorize; 14 | import org.springframework.web.bind.annotation.*; 15 | 16 | import javax.validation.constraints.NotEmpty; 17 | import java.util.Optional; 18 | 19 | @PreAuthorize("isAuthenticated()") 20 | @RestController 21 | @RequestMapping(value = "/", produces = "application/vnd.api.v1+json") 22 | public class AnalyticsController { 23 | 24 | private static final Logger LOG = LoggerFactory.getLogger(AnalyticsController.class); 25 | 26 | private final AnalyticsService analyticsService; 27 | 28 | public AnalyticsController(AnalyticsService service) { 29 | this.analyticsService = service; 30 | } 31 | 32 | @GetMapping("/get-word-count-by-word/{word}") 33 | @Operation(summary = "Get analytics by word.") 34 | @ApiResponses(value = { 35 | @ApiResponse(responseCode = "200", description = "Success.", content = { 36 | @Content(mediaType = "application/vnd.api.v1+json", 37 | schema = @Schema(implementation = AnalyticsResponseModel.class)) 38 | }), 39 | @ApiResponse(responseCode = "400", description = "Not found."), 40 | @ApiResponse(responseCode = "500", description = "Unexpected error.")}) 41 | public @ResponseBody 42 | ResponseEntity getWordCountByWord(@PathVariable @NotEmpty String word) { 43 | Optional response = analyticsService.getWordAnalytics(word); 44 | if (response.isPresent()) { 45 | LOG.info("Analytics data returned with id {}", response.get().getId()); 46 | return ResponseEntity.ok(response.get()); 47 | } 48 | return ResponseEntity.ok(AnalyticsResponseModel.builder().build()); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/business/AnalyticsService.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.business; 2 | 3 | import com.microservices.analytics.service.model.AnalyticsResponseModel; 4 | 5 | import java.util.Optional; 6 | 7 | public interface AnalyticsService { 8 | Optional getWordAnalytics(String word); 9 | } 10 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/business/KafkaConsumer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.business; 2 | 3 | import org.apache.avro.specific.SpecificRecordBase; 4 | 5 | import java.util.List; 6 | 7 | public interface KafkaConsumer { 8 | void receive(List messages, List keys, List partitions, List offsets); 9 | } 10 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/business/impl/TwitterAnalyticsService.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.business.impl; 2 | 3 | import com.microservices.analytics.service.business.AnalyticsService; 4 | import com.microservices.analytics.service.dataaccess.repository.AnalyticsRepository; 5 | import com.microservices.analytics.service.model.AnalyticsResponseModel; 6 | import com.microservices.analytics.service.transformer.EntityToResponseModelTransformer; 7 | import org.springframework.data.domain.PageRequest; 8 | import org.springframework.stereotype.Service; 9 | 10 | import java.util.Optional; 11 | 12 | @Service 13 | public class TwitterAnalyticsService implements AnalyticsService { 14 | 15 | private final AnalyticsRepository analyticsRepository; 16 | private final EntityToResponseModelTransformer entityToResponseModelTransformer; 17 | 18 | public TwitterAnalyticsService(AnalyticsRepository analyticsRepository, EntityToResponseModelTransformer entityToResponseModelTransformer) { 19 | this.analyticsRepository = analyticsRepository; 20 | this.entityToResponseModelTransformer = entityToResponseModelTransformer; 21 | } 22 | 23 | @Override 24 | public Optional getWordAnalytics(String word) { 25 | return entityToResponseModelTransformer.getResponseModel( 26 | analyticsRepository.getAnalyticsEntitiesByWord(word, PageRequest.of(0, 1)) 27 | .stream().findFirst().orElse(null)); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/dataaccess/entity/AnalyticsEntity.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.dataaccess.entity; 2 | 3 | import lombok.*; 4 | 5 | import javax.persistence.*; 6 | import javax.validation.constraints.NotNull; 7 | import java.time.LocalDateTime; 8 | import java.util.Objects; 9 | import java.util.UUID; 10 | 11 | 12 | @Getter 13 | @Setter 14 | @NoArgsConstructor 15 | @AllArgsConstructor 16 | @Entity 17 | @Table(name = "twitter_analytics") 18 | public class AnalyticsEntity implements BaseEntity { 19 | 20 | @Id 21 | /* 22 | Auto-generated columns require to obtain the id during insert operation and because of that it will break the batch operation 23 | */ 24 | @Column(name = "id", columnDefinition = "uuid") 25 | private UUID id; 26 | 27 | @NotNull 28 | @Column(name = "word") 29 | private String word; 30 | 31 | @NotNull 32 | @Column(name = "word_count") 33 | private Long wordCount; 34 | 35 | @NotNull 36 | @Column(name = "record_date") 37 | private LocalDateTime recordDate; 38 | 39 | @Override 40 | public boolean equals(Object o) { 41 | if (this == o) return true; 42 | if (o == null || getClass() != o.getClass()) return false; 43 | AnalyticsEntity that = (AnalyticsEntity) o; 44 | return Objects.equals(id, that.id); 45 | } 46 | 47 | @Override 48 | public int hashCode() { 49 | return Objects.hash(id); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/dataaccess/entity/BaseEntity.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.dataaccess.entity; 2 | 3 | public interface BaseEntity{ 4 | PK getId(); 5 | } 6 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/dataaccess/repository/AnalyticsCustomRepository.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.dataaccess.repository; 2 | 3 | import java.util.Collection; 4 | 5 | /** 6 | * Spring Data Customized CRUD Repository for entity. 7 | * 8 | * @param 9 | */ 10 | public interface AnalyticsCustomRepository { 11 | 12 | PK persist(S entity); 13 | 14 | void batchPersist(Collection entities); 15 | 16 | S merge(S entity); 17 | 18 | void batchMerge(Collection entities); 19 | 20 | void clear(); 21 | } 22 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/dataaccess/repository/AnalyticsRepository.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.dataaccess.repository; 2 | 3 | import com.microservices.analytics.service.dataaccess.entity.AnalyticsEntity; 4 | import org.springframework.data.domain.Pageable; 5 | import org.springframework.data.jpa.repository.JpaRepository; 6 | import org.springframework.data.jpa.repository.Query; 7 | import org.springframework.data.repository.query.Param; 8 | 9 | import java.util.List; 10 | import java.util.UUID; 11 | 12 | public interface AnalyticsRepository extends JpaRepository, 13 | AnalyticsCustomRepository { 14 | 15 | @Query(value = "select e from AnalyticsEntity e where e.word=:word order by e.recordDate desc") 16 | List getAnalyticsEntitiesByWord(@Param("word") String word, Pageable pageable); 17 | } 18 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/dataaccess/repository/impl/AnalyticsRepositoryImpl.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.dataaccess.repository.impl; 2 | 3 | import com.microservices.analytics.service.dataaccess.entity.BaseEntity; 4 | import com.microservices.analytics.service.dataaccess.repository.AnalyticsCustomRepository; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.stereotype.Repository; 9 | 10 | import javax.persistence.EntityManager; 11 | import javax.persistence.PersistenceContext; 12 | import javax.transaction.Transactional; 13 | import java.util.Collection; 14 | 15 | @Repository 16 | public class AnalyticsRepositoryImpl, PK> implements AnalyticsCustomRepository { 17 | 18 | private static final Logger LOG = LoggerFactory.getLogger(AnalyticsRepositoryImpl.class); 19 | 20 | @PersistenceContext 21 | protected EntityManager em; 22 | 23 | @Value("${spring.jpa.properties.hibernate.jdbc.batch_size:50}") 24 | protected int batchSize; 25 | 26 | 27 | @Override 28 | @Transactional 29 | public PK persist(S entity) { 30 | this.em.persist(entity); 31 | return entity.getId(); 32 | } 33 | 34 | @Override 35 | @Transactional 36 | public void batchPersist(Collection entities) { 37 | if (entities.isEmpty()) { 38 | LOG.info("No entity found to insert!"); 39 | return; 40 | } 41 | int batchCnt = 0; 42 | for (S entity : entities) { 43 | LOG.trace("Persisting entity with id {}", entity.getId()); 44 | this.em.persist(entity); 45 | batchCnt++; 46 | if (batchCnt % batchSize == 0) { 47 | this.em.flush(); 48 | this.em.clear(); 49 | } 50 | } 51 | if (batchCnt % batchSize != 0) { 52 | this.em.flush(); 53 | this.em.clear(); 54 | } 55 | } 56 | 57 | @Override 58 | @Transactional 59 | public S merge(S entity) { 60 | return this.em.merge(entity); 61 | } 62 | 63 | @Override 64 | @Transactional 65 | public void batchMerge(Collection entities) { 66 | if (entities.isEmpty()) { 67 | LOG.info("No entity found to insert!"); 68 | return; 69 | } 70 | int batchCnt = 0; 71 | for (S entity : entities) { 72 | LOG.trace("Merging entity with id {}", entity.getId()); 73 | this.em.merge(entity); 74 | batchCnt++; 75 | if (batchCnt % batchSize == 0) { 76 | this.em.flush(); 77 | this.em.clear(); 78 | } 79 | } 80 | if (batchCnt % batchSize != 0) { 81 | this.em.flush(); 82 | this.em.clear(); 83 | } 84 | } 85 | 86 | @Override 87 | public void clear() { 88 | 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/model/AnalyticsResponseModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | import java.util.UUID; 9 | 10 | @Data 11 | @NoArgsConstructor 12 | @AllArgsConstructor 13 | @Builder 14 | public class AnalyticsResponseModel { 15 | private UUID id; 16 | private String word; 17 | private long wordCount; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/security/AnalyticsUser.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.security; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import org.springframework.security.core.GrantedAuthority; 6 | import org.springframework.security.core.userdetails.UserDetails; 7 | 8 | import java.util.Collection; 9 | 10 | import static com.microservices.analytics.service.Constants.NA; 11 | 12 | 13 | @Builder 14 | @Getter 15 | public class AnalyticsUser implements UserDetails { 16 | 17 | private String username; 18 | 19 | private Collection authorities; 20 | 21 | public void setAuthorities(Collection authorities) { 22 | this.authorities = authorities; 23 | } 24 | 25 | @Override 26 | public Collection getAuthorities() { 27 | return authorities; 28 | } 29 | 30 | @Override 31 | public String getPassword() { 32 | return NA; 33 | } 34 | 35 | @Override 36 | public String getUsername() { 37 | return username; 38 | } 39 | 40 | @Override 41 | public boolean isAccountNonExpired() { 42 | return true; 43 | } 44 | 45 | @Override 46 | public boolean isAccountNonLocked() { 47 | return true; 48 | } 49 | 50 | @Override 51 | public boolean isCredentialsNonExpired() { 52 | return true; 53 | } 54 | 55 | @Override 56 | public boolean isEnabled() { 57 | return true; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/security/AnalyticsUserDetailsService.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.security; 2 | 3 | import org.springframework.security.core.userdetails.UserDetails; 4 | import org.springframework.security.core.userdetails.UserDetailsService; 5 | import org.springframework.security.core.userdetails.UsernameNotFoundException; 6 | import org.springframework.stereotype.Service; 7 | 8 | @Service 9 | public class AnalyticsUserDetailsService implements UserDetailsService { 10 | 11 | @Override 12 | public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException { 13 | return AnalyticsUser.builder() 14 | .username(username) 15 | .build(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/security/AudienceValidator.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.security; 2 | 3 | import com.microservices.config.AnalyticsServiceConfigData; 4 | import org.springframework.beans.factory.annotation.Qualifier; 5 | import org.springframework.security.oauth2.core.OAuth2Error; 6 | import org.springframework.security.oauth2.core.OAuth2TokenValidator; 7 | import org.springframework.security.oauth2.core.OAuth2TokenValidatorResult; 8 | import org.springframework.security.oauth2.jwt.Jwt; 9 | import org.springframework.stereotype.Component; 10 | 11 | @Qualifier(value = "analytics-service-audience-validator") 12 | @Component 13 | public class AudienceValidator implements OAuth2TokenValidator { 14 | 15 | private final AnalyticsServiceConfigData analyticsServiceConfigData; 16 | 17 | public AudienceValidator(AnalyticsServiceConfigData config) { 18 | analyticsServiceConfigData = config; 19 | } 20 | 21 | public OAuth2TokenValidatorResult validate(Jwt jwt) { 22 | if (jwt.getAudience().contains(analyticsServiceConfigData.getCustomAudience())) { 23 | return OAuth2TokenValidatorResult.success(); 24 | } else { 25 | OAuth2Error audienceError = 26 | new OAuth2Error("invalid_token", "The required audience " + 27 | analyticsServiceConfigData.getCustomAudience() + " is missing!", 28 | null); 29 | return OAuth2TokenValidatorResult.failure(audienceError); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/transformer/AvroToDbEntityModelTransformer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.transformer; 2 | 3 | import com.microservices.analytics.service.dataaccess.entity.AnalyticsEntity; 4 | import com.microservices.kafka.avro.model.TwitterAnalyticsAvroModel; 5 | import org.springframework.stereotype.Component; 6 | 7 | import java.time.Instant; 8 | import java.time.LocalDateTime; 9 | import java.time.ZoneOffset; 10 | import java.util.List; 11 | import java.util.UUID; 12 | 13 | import static java.util.stream.Collectors.toList; 14 | 15 | @Component 16 | public class AvroToDbEntityModelTransformer { 17 | 18 | 19 | 20 | public List getEntityModel(List avroModels) { 21 | return avroModels.stream() 22 | .map(avroModel -> new AnalyticsEntity( 23 | UUID.randomUUID() 24 | , avroModel.getWord() 25 | , avroModel.getWordCount() 26 | , LocalDateTime.ofInstant(Instant.ofEpochSecond(avroModel.getCreatedAt()), ZoneOffset.UTC))) 27 | .collect(toList()); 28 | } 29 | 30 | 31 | } 32 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/java/com/microservices/analytics/service/transformer/EntityToResponseModelTransformer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.analytics.service.transformer; 2 | 3 | import com.microservices.analytics.service.dataaccess.entity.AnalyticsEntity; 4 | import com.microservices.analytics.service.model.AnalyticsResponseModel; 5 | import org.springframework.stereotype.Component; 6 | 7 | import java.util.Optional; 8 | 9 | @Component 10 | public class EntityToResponseModelTransformer { 11 | 12 | public Optional getResponseModel(AnalyticsEntity twitterAnalyticsEntity) { 13 | if (twitterAnalyticsEntity == null) 14 | return Optional.empty(); 15 | return Optional.ofNullable(AnalyticsResponseModel 16 | .builder() 17 | .id(twitterAnalyticsEntity.getId()) 18 | .word(twitterAnalyticsEntity.getWord()) 19 | .wordCount(twitterAnalyticsEntity.getWordCount()) 20 | .build()); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /services/analytics-service/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | application: 3 | name: analytics-service 4 | profiles: 5 | active: analytics 6 | config: 7 | import: 'configserver:' 8 | cloud: 9 | config: 10 | name: analytics-service, config-client 11 | username: spring_cloud_user 12 | password: '1234' 13 | 14 | eureka: 15 | client: 16 | serviceUrl: 17 | defaultZone: http://discovery-service-1:8761/eureka/,http://discovery-service-2:8762/eureka/ 18 | instance: 19 | lease-expiration-duration-in-seconds: 5 20 | lease-renewal-interval-in-seconds: 2 21 | 22 | log: 23 | app-name: analytics-service -------------------------------------------------------------------------------- /services/analytics-service/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /services/config-server/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../pom.xml 10 | 11 | 4.0.0 12 | 13 | config-server 14 | 15 | 16 | 17 | com.microservices 18 | config 19 | 20 | 21 | org.springframework.cloud 22 | spring-cloud-config-server 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-autoconfigure 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-security 31 | 32 | 33 | 34 | 35 | 36 | 37 | org.springframework.boot 38 | spring-boot-maven-plugin 39 | 40 | 41 | ${project.groupId}/config.server:${project.version} 42 | 43 | 44 | 45 | 46 | install 47 | 48 | build-image 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /services/config-server/src/main/java/com/microservices/config/server/ConfigServer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config.server; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.cloud.config.server.EnableConfigServer; 6 | 7 | @EnableConfigServer 8 | @SpringBootApplication 9 | public class ConfigServer { 10 | public static void main(String[] args) { 11 | SpringApplication.run(ConfigServer.class, args); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /services/config-server/src/main/java/com/microservices/config/server/config/SecurityConfig.java: -------------------------------------------------------------------------------- 1 | package com.microservices.config.server.config; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.security.config.annotation.web.configuration.WebSecurityCustomizer; 6 | 7 | @Configuration 8 | public class SecurityConfig { 9 | // for semplicity skip the control on encrpyt/decrypt and actuator endpoints, NOT ALLOWED IN PRODUCTION! 10 | @Bean 11 | public WebSecurityCustomizer webSecurityCustomizer() { 12 | return (web) -> web.ignoring() 13 | .antMatchers("/actuator/**") 14 | .antMatchers("/encrypt/**") 15 | .antMatchers("/decrypt/**"); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /services/config-server/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 8888 3 | 4 | spring: 5 | application: 6 | name: configserver 7 | cloud: 8 | config: 9 | server: 10 | git: 11 | uri: https://github.com/Armando1514/Event-Driven-Microservices.git 12 | default-label: main 13 | search-paths: config-server-repository 14 | clone-on-start: true 15 | fail-fast: true 16 | security: 17 | user: 18 | name: spring_cloud_user 19 | password: '1234' 20 | 21 | log: 22 | app-name: config-server 23 | 24 | logging: 25 | level: 26 | ROOT: INFO 27 | org.spring.framework.cloud.config: DEBUG -------------------------------------------------------------------------------- /services/config-server/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /services/discovery-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | microservices 7 | com.microservices 8 | 0.0.1-SNAPSHOT 9 | ../../pom.xml 10 | 11 | 12 | 4.0.0 13 | 14 | com.microservices 15 | discovery-service 16 | 0.0.1-SNAPSHOT 17 | 18 | discovery-service 19 | 20 | 21 | 22 | org.springframework.cloud 23 | spring-cloud-starter-netflix-eureka-server 24 | 25 | 26 | javax.servlet 27 | servlet-api 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | org.springframework.boot 37 | spring-boot-maven-plugin 38 | 39 | 40 | ${project.groupId}/discovery.service:${project.version} 41 | 42 | 43 | 44 | 45 | install 46 | 47 | build-image 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /services/discovery-service/src/main/java/com/microservices/discovery/service/ServiceRegistrationAndDiscoveryServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.microservices.discovery.service; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.cloud.netflix.eureka.server.EnableEurekaServer; 6 | 7 | @EnableEurekaServer 8 | @SpringBootApplication 9 | public class ServiceRegistrationAndDiscoveryServiceApplication { 10 | public static void main(String[] args) { 11 | SpringApplication.run(ServiceRegistrationAndDiscoveryServiceApplication.class, args); 12 | } 13 | 14 | } -------------------------------------------------------------------------------- /services/discovery-service/src/main/resources/application-singleserver.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 8761 3 | 4 | spring: 5 | application: 6 | name: eureka-discovery-service 7 | 8 | eureka: 9 | client: 10 | register-with-eureka: false 11 | fetch-registry: false 12 | instance: 13 | appname: service-registry-cluster 14 | 15 | logging: 16 | level: 17 | ROOT: DEBUG 18 | com.netflix.eureka: OFF 19 | com.netflix.discovery: OFF 20 | 21 | log: 22 | app-name: discovery-service -------------------------------------------------------------------------------- /services/discovery-service/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | eureka: 2 | client: 3 | register-with-eureka: true 4 | fetch-registry: true 5 | instance: 6 | appname: service-registry-cluster 7 | server: 8 | enable-self-preservation: false 9 | expected-client-renewal-interval-seconds: 3 10 | eviction-interval-timer-in-ms: 2000 11 | 12 | spring: 13 | cloud: 14 | loadbalancer: 15 | ribbon: 16 | enabled: false 17 | profiles: 18 | active: singleserver 19 | 20 | 21 | --- 22 | server: 23 | port: 8761 24 | spring: 25 | profiles: peer1 26 | eureka: 27 | instance: 28 | hostname: discovery-service-1 29 | metadataMap: 30 | instanceId: ${spring.application.name}:${spring.application.instance_id:${random.value}} 31 | client: 32 | serviceUrl: 33 | defaultZone: http://discovery-service-1:8761/eureka/,http://discovery-service-2:8762/eureka/ 34 | log: 35 | app-name: discovery-service-1 36 | 37 | --- 38 | server: 39 | port: 8762 40 | spring: 41 | profiles: peer2 42 | eureka: 43 | instance: 44 | hostname: discovery-service-2 45 | metadataMap: 46 | instanceId: ${spring.application.name}:${spring.application.instance_id:${random.value}} 47 | client: 48 | serviceUrl: 49 | defaultZone: http://discovery-service-2:8762/eureka/,http://discovery-service-1:8761/eureka/ 50 | log: 51 | app-name: discovery-service-2 52 | -------------------------------------------------------------------------------- /services/discovery-service/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | %d{yyyy-MM-dd HH:mm:ss} %-5level [%thread] %logger{36} - %msg%n 14 | 15 | 16 | 17 | 18 | 20 | ${DEV_HOME}/${APP_NAME}.log 21 | 22 | 23 | %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n 24 | 25 | 26 | 27 | 28 | ${DEV_HOME}/archived/${APP_NAME}-log.%d{yyyy-MM-dd}.%i.log 29 | 30 | 32 | 10MB 33 | 34 | 35 | 36 | 37 | 38 | ${DEV_HOME}/logstash/${APP_NAME}.log 39 | 40 | 41 | ${DEV_HOME}/logstash/archived/${APP_NAME}-log.%d{yyyy-MM-dd}.%i.log 42 | 43 | 45 | 10MB 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/Constants.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service; 2 | 3 | public class Constants { 4 | public static final String NA = "N/A"; 5 | } 6 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/ElasticQueryServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service; 2 | 3 | 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.cloud.client.discovery.EnableDiscoveryClient; 7 | import org.springframework.context.annotation.ComponentScan; 8 | @EnableDiscoveryClient 9 | @SpringBootApplication 10 | @ComponentScan(basePackages = "com.microservices") 11 | public class ElasticQueryServiceApplication { 12 | public static void main(String[] args) { 13 | SpringApplication.run(ElasticQueryServiceApplication.class, args); 14 | } 15 | } -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/QueryType.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service; 2 | 3 | 4 | public enum QueryType { 5 | 6 | KAFKA_STATE_STORE("KAFKA_STATE_STORE"), ANALYTICS_DATABASE("ANALYTICS_DATABASE"); 7 | 8 | private String type; 9 | 10 | QueryType(String type) { 11 | this.type = type; 12 | } 13 | 14 | public String getType() { 15 | return type; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/api/error/handler/ElasticQueryServiceErrorHandler.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.api.error.handler; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.http.HttpStatus; 6 | import org.springframework.http.ResponseEntity; 7 | import org.springframework.security.access.AccessDeniedException; 8 | import org.springframework.validation.FieldError; 9 | import org.springframework.web.bind.MethodArgumentNotValidException; 10 | import org.springframework.web.bind.annotation.ControllerAdvice; 11 | import org.springframework.web.bind.annotation.ExceptionHandler; 12 | 13 | import java.util.HashMap; 14 | import java.util.Map; 15 | 16 | @ControllerAdvice 17 | public class ElasticQueryServiceErrorHandler { 18 | 19 | private static final Logger LOG = LoggerFactory.getLogger(ElasticQueryServiceErrorHandler.class); 20 | 21 | @ExceptionHandler(AccessDeniedException.class) 22 | public ResponseEntity handle(AccessDeniedException e) { 23 | LOG.error("Access denied exception!", e); 24 | return ResponseEntity.status(HttpStatus.FORBIDDEN).body("You are not authorized to access this resource"); 25 | } 26 | 27 | @ExceptionHandler(IllegalArgumentException.class) 28 | public ResponseEntity handle(IllegalArgumentException e) { 29 | LOG.error("Illegal number exception!", e); 30 | return ResponseEntity.badRequest().body("Illegal argument exception: " + e.getMessage()); 31 | } 32 | 33 | @ExceptionHandler(RuntimeException.class) 34 | public ResponseEntity handle(RuntimeException e) { 35 | LOG.error("Service runtime exception!", e); 36 | return ResponseEntity.badRequest().body("Service runtime exception: " + e.getMessage()); 37 | } 38 | 39 | @ExceptionHandler(Exception.class) 40 | public ResponseEntity handle(Exception e) { 41 | LOG.error("Internal error!", e); 42 | return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Internal error exception."); 43 | } 44 | 45 | @ExceptionHandler(MethodArgumentNotValidException.class) 46 | public ResponseEntity> handle(MethodArgumentNotValidException e) { 47 | LOG.error("Method argument validation exception", e); 48 | Map errors = new HashMap<>(); 49 | e.getBindingResult().getAllErrors().forEach(error -> 50 | errors.put(((FieldError) error).getField(), error.getDefaultMessage() )); 51 | return ResponseEntity.badRequest().body(errors); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/business/ElasticQueryService.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.business; 2 | 3 | import com.microservices.query.service.model.ElasticQueryServiceAnalyticsResponseModel; 4 | import com.microservices.query.service.model.ElasticQueryServiceResponseModel; 5 | 6 | import java.util.List; 7 | 8 | public interface ElasticQueryService { 9 | 10 | ElasticQueryServiceResponseModel getDocumentById(String id); 11 | 12 | ElasticQueryServiceAnalyticsResponseModel getDocumentByText(String text, String accessToken); 13 | 14 | List getAllDocuments(); 15 | } 16 | 17 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/business/QueryUserService.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.business; 2 | 3 | import com.microservices.query.service.dataaccess.entity.UserPermission; 4 | 5 | import java.util.List; 6 | import java.util.Optional; 7 | 8 | public interface QueryUserService { 9 | 10 | Optional> findAllPermissionsByUsername(String username); 11 | } 12 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/business/impl/TwitterQueryUserService.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.business.impl; 2 | 3 | import com.microservices.query.service.business.QueryUserService; 4 | import com.microservices.query.service.dataaccess.entity.UserPermission; 5 | import com.microservices.query.service.dataaccess.repository.UserPermissionRepository; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.stereotype.Service; 9 | 10 | import java.util.List; 11 | import java.util.Optional; 12 | 13 | @Service 14 | public class TwitterQueryUserService implements QueryUserService { 15 | 16 | private static final Logger LOG = LoggerFactory.getLogger(TwitterQueryUserService.class); 17 | 18 | 19 | private final UserPermissionRepository userPermissionRepository; 20 | 21 | public TwitterQueryUserService(UserPermissionRepository permissionRepository) { 22 | this.userPermissionRepository = permissionRepository; 23 | } 24 | 25 | @Override 26 | public Optional> findAllPermissionsByUsername(String username) { 27 | LOG.info("Finding permissions by username {}", username); 28 | return userPermissionRepository.findPermissionsByUsername(username); 29 | } 30 | } -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/config/WebClientConfig.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.config; 2 | 3 | import com.microservices.config.ElasticQueryServiceConfigData; 4 | import io.netty.channel.ChannelOption; 5 | import io.netty.handler.timeout.ReadTimeoutHandler; 6 | import io.netty.handler.timeout.WriteTimeoutHandler; 7 | import org.apache.http.HttpHeaders; 8 | import org.springframework.cloud.client.loadbalancer.LoadBalanced; 9 | import org.springframework.context.annotation.Bean; 10 | import org.springframework.context.annotation.Configuration; 11 | import org.springframework.http.client.reactive.ReactorClientHttpConnector; 12 | import org.springframework.web.reactive.function.client.WebClient; 13 | import reactor.netty.http.client.HttpClient; 14 | 15 | import java.util.concurrent.TimeUnit; 16 | 17 | @Configuration 18 | public class WebClientConfig { 19 | 20 | private final ElasticQueryServiceConfigData.WebClient elasticQueryServiceConfigData; 21 | 22 | public WebClientConfig(ElasticQueryServiceConfigData queryServiceConfigData) { 23 | this.elasticQueryServiceConfigData = queryServiceConfigData.getWebClient(); 24 | } 25 | 26 | @LoadBalanced 27 | @Bean 28 | WebClient.Builder webClientBuilder() { 29 | return WebClient.builder() 30 | .defaultHeader(HttpHeaders.CONTENT_TYPE, elasticQueryServiceConfigData.getContentType()) 31 | .defaultHeader(HttpHeaders.ACCEPT, elasticQueryServiceConfigData.getAcceptType()) 32 | .clientConnector(new ReactorClientHttpConnector(getHttpClient())) 33 | .codecs(clientCodecConfigurer -> 34 | clientCodecConfigurer 35 | .defaultCodecs() 36 | .maxInMemorySize(elasticQueryServiceConfigData.getMaxInMemorySize())); 37 | } 38 | 39 | private HttpClient getHttpClient() { 40 | return HttpClient.create() 41 | .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, elasticQueryServiceConfigData.getConnectTimeoutMs()) 42 | .doOnConnected(connection -> { 43 | connection.addHandlerLast(new ReadTimeoutHandler(elasticQueryServiceConfigData.getReadTimeoutMs(), 44 | TimeUnit.MILLISECONDS)); 45 | connection.addHandlerLast(new WriteTimeoutHandler(elasticQueryServiceConfigData.getWriteTimeoutMs(), 46 | TimeUnit.MILLISECONDS)); 47 | }); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/dataaccess/entity/UserPermission.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.dataaccess.entity; 2 | 3 | 4 | import lombok.Data; 5 | 6 | 7 | import javax.persistence.Entity; 8 | import javax.persistence.Id; 9 | import javax.validation.constraints.NotNull; 10 | import java.util.UUID; 11 | 12 | 13 | @Entity 14 | @Data 15 | public class UserPermission { 16 | 17 | @NotNull 18 | @Id 19 | private UUID id; 20 | 21 | @NotNull 22 | private String username; 23 | 24 | @NotNull 25 | private String documentId; 26 | 27 | @NotNull 28 | private String permissionType; 29 | } 30 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/dataaccess/repository/UserPermissionRepository.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.dataaccess.repository; 2 | 3 | import com.microservices.query.service.dataaccess.entity.UserPermission; 4 | import org.springframework.data.jpa.repository.JpaRepository; 5 | import org.springframework.data.jpa.repository.Query; 6 | import org.springframework.data.repository.query.Param; 7 | 8 | import java.util.List; 9 | import java.util.Optional; 10 | import java.util.UUID; 11 | 12 | public interface UserPermissionRepository extends JpaRepository { 13 | 14 | @Query(nativeQuery = true, value = 15 | "select p.user_permission_id as id, u.username, d.document_id, p.permission_type " + 16 | "from users u, user_permissions p, documents d " + 17 | "where u.id = p.user_id " + 18 | "and d.id = p.document_id " + 19 | "and u.username = :username") 20 | Optional> findPermissionsByUsername(@Param("username") String username); 21 | } 22 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/model/ElasticQueryServiceAnalyticsResponseModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | import java.util.List; 9 | 10 | @Data 11 | @NoArgsConstructor 12 | @AllArgsConstructor 13 | @Builder 14 | public class ElasticQueryServiceAnalyticsResponseModel { 15 | private List queryResponseModels; 16 | private Long wordCount; 17 | } 18 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/model/ElasticQueryServiceRequestModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.model; 2 | 3 | 4 | import lombok.AllArgsConstructor; 5 | import lombok.Builder; 6 | import lombok.Data; 7 | import lombok.NoArgsConstructor; 8 | 9 | import javax.validation.constraints.NotEmpty; 10 | 11 | @Data 12 | @Builder 13 | @NoArgsConstructor 14 | @AllArgsConstructor 15 | public class ElasticQueryServiceRequestModel { 16 | 17 | private String id; 18 | @NotEmpty 19 | private String text; 20 | } 21 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/model/ElasticQueryServiceResponseModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | import org.springframework.hateoas.RepresentationModel; 8 | 9 | import java.time.ZonedDateTime; 10 | 11 | @Data 12 | @Builder 13 | @NoArgsConstructor 14 | @AllArgsConstructor 15 | public class ElasticQueryServiceResponseModel extends RepresentationModel { 16 | 17 | private String id; 18 | private Long userId; 19 | private String text; 20 | private ZonedDateTime createdAt; 21 | 22 | } 23 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/model/ElasticQueryServiceWordCountResponseModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.model; 2 | 3 | 4 | import lombok.AllArgsConstructor; 5 | import lombok.Builder; 6 | import lombok.Data; 7 | import lombok.NoArgsConstructor; 8 | 9 | @Data 10 | @NoArgsConstructor 11 | @AllArgsConstructor 12 | @Builder 13 | public class ElasticQueryServiceWordCountResponseModel { 14 | private String word; 15 | private Long wordCount; 16 | } 17 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/model/assembler/ElasticQueryServiceResponseModelAssembler.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.model.assembler; 2 | 3 | import com.microservices.elastic.model.index.impl.TwitterIndexModel; 4 | import com.microservices.query.service.api.ElasticDocumentController; 5 | import com.microservices.query.service.model.ElasticQueryServiceResponseModel; 6 | import com.microservices.query.service.transformer.ElasticToResponseModelTransformer; 7 | import org.springframework.hateoas.server.mvc.RepresentationModelAssemblerSupport; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.util.List; 11 | import java.util.stream.Collectors; 12 | 13 | import static org.springframework.hateoas.server.mvc.WebMvcLinkBuilder.linkTo; 14 | import static org.springframework.hateoas.server.mvc.WebMvcLinkBuilder.methodOn; 15 | 16 | @Component 17 | public class ElasticQueryServiceResponseModelAssembler extends RepresentationModelAssemblerSupport { 18 | 19 | private final ElasticToResponseModelTransformer elasticToResponseModelTransformer; 20 | 21 | public ElasticQueryServiceResponseModelAssembler(ElasticToResponseModelTransformer elasticToResponseModelTransformer) { 22 | super(ElasticDocumentController.class, ElasticQueryServiceResponseModel.class); 23 | 24 | this.elasticToResponseModelTransformer = elasticToResponseModelTransformer; 25 | } 26 | 27 | @Override 28 | public ElasticQueryServiceResponseModel toModel(TwitterIndexModel twitterIndexModel) { 29 | ElasticQueryServiceResponseModel responseModel = 30 | elasticToResponseModelTransformer.getResponseModel(twitterIndexModel); 31 | responseModel.add( 32 | linkTo(methodOn(ElasticDocumentController.class) 33 | .getDocumentById((twitterIndexModel.getId()))) 34 | .withSelfRel() 35 | ); 36 | responseModel.add( 37 | linkTo(ElasticDocumentController.class) 38 | .withRel("documents")); 39 | return responseModel; 40 | } 41 | 42 | public List toModels(List twitterIndexModels) { 43 | return twitterIndexModels.stream().map(this::toModel).collect(Collectors.toList()); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/security/AudienceValidator.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.security; 2 | 3 | import com.microservices.config.ElasticQueryServiceConfigData; 4 | import org.springframework.beans.factory.annotation.Qualifier; 5 | import org.springframework.security.oauth2.core.OAuth2Error; 6 | import org.springframework.security.oauth2.core.OAuth2TokenValidator; 7 | import org.springframework.security.oauth2.core.OAuth2TokenValidatorResult; 8 | import org.springframework.security.oauth2.jwt.Jwt; 9 | import org.springframework.stereotype.Component; 10 | 11 | @Qualifier("elastic-query-service-audience-validator") 12 | @Component 13 | public class AudienceValidator implements OAuth2TokenValidator { 14 | 15 | private final ElasticQueryServiceConfigData elasticQueryServiceConfigData; 16 | 17 | public AudienceValidator(ElasticQueryServiceConfigData configData) { 18 | this.elasticQueryServiceConfigData = configData; 19 | } 20 | 21 | @Override 22 | public OAuth2TokenValidatorResult validate(Jwt jwt) { 23 | if (jwt.getAudience().contains(elasticQueryServiceConfigData.getCustomAudience())) { 24 | return OAuth2TokenValidatorResult.success(); 25 | } else { 26 | OAuth2Error audienceError = 27 | new OAuth2Error("invalid_token", "The required audience " + 28 | elasticQueryServiceConfigData.getCustomAudience() + " is missing!", 29 | null); 30 | return OAuth2TokenValidatorResult.failure(audienceError); 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/security/PermissionType.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.security; 2 | 3 | public enum PermissionType { 4 | READ("READ"), WRITE("WRITE"), ADMIN("ADMIN"); 5 | 6 | private String type; 7 | 8 | PermissionType(String type){ 9 | this.type = type; 10 | } 11 | 12 | public String getType() { 13 | return this.type; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/security/TwitterQueryUser.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.security; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import org.springframework.security.core.GrantedAuthority; 6 | import org.springframework.security.core.userdetails.UserDetails; 7 | 8 | import java.util.Collection; 9 | import java.util.Map; 10 | 11 | import static com.microservices.query.service.Constants.NA; 12 | 13 | @Builder 14 | @Getter 15 | public class TwitterQueryUser implements UserDetails { 16 | 17 | private String username; 18 | 19 | private Collection authorities; 20 | 21 | private Map permissions; 22 | 23 | public void setAuthorities(Collection authorities) { 24 | this.authorities = authorities; 25 | } 26 | 27 | @Override 28 | public Collection getAuthorities() { 29 | return authorities; 30 | } 31 | 32 | @Override 33 | public String getPassword() { 34 | return NA; 35 | } 36 | 37 | @Override 38 | public String getUsername() { 39 | return username; 40 | } 41 | 42 | @Override 43 | public boolean isAccountNonExpired() { 44 | return true; 45 | } 46 | 47 | @Override 48 | public boolean isAccountNonLocked() { 49 | return true; 50 | } 51 | 52 | @Override 53 | public boolean isCredentialsNonExpired() { 54 | return true; 55 | } 56 | 57 | @Override 58 | public boolean isEnabled() { 59 | return true; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/security/TwitterQueryUserDetailsService.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.security; 2 | 3 | 4 | import com.microservices.query.service.business.QueryUserService; 5 | import com.microservices.query.service.transformer.UserPermissionsToUserDetailTransformer; 6 | import org.springframework.security.core.userdetails.UserDetails; 7 | import org.springframework.security.core.userdetails.UserDetailsService; 8 | import org.springframework.security.core.userdetails.UsernameNotFoundException; 9 | import org.springframework.stereotype.Service; 10 | 11 | @Service 12 | public class TwitterQueryUserDetailsService implements UserDetailsService { 13 | 14 | private final QueryUserService queryUserService; 15 | 16 | private final UserPermissionsToUserDetailTransformer userPermissionsToUserDetailTransformer; 17 | 18 | public TwitterQueryUserDetailsService(QueryUserService queryUserService, UserPermissionsToUserDetailTransformer userPermissionsToUserDetailTransformer) { 19 | this.queryUserService = queryUserService; 20 | this.userPermissionsToUserDetailTransformer = userPermissionsToUserDetailTransformer; 21 | } 22 | 23 | @Override 24 | public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException { 25 | return queryUserService 26 | .findAllPermissionsByUsername(username) 27 | .map(userPermissionsToUserDetailTransformer::getUserDetails) 28 | .orElseThrow( 29 | () -> new UsernameNotFoundException("No user found with name " + username)); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/transformer/ElasticToResponseModelTransformer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.transformer; 2 | 3 | import com.microservices.elastic.model.index.impl.TwitterIndexModel; 4 | import com.microservices.query.service.model.ElasticQueryServiceResponseModel; 5 | import org.springframework.stereotype.Component; 6 | 7 | import java.util.List; 8 | import java.util.stream.Collectors; 9 | 10 | @Component 11 | public class ElasticToResponseModelTransformer { 12 | 13 | public ElasticQueryServiceResponseModel getResponseModel(TwitterIndexModel twitterIndexModel) { 14 | return ElasticQueryServiceResponseModel 15 | .builder() 16 | .id(twitterIndexModel.getId()) 17 | .userId(twitterIndexModel.getUserId()) 18 | .text(twitterIndexModel.getText()) 19 | .createdAt(twitterIndexModel.getCreatedAt()) 20 | .build(); 21 | } 22 | 23 | public List getResponseModels(List twitterIndexModels) { 24 | return twitterIndexModels.stream().map(this::getResponseModel).collect(Collectors.toList()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/java/com/microservices/query/service/transformer/UserPermissionsToUserDetailTransformer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.query.service.transformer; 2 | 3 | import com.microservices.query.service.dataaccess.entity.UserPermission; 4 | import com.microservices.query.service.security.PermissionType; 5 | import com.microservices.query.service.security.TwitterQueryUser; 6 | import org.springframework.stereotype.Component; 7 | 8 | import java.util.List; 9 | import java.util.stream.Collectors; 10 | 11 | @Component 12 | public class UserPermissionsToUserDetailTransformer { 13 | 14 | public TwitterQueryUser getUserDetails(List userPermissions) { 15 | return TwitterQueryUser.builder() 16 | .username(userPermissions.get(0).getUsername()) 17 | .permissions(userPermissions.stream() 18 | .collect(Collectors.toMap( 19 | UserPermission::getDocumentId, 20 | permission -> PermissionType.valueOf(permission.getPermissionType()) 21 | ))) 22 | .build(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | logging: 2 | level: 3 | ROOT: DEBUG 4 | com.microservices: DEBUG 5 | 6 | spring: 7 | application: 8 | name: elastic-query-service 9 | profiles: 10 | active: elastic_query 11 | config: 12 | import: 'configserver:' 13 | cloud: 14 | config: 15 | name: elastic-query-service,config-client 16 | username: spring_cloud_user 17 | password: '1234' 18 | 19 | eureka: 20 | client: 21 | serviceUrl: 22 | defaultZone: http://discovery-service-1:8761/eureka/,http://discovery-service-2:8762/eureka/ 23 | instance: 24 | lease-expiration-duration-in-seconds: 5 25 | lease-renewal-interval-in-seconds: 2 26 | 27 | 28 | log: 29 | appName: elastic-query-service 30 | 31 | 32 | -------------------------------------------------------------------------------- /services/elastic-query-service/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/java/com/microservices/elastic/query/web/client/ElasticQueryWebClientApplication.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client; 2 | 3 | 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.cloud.client.discovery.EnableDiscoveryClient; 7 | import org.springframework.context.annotation.ComponentScan; 8 | @EnableDiscoveryClient 9 | 10 | @SpringBootApplication 11 | @ComponentScan(basePackages = "com.microservices") 12 | public class ElasticQueryWebClientApplication { 13 | public static void main(String[] args) { 14 | SpringApplication.run(ElasticQueryWebClientApplication.class, args); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/java/com/microservices/elastic/query/web/client/api/QueryController.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client.api; 2 | 3 | import com.microservices.elastic.query.web.client.model.ElasticQueryWebClientAnalyticsResponseModel; 4 | import com.microservices.elastic.query.web.client.model.ElasticQueryWebClientRequestModel; 5 | import com.microservices.elastic.query.web.client.service.ElasticQueryWebClient; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.stereotype.Controller; 9 | import org.springframework.ui.Model; 10 | import org.springframework.web.bind.annotation.GetMapping; 11 | import org.springframework.web.bind.annotation.PostMapping; 12 | 13 | import javax.validation.Valid; 14 | 15 | 16 | @Controller 17 | public class QueryController { 18 | 19 | private static final Logger LOG = LoggerFactory.getLogger(QueryController.class); 20 | 21 | private final ElasticQueryWebClient elasticQueryWebClient; 22 | 23 | public QueryController(ElasticQueryWebClient webClient) { 24 | this.elasticQueryWebClient = webClient; 25 | } 26 | 27 | @GetMapping("") 28 | public String index() { 29 | return "index"; 30 | } 31 | 32 | @GetMapping("/error") 33 | public String error() { 34 | return "error"; 35 | } 36 | 37 | @GetMapping("/home") 38 | public String home(Model model) { 39 | model.addAttribute("elasticQueryWebClientRequestModel", 40 | ElasticQueryWebClientRequestModel.builder().build()); 41 | return "home"; 42 | } 43 | 44 | @PostMapping("/query-by-text") 45 | public String queryByText(@Valid ElasticQueryWebClientRequestModel requestModel, 46 | Model model) { 47 | LOG.info("Querying with text {}", requestModel.getText()); 48 | ElasticQueryWebClientAnalyticsResponseModel responseModel = elasticQueryWebClient.getDataByText(requestModel); 49 | model.addAttribute("elasticQueryWebClientResponseModels", 50 | responseModel.getQueryResponseModels()); 51 | model.addAttribute("wordCount", responseModel.getWordCount()); 52 | model.addAttribute("fallbackMessage", responseModel.getFallbackMessage()); 53 | model.addAttribute("searchText", requestModel.getText()); 54 | model.addAttribute("elasticQueryWebClientRequestModel", 55 | ElasticQueryWebClientRequestModel.builder().build()); 56 | return "home"; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/java/com/microservices/elastic/query/web/client/exception/ElasticQueryWebClientException.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client.exception; 2 | 3 | 4 | public class ElasticQueryWebClientException extends RuntimeException { 5 | 6 | public ElasticQueryWebClientException() { super(); } 7 | 8 | public ElasticQueryWebClientException(String message) { super(message); } 9 | 10 | public ElasticQueryWebClientException(String message, Throwable t) { super(message, t); } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/java/com/microservices/elastic/query/web/client/model/ElasticQueryWebClientAnalyticsResponseModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | import java.util.List; 9 | 10 | @Data 11 | @Builder 12 | @NoArgsConstructor 13 | @AllArgsConstructor 14 | public class ElasticQueryWebClientAnalyticsResponseModel { 15 | private List queryResponseModels; 16 | private Long wordCount; 17 | private String fallbackMessage; 18 | } 19 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/java/com/microservices/elastic/query/web/client/model/ElasticQueryWebClientRequestModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | import javax.validation.constraints.NotEmpty; 9 | 10 | @Data 11 | @Builder 12 | @NoArgsConstructor 13 | @AllArgsConstructor 14 | public class ElasticQueryWebClientRequestModel { 15 | private String id; 16 | @NotEmpty 17 | private String text; 18 | } 19 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/java/com/microservices/elastic/query/web/client/model/ElasticQueryWebClientResponseModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | import java.time.ZonedDateTime; 9 | 10 | @Data 11 | @Builder 12 | @NoArgsConstructor 13 | @AllArgsConstructor 14 | public class ElasticQueryWebClientResponseModel { 15 | private String id; 16 | private Long userId; 17 | private String text; 18 | private ZonedDateTime createdAt; 19 | } 20 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/java/com/microservices/elastic/query/web/client/service/ElasticQueryWebClient.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client.service; 2 | 3 | 4 | 5 | import com.microservices.elastic.query.web.client.model.ElasticQueryWebClientAnalyticsResponseModel; 6 | import com.microservices.elastic.query.web.client.model.ElasticQueryWebClientRequestModel; 7 | 8 | 9 | public interface ElasticQueryWebClient { 10 | ElasticQueryWebClientAnalyticsResponseModel getDataByText(ElasticQueryWebClientRequestModel requestModel); 11 | } 12 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 8188 3 | servlet: 4 | context-path: /elastic-query-web-client 5 | logging: 6 | level: 7 | root: DEBUG 8 | 9 | spring: 10 | application: 11 | name: elastic-query-web-client-2 12 | profiles: 13 | active: elastic_query_web_2 14 | config: 15 | import: 'configserver:' 16 | cloud: 17 | config: 18 | name: elastic-query-web-client-2,config-client 19 | username: spring_cloud_user 20 | password: '1234' 21 | 22 | eureka: 23 | client: 24 | serviceUrl: 25 | defaultZone: http://discovery-service-1:8761/eureka/,http://discovery-service-2:8762/eureka/ 26 | instance: 27 | lease-expiration-duration-in-seconds: 5 28 | lease-renewal-interval-in-seconds: 2 29 | 30 | log: 31 | app-name: elastic-query-web-client-2 32 | 33 | 34 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/resources/templates/error.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 |

Query Client

5 |
6 |

An error occurred

7 |
    8 |
  • 9 |
  • 10 |
11 | Main page 12 |
13 |
14 | 15 |
16 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/resources/templates/footer.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/resources/templates/header.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Query Client 5 | 6 | 7 | 8 | 9 | 10 | 11 | Twitter Search Engine 12 | 13 | 14 | 15 | 16 | 42 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/resources/templates/home.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 |

Query Client

5 |
6 |
7 |
8 | 9 | 10 |
11 | 12 |
13 |
14 |
15 |

16 | 17 |

18 |
19 |

Search results for '' with word count: ''

20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 |
iduser-idtextdate
38 |
39 |
40 | 41 |
42 |
43 |
44 | 45 |
46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /services/elastic-query-web-client-2/src/main/resources/templates/index.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 |
5 |

Please login to start searching

6 |
7 | 8 | 11 |
12 | 13 |
14 | 15 | 16 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/java/com/microservices/elastic/query/web/client/ElasticQueryWebClientApplication.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client; 2 | 3 | 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.cloud.client.discovery.EnableDiscoveryClient; 7 | import org.springframework.context.annotation.ComponentScan; 8 | @EnableDiscoveryClient 9 | 10 | @SpringBootApplication 11 | @ComponentScan(basePackages = "com.microservices") 12 | public class ElasticQueryWebClientApplication { 13 | public static void main(String[] args) { 14 | SpringApplication.run(ElasticQueryWebClientApplication.class, args); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/java/com/microservices/elastic/query/web/client/api/QueryController.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client.api; 2 | 3 | import com.microservices.elastic.query.web.client.model.ElasticQueryWebClientAnalyticsResponseModel; 4 | import com.microservices.elastic.query.web.client.model.ElasticQueryWebClientRequestModel; 5 | import com.microservices.elastic.query.web.client.service.ElasticQueryWebClient; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.stereotype.Controller; 9 | import org.springframework.ui.Model; 10 | import org.springframework.web.bind.annotation.GetMapping; 11 | import org.springframework.web.bind.annotation.PostMapping; 12 | 13 | import javax.validation.Valid; 14 | 15 | 16 | @Controller 17 | public class QueryController { 18 | 19 | private static final Logger LOG = LoggerFactory.getLogger(QueryController.class); 20 | 21 | private final ElasticQueryWebClient elasticQueryWebClient; 22 | 23 | public QueryController(ElasticQueryWebClient webClient) { 24 | this.elasticQueryWebClient = webClient; 25 | } 26 | 27 | @GetMapping("") 28 | public String index() { 29 | return "index"; 30 | } 31 | 32 | @GetMapping("/error") 33 | public String error() { 34 | return "error"; 35 | } 36 | 37 | @GetMapping("/home") 38 | public String home(Model model) { 39 | model.addAttribute("elasticQueryWebClientRequestModel", 40 | ElasticQueryWebClientRequestModel.builder().build()); 41 | return "home"; 42 | } 43 | 44 | @PostMapping("/query-by-text") 45 | public String queryByText(@Valid ElasticQueryWebClientRequestModel requestModel, 46 | Model model) { 47 | LOG.info("Querying with text {}", requestModel.getText()); 48 | ElasticQueryWebClientAnalyticsResponseModel responseModel = elasticQueryWebClient.getDataByText(requestModel); 49 | model.addAttribute("elasticQueryWebClientResponseModels", 50 | responseModel.getQueryResponseModels()); 51 | model.addAttribute("wordCount", responseModel.getWordCount()); 52 | model.addAttribute("fallbackMessage", responseModel.getFallbackMessage()); 53 | model.addAttribute("searchText", requestModel.getText()); 54 | model.addAttribute("elasticQueryWebClientRequestModel", 55 | ElasticQueryWebClientRequestModel.builder().build()); 56 | return "home"; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/java/com/microservices/elastic/query/web/client/exception/ElasticQueryWebClientException.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client.exception; 2 | 3 | 4 | public class ElasticQueryWebClientException extends RuntimeException { 5 | 6 | public ElasticQueryWebClientException() { super(); } 7 | 8 | public ElasticQueryWebClientException(String message) { super(message); } 9 | 10 | public ElasticQueryWebClientException(String message, Throwable t) { super(message, t); } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/java/com/microservices/elastic/query/web/client/model/ElasticQueryWebClientAnalyticsResponseModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | import java.util.List; 9 | 10 | @Data 11 | @Builder 12 | @NoArgsConstructor 13 | @AllArgsConstructor 14 | public class ElasticQueryWebClientAnalyticsResponseModel { 15 | private List queryResponseModels; 16 | private Long wordCount; 17 | private String fallbackMessage; 18 | } 19 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/java/com/microservices/elastic/query/web/client/model/ElasticQueryWebClientRequestModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | import javax.validation.constraints.NotEmpty; 9 | 10 | @Data 11 | @Builder 12 | @NoArgsConstructor 13 | @AllArgsConstructor 14 | public class ElasticQueryWebClientRequestModel { 15 | private String id; 16 | @NotEmpty 17 | private String text; 18 | } 19 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/java/com/microservices/elastic/query/web/client/model/ElasticQueryWebClientResponseModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | import java.time.ZonedDateTime; 9 | 10 | @Data 11 | @Builder 12 | @NoArgsConstructor 13 | @AllArgsConstructor 14 | public class ElasticQueryWebClientResponseModel { 15 | private String id; 16 | private Long userId; 17 | private String text; 18 | private ZonedDateTime createdAt; 19 | } 20 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/java/com/microservices/elastic/query/web/client/service/ElasticQueryWebClient.java: -------------------------------------------------------------------------------- 1 | package com.microservices.elastic.query.web.client.service; 2 | 3 | 4 | 5 | import com.microservices.elastic.query.web.client.model.ElasticQueryWebClientAnalyticsResponseModel; 6 | import com.microservices.elastic.query.web.client.model.ElasticQueryWebClientRequestModel; 7 | 8 | 9 | public interface ElasticQueryWebClient { 10 | ElasticQueryWebClientAnalyticsResponseModel getDataByText(ElasticQueryWebClientRequestModel requestModel); 11 | } 12 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | logging: 2 | level: 3 | root: DEBUG 4 | 5 | spring: 6 | application: 7 | name: elastic-query-web-client 8 | profiles: 9 | active: elastic_query_web 10 | config: 11 | import: 'configserver:' 12 | cloud: 13 | config: 14 | name: elastic-query-web-client,config-client 15 | username: spring_cloud_user 16 | password: '1234' 17 | 18 | eureka: 19 | client: 20 | serviceUrl: 21 | defaultZone: http://discovery-service-1:8761/eureka/,http://discovery-service-2:8762/eureka/ 22 | instance: 23 | lease-expiration-duration-in-seconds: 5 24 | lease-renewal-interval-in-seconds: 2 25 | 26 | log: 27 | app-name: elastic-query-web-client -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/resources/templates/error.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 |

Query Client

5 |
6 |

An error occurred

7 |
    8 |
  • 9 |
  • 10 |
11 | Main page 12 |
13 |
14 | 15 |
16 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/resources/templates/footer.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/resources/templates/header.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Query Client 5 | 6 | 7 | 8 | 9 | 10 | 11 | Twitter Search Engine 12 | 13 | 14 | 15 | 16 | 42 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/resources/templates/home.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 |

Query Client

5 |
6 |
7 |
8 | 9 | 10 |
11 | 12 |
13 |
14 |
15 |

16 | 17 |

18 |
19 |

Search results for '' with word count: ''

20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 |
iduser-idtextdate
38 |
39 |
40 | 41 |
42 |
43 |
44 | 45 |
46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /services/elastic-query-web-client/src/main/resources/templates/index.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 |
5 |

Please login to start searching

6 |
7 | 8 | 11 |
12 | 13 |
14 | 15 | 16 | -------------------------------------------------------------------------------- /services/gateway-service/src/main/java/com/microservices/gateway/service/GatewayServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.microservices.gateway.service; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.cloud.client.discovery.EnableDiscoveryClient; 6 | import org.springframework.context.annotation.ComponentScan; 7 | 8 | @EnableDiscoveryClient 9 | @SpringBootApplication 10 | @ComponentScan(basePackages = "com.microservices") 11 | public class GatewayServiceApplication { 12 | 13 | public static void main(String[] args) { 14 | SpringApplication.run(GatewayServiceApplication.class, args); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /services/gateway-service/src/main/java/com/microservices/gateway/service/config/WebSecurityConfig.java: -------------------------------------------------------------------------------- 1 | package com.microservices.gateway.service.config; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.security.config.web.server.ServerHttpSecurity; 6 | import org.springframework.security.web.server.SecurityWebFilterChain; 7 | 8 | @Configuration 9 | public class WebSecurityConfig { 10 | 11 | @Bean 12 | public SecurityWebFilterChain webFluxSecurityConfig(ServerHttpSecurity httpSecurity) { 13 | httpSecurity.authorizeExchange() 14 | .anyExchange() 15 | .permitAll(); 16 | httpSecurity.csrf().disable(); 17 | return httpSecurity.build(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /services/gateway-service/src/main/java/com/microservices/gateway/service/controller/FallbackController.java: -------------------------------------------------------------------------------- 1 | package com.microservices.gateway.service.controller; 2 | 3 | import com.microservices.gateway.service.model.AnalyticsDataFallbackModel; 4 | import com.microservices.gateway.service.model.QueryServiceFallbackModel; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.http.ResponseEntity; 9 | import org.springframework.web.bind.annotation.PostMapping; 10 | import org.springframework.web.bind.annotation.RequestMapping; 11 | import org.springframework.web.bind.annotation.RestController; 12 | 13 | @RestController 14 | @RequestMapping("/fallback") 15 | public class FallbackController { 16 | 17 | private static final Logger LOG = LoggerFactory.getLogger(FallbackController.class); 18 | 19 | @Value("${server.port}") 20 | private String port; 21 | 22 | @PostMapping("/query-fallback") 23 | public ResponseEntity queryServiceFallback() { 24 | LOG.info("Returning fallback result for elastic-query-service! on port {}", port); 25 | return ResponseEntity.ok(QueryServiceFallbackModel.builder() 26 | .fallbackMessage("Fallback result for elastic-query-service!") 27 | .build()); 28 | } 29 | 30 | @PostMapping("/analytics-fallback") 31 | public ResponseEntity analyticsServiceFallback() { 32 | LOG.info("Returning fallback result for analytics-service! on port {}", port); 33 | return ResponseEntity.ok(AnalyticsDataFallbackModel.builder() 34 | .wordCount(0L) 35 | .build()); 36 | } 37 | 38 | 39 | @PostMapping("/streams-fallback") 40 | public ResponseEntity streamsServiceFallback() { 41 | LOG.info("Returning fallback result for kafka-streams-service! on port {}", port); 42 | return ResponseEntity.ok(AnalyticsDataFallbackModel.builder() 43 | .wordCount(0L) 44 | .build()); 45 | } 46 | 47 | 48 | } 49 | -------------------------------------------------------------------------------- /services/gateway-service/src/main/java/com/microservices/gateway/service/model/AnalyticsDataFallbackModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.gateway.service.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | @Data 9 | @NoArgsConstructor 10 | @AllArgsConstructor 11 | @Builder 12 | public class AnalyticsDataFallbackModel { 13 | private Long wordCount; 14 | } 15 | -------------------------------------------------------------------------------- /services/gateway-service/src/main/java/com/microservices/gateway/service/model/QueryServiceFallbackModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.gateway.service.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | @Data 9 | @NoArgsConstructor 10 | @AllArgsConstructor 11 | @Builder 12 | public class QueryServiceFallbackModel { 13 | private String fallbackMessage; 14 | } 15 | -------------------------------------------------------------------------------- /services/gateway-service/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | logging: 2 | level.root: DEBUG 3 | 4 | server: 5 | port: 9092 6 | 7 | spring: 8 | application: 9 | name: gateway-service 10 | profiles: 11 | active: gateway 12 | config: 13 | import: 'configserver:' 14 | cloud: 15 | config: 16 | name: gateway-service,config-client 17 | username: spring_cloud_user 18 | password: '1234' 19 | 20 | eureka: 21 | client: 22 | serviceUrl: 23 | defaultZone: http://localhost:8761/eureka/ 24 | instance: 25 | lease-expiration-duration-in-seconds: 5 26 | lease-renewal-interval-in-seconds: 2 27 | 28 | log: 29 | app-name: gateway-service -------------------------------------------------------------------------------- /services/gateway-service/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/java/com/microservices/kafka/streams/service/Constants.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.streams.service; 2 | 3 | public class Constants { 4 | public static final String NA = "N/A"; 5 | } 6 | -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/java/com/microservices/kafka/streams/service/KafkaStreamsServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.streams.service; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.boot.CommandLineRunner; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.autoconfigure.SpringBootApplication; 8 | import org.springframework.cloud.client.discovery.EnableDiscoveryClient; 9 | import org.springframework.context.annotation.ComponentScan; 10 | import com.microservices.kafka.streams.service.init.StreamsInitializer; 11 | import com.microservices.kafka.streams.service.runner.StreamsRunner; 12 | 13 | @EnableDiscoveryClient 14 | 15 | @SpringBootApplication 16 | @ComponentScan(basePackages = {"com.microservices"}) 17 | public class KafkaStreamsServiceApplication implements CommandLineRunner { 18 | 19 | private static final Logger LOG = LoggerFactory.getLogger(KafkaStreamsServiceApplication.class); 20 | 21 | private final StreamsRunner streamsRunner; 22 | 23 | private final StreamsInitializer streamsInitializer; 24 | 25 | public KafkaStreamsServiceApplication(StreamsRunner runner, 26 | StreamsInitializer initializer) { 27 | this.streamsRunner = runner; 28 | this.streamsInitializer = initializer; 29 | } 30 | 31 | public static void main(String[] args) { 32 | SpringApplication.run(KafkaStreamsServiceApplication.class, args); 33 | } 34 | 35 | @Override 36 | public void run(String... args) { 37 | LOG.info("App starts..."); 38 | streamsInitializer.init(); 39 | streamsRunner.start(); 40 | } 41 | 42 | } -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/java/com/microservices/kafka/streams/service/api/KafkaStreamsController.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.streams.service.api; 2 | 3 | import com.microservices.kafka.streams.service.model.KafkaStreamsResponseModel; 4 | import com.microservices.kafka.streams.service.runner.StreamsRunner; 5 | import io.swagger.v3.oas.annotations.Operation; 6 | import io.swagger.v3.oas.annotations.media.Content; 7 | import io.swagger.v3.oas.annotations.media.Schema; 8 | import io.swagger.v3.oas.annotations.responses.ApiResponse; 9 | import io.swagger.v3.oas.annotations.responses.ApiResponses; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.springframework.http.ResponseEntity; 13 | import org.springframework.security.access.prepost.PreAuthorize; 14 | import org.springframework.web.bind.annotation.*; 15 | 16 | import javax.validation.constraints.NotEmpty; 17 | 18 | @PreAuthorize("isAuthenticated()") 19 | @RestController 20 | @RequestMapping(value = "/", produces = "application/vnd.api.v1+json") 21 | public class KafkaStreamsController { 22 | 23 | private static final Logger LOG = LoggerFactory.getLogger(KafkaStreamsController.class); 24 | 25 | private final StreamsRunner kafkaStreamsRunner; 26 | 27 | public KafkaStreamsController(StreamsRunner streamsRunner) { 28 | this.kafkaStreamsRunner = streamsRunner; 29 | } 30 | 31 | @GetMapping("get-word-count-by-word/{word}") 32 | @Operation(summary = "Get word count by word.") 33 | @ApiResponses(value = { 34 | @ApiResponse(responseCode = "200", description = "Success.", content = { 35 | @Content(mediaType = "application/vnd.api.v1+json", 36 | schema = @Schema(implementation = KafkaStreamsResponseModel.class)) 37 | }), 38 | @ApiResponse(responseCode = "400", description = "Not found."), 39 | @ApiResponse(responseCode = "500", description = "Unexpected error.")}) 40 | public @ResponseBody 41 | ResponseEntity getWordCountByWord( 42 | @PathVariable @NotEmpty String word) { 43 | Long wordCount = kafkaStreamsRunner.getValueByKey(word); 44 | LOG.info("Word count {} returned for word {}", wordCount, word); 45 | return ResponseEntity.ok(KafkaStreamsResponseModel.builder() 46 | .word(word) 47 | .wordCount(wordCount) 48 | .build()); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/java/com/microservices/kafka/streams/service/config/KafkaStreamsConfig.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.streams.service.config; 2 | 3 | 4 | import com.microservices.config.KafkaConfigData; 5 | import com.microservices.config.KafkaStreamsConfigData; 6 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; 7 | import org.apache.kafka.common.serialization.Serdes; 8 | import org.apache.kafka.streams.StreamsConfig; 9 | import org.springframework.beans.factory.annotation.Qualifier; 10 | import org.springframework.context.annotation.Bean; 11 | import org.springframework.context.annotation.Configuration; 12 | 13 | import java.util.Properties; 14 | 15 | @Configuration 16 | public class KafkaStreamsConfig { 17 | 18 | private final KafkaConfigData kafkaConfigData; 19 | 20 | private final KafkaStreamsConfigData kafkaStreamsConfigData; 21 | 22 | public KafkaStreamsConfig(KafkaConfigData kafkaConfig, 23 | KafkaStreamsConfigData kafkaStreamsConfig) { 24 | this.kafkaConfigData = kafkaConfig; 25 | this.kafkaStreamsConfigData = kafkaStreamsConfig; 26 | } 27 | 28 | 29 | @Bean 30 | @Qualifier("streamConfiguration") 31 | public Properties streamsConfiguration() { 32 | Properties streamsConfiguration = new Properties(); 33 | streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, kafkaStreamsConfigData.getApplicationID()); 34 | streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaConfigData.getBootstrapServers()); 35 | streamsConfiguration.put(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, 36 | kafkaConfigData.getSchemaRegistryUrl()); 37 | streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); 38 | streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); 39 | streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, kafkaStreamsConfigData.getStateFileLocation()); 40 | return streamsConfiguration; 41 | } 42 | 43 | 44 | } 45 | -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/java/com/microservices/kafka/streams/service/init/StreamsInitializer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.streams.service.init; 2 | 3 | public interface StreamsInitializer { 4 | void init(); 5 | } 6 | -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/java/com/microservices/kafka/streams/service/init/impl/KafkaStreamsInitializer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.streams.service.init.impl; 2 | 3 | import com.microservices.config.KafkaConfigData; 4 | import com.microservices.kafka.admin.client.KafkaAdminClient; 5 | import com.microservices.kafka.streams.service.init.StreamsInitializer; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.stereotype.Component; 9 | 10 | @Component 11 | public class KafkaStreamsInitializer implements StreamsInitializer { 12 | 13 | private static final Logger LOG = LoggerFactory.getLogger(KafkaStreamsInitializer.class); 14 | 15 | private final KafkaConfigData kafkaConfigData; 16 | 17 | private final KafkaAdminClient kafkaAdminClient; 18 | 19 | public KafkaStreamsInitializer(KafkaConfigData configData, KafkaAdminClient adminClient) { 20 | this.kafkaConfigData = configData; 21 | this.kafkaAdminClient = adminClient; 22 | } 23 | 24 | @Override 25 | public void init() { 26 | kafkaAdminClient.checkTopicsCreated(); 27 | kafkaAdminClient.checkSchemaRegistry(); 28 | LOG.info("Topics with name {} is ready for operations!", kafkaConfigData.getTopicNamesToCreate().toArray()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/java/com/microservices/kafka/streams/service/model/KafkaStreamsResponseModel.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.streams.service.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | @Data 9 | @NoArgsConstructor 10 | @AllArgsConstructor 11 | @Builder 12 | public class KafkaStreamsResponseModel { 13 | private String word; 14 | private Long wordCount; 15 | } 16 | -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/java/com/microservices/kafka/streams/service/runner/StreamsRunner.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.streams.service.runner; 2 | 3 | public interface StreamsRunner { 4 | void start(); 5 | default V getValueByKey(K key) { 6 | return null; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/java/com/microservices/kafka/streams/service/security/AudienceValidator.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.streams.service.security; 2 | 3 | import com.microservices.config.KafkaStreamsServiceConfigData; 4 | import org.springframework.beans.factory.annotation.Qualifier; 5 | import org.springframework.security.oauth2.core.OAuth2Error; 6 | import org.springframework.security.oauth2.core.OAuth2TokenValidator; 7 | import org.springframework.security.oauth2.core.OAuth2TokenValidatorResult; 8 | import org.springframework.security.oauth2.jwt.Jwt; 9 | import org.springframework.stereotype.Component; 10 | 11 | @Qualifier(value = "kafka-streams-service-audience-validator") 12 | @Component 13 | public class AudienceValidator implements OAuth2TokenValidator { 14 | 15 | private final KafkaStreamsServiceConfigData kafkaStreamsServiceConfig; 16 | 17 | public AudienceValidator(KafkaStreamsServiceConfigData config) { 18 | kafkaStreamsServiceConfig = config; 19 | } 20 | 21 | public OAuth2TokenValidatorResult validate(Jwt jwt) { 22 | if (jwt.getAudience().contains(kafkaStreamsServiceConfig.getCustomAudience())) { 23 | return OAuth2TokenValidatorResult.success(); 24 | } else { 25 | OAuth2Error audienceError = 26 | new OAuth2Error("invalid_token", "The required audience " + 27 | kafkaStreamsServiceConfig.getCustomAudience() + " is missing!", 28 | null); 29 | return OAuth2TokenValidatorResult.failure(audienceError); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/java/com/microservices/kafka/streams/service/security/KafkaStreamsUser.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.streams.service.security; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import org.springframework.security.core.GrantedAuthority; 6 | import org.springframework.security.core.userdetails.UserDetails; 7 | 8 | import java.util.Collection; 9 | 10 | import static com.microservices.kafka.streams.service.Constants.NA; 11 | 12 | 13 | @Builder 14 | @Getter 15 | public class KafkaStreamsUser implements UserDetails { 16 | 17 | private String username; 18 | 19 | private Collection authorities; 20 | 21 | public void setAuthorities(Collection authorities) { 22 | this.authorities = authorities; 23 | } 24 | 25 | @Override 26 | public Collection getAuthorities() { 27 | return authorities; 28 | } 29 | 30 | @Override 31 | public String getPassword() { 32 | return NA; 33 | } 34 | 35 | @Override 36 | public String getUsername() { 37 | return username; 38 | } 39 | 40 | @Override 41 | public boolean isAccountNonExpired() { 42 | return true; 43 | } 44 | 45 | @Override 46 | public boolean isAccountNonLocked() { 47 | return true; 48 | } 49 | 50 | @Override 51 | public boolean isCredentialsNonExpired() { 52 | return true; 53 | } 54 | 55 | @Override 56 | public boolean isEnabled() { 57 | return true; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/java/com/microservices/kafka/streams/service/security/KafkaStreamsUserDetailsService.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.streams.service.security; 2 | 3 | import org.springframework.security.core.userdetails.UserDetails; 4 | import org.springframework.security.core.userdetails.UserDetailsService; 5 | import org.springframework.security.core.userdetails.UsernameNotFoundException; 6 | import org.springframework.stereotype.Service; 7 | 8 | @Service 9 | public class KafkaStreamsUserDetailsService implements UserDetailsService { 10 | 11 | @Override 12 | public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException { 13 | return KafkaStreamsUser.builder() 14 | .username(username) 15 | .build(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | application: 3 | name: kafka-streams-service 4 | profiles: 5 | active: kafka_streams 6 | config: 7 | import: 'configserver:' 8 | cloud: 9 | config: 10 | name: kafka-streams-service,config-client 11 | username: spring_cloud_user 12 | password: '1234' 13 | eureka: 14 | client: 15 | serviceUrl: 16 | defaultZone: http://discovery-service-1:8761/eureka/,http://discovery-service-2:8762/eureka/ 17 | instance: 18 | lease-expiration-duration-in-seconds: 5 19 | lease-renewal-interval-in-seconds: 2 20 | 21 | log: 22 | app-name: kafka-streams-service -------------------------------------------------------------------------------- /services/kafka-streams-service/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /services/kafka-to-elastic-service/src/main/java/com/microservices/kafka/service/KafkaToElasticServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.service; 2 | 3 | 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.context.annotation.ComponentScan; 7 | 8 | @SpringBootApplication 9 | @ComponentScan(basePackages = "com.microservices") 10 | public class KafkaToElasticServiceApplication { 11 | public static void main(String[] args) { 12 | SpringApplication.run(KafkaToElasticServiceApplication.class, args); 13 | } 14 | } 15 | 16 | -------------------------------------------------------------------------------- /services/kafka-to-elastic-service/src/main/java/com/microservices/kafka/service/consumer/KafkaConsumer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.service.consumer; 2 | 3 | import org.apache.avro.specific.SpecificRecordBase; 4 | 5 | import java.util.List; 6 | 7 | 8 | public interface KafkaConsumer { 9 | void receive(List messages, List keys, List partitions, List offsets); 10 | } 11 | -------------------------------------------------------------------------------- /services/kafka-to-elastic-service/src/main/java/com/microservices/kafka/service/transformer/AvroToElasticModelTransformer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.kafka.service.transformer; 2 | 3 | import com.microservices.elastic.model.index.impl.TwitterIndexModel; 4 | import com.microservices.kafka.avro.model.TwitterAvroModel; 5 | import org.springframework.stereotype.Component; 6 | 7 | import java.time.Instant; 8 | import java.time.ZoneId; 9 | import java.time.ZonedDateTime; 10 | import java.util.List; 11 | import java.util.stream.Collectors; 12 | 13 | @Component 14 | public class AvroToElasticModelTransformer { 15 | 16 | public List getElasticModels(List avroModels) { 17 | return avroModels.stream() 18 | .map(avroModel -> TwitterIndexModel 19 | .builder() 20 | .userId(avroModel.getUserId()) 21 | .id(String.valueOf(avroModel.getId())) 22 | .text(avroModel.getText()) 23 | .createdAt(ZonedDateTime.ofInstant(Instant.ofEpochMilli(avroModel.getCreatedAt()), 24 | ZoneId.systemDefault())) 25 | .build() 26 | ).collect(Collectors.toList()); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /services/kafka-to-elastic-service/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | application: 3 | name: kafka-to-elastic-service 4 | profiles: 5 | active: kafka_to_elastic 6 | config: 7 | import: 'configserver:' 8 | cloud: 9 | config: 10 | name: kafka-to-elastic-service,config-client 11 | username: spring_cloud_user 12 | password: '1234' 13 | 14 | log: 15 | app-name: kafka-to-elastic-service 16 | 17 | -------------------------------------------------------------------------------- /services/kafka-to-elastic-service/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /services/twitter-to-kafka-service/src/main/java/com/microservices/TwitterToKafkaServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.microservices; 2 | 3 | 4 | import com.microservices.init.StreamInitializer; 5 | import com.microservices.runner.StreamRunner; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.boot.CommandLineRunner; 9 | import org.springframework.boot.SpringApplication; 10 | import org.springframework.boot.autoconfigure.SpringBootApplication; 11 | import org.springframework.context.annotation.ComponentScan; 12 | @SpringBootApplication 13 | // Component Scan here needed to scan beans in other module, in our case config module. 14 | 15 | @ComponentScan(basePackages = "com.microservices") 16 | public class TwitterToKafkaServiceApplication implements CommandLineRunner { 17 | 18 | private static final Logger LOG = LoggerFactory.getLogger(TwitterToKafkaServiceApplication.class); 19 | 20 | private final StreamRunner streamRunner; 21 | 22 | private final StreamInitializer streamInitializer; 23 | 24 | 25 | public TwitterToKafkaServiceApplication(StreamRunner runner, StreamInitializer initializer) { 26 | this.streamRunner = runner; 27 | this.streamInitializer = initializer; 28 | } 29 | 30 | public static void main(String[] args) { 31 | SpringApplication.run(TwitterToKafkaServiceApplication.class, args); 32 | } 33 | 34 | @Override 35 | public void run(String... args) throws Exception { 36 | LOG.info("App starts..."); 37 | streamInitializer.init(); 38 | streamRunner.start(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /services/twitter-to-kafka-service/src/main/java/com/microservices/init/StreamInitializer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.init; 2 | 3 | public interface StreamInitializer { 4 | void init(); 5 | } 6 | -------------------------------------------------------------------------------- /services/twitter-to-kafka-service/src/main/java/com/microservices/init/impl/KafkaStreamInitializer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.init.impl; 2 | 3 | import com.microservices.config.KafkaConfigData; 4 | import com.microservices.init.StreamInitializer; 5 | import com.microservices.kafka.admin.client.KafkaAdminClient; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.stereotype.Component; 9 | 10 | @Component 11 | public class KafkaStreamInitializer implements StreamInitializer { 12 | 13 | private static final Logger LOG = LoggerFactory.getLogger(KafkaStreamInitializer.class); 14 | 15 | private final KafkaConfigData kafkaConfigData; 16 | 17 | private final KafkaAdminClient kafkaAdminClient; 18 | 19 | public KafkaStreamInitializer(KafkaConfigData configData, KafkaAdminClient adminClient) { 20 | this.kafkaConfigData = configData; 21 | this.kafkaAdminClient = adminClient; 22 | } 23 | 24 | @Override 25 | public void init() { 26 | kafkaAdminClient.createTopics(); 27 | kafkaAdminClient.checkSchemaRegistry(); 28 | LOG.info("Topics with name {} is ready for operations!", kafkaConfigData.getTopicNamesToCreate().toArray()); 29 | } 30 | } 31 | 32 | 33 | -------------------------------------------------------------------------------- /services/twitter-to-kafka-service/src/main/java/com/microservices/listener/TwitterKafkaStatusListener.java: -------------------------------------------------------------------------------- 1 | package com.microservices.listener; 2 | 3 | import com.microservices.config.KafkaConfigData; 4 | import com.microservices.kafka.avro.model.TwitterAvroModel; 5 | import com.microservices.kafka.producer.config.service.KafkaProducer; 6 | import com.microservices.transformer.TwitterStatusToAvroTransformer; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.stereotype.Component; 10 | import twitter4j.Status; 11 | import twitter4j.StatusAdapter; 12 | 13 | 14 | @Component 15 | public class TwitterKafkaStatusListener extends StatusAdapter { 16 | 17 | private static final Logger LOG = LoggerFactory.getLogger(TwitterKafkaStatusListener.class); 18 | 19 | private final KafkaConfigData kafkaConfigData; 20 | 21 | private final KafkaProducer kafkaProducer; 22 | 23 | private final TwitterStatusToAvroTransformer twitterStatusToAvroTransformer; 24 | 25 | public TwitterKafkaStatusListener(KafkaConfigData kafkaConfigData, KafkaProducer kafkaProducer, TwitterStatusToAvroTransformer twitterStatusToAvroTransformer) { 26 | this.kafkaConfigData = kafkaConfigData; 27 | this.kafkaProducer = kafkaProducer; 28 | this.twitterStatusToAvroTransformer = twitterStatusToAvroTransformer; 29 | } 30 | 31 | @Override 32 | public void onStatus(Status status){ 33 | LOG.info("Twitter status with text {} sending to kafka topic {}", status.getText(), kafkaConfigData.getTopicName()); 34 | TwitterAvroModel twitterAvroModel = twitterStatusToAvroTransformer.getTwitterAvroModelFromStatus(status); 35 | kafkaProducer.send(kafkaConfigData.getTopicName(), twitterAvroModel.getUserId(), twitterAvroModel); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /services/twitter-to-kafka-service/src/main/java/com/microservices/runner/StreamRunner.java: -------------------------------------------------------------------------------- 1 | package com.microservices.runner; 2 | 3 | import twitter4j.TwitterException; 4 | 5 | public interface StreamRunner { 6 | void start() throws TwitterException; 7 | } 8 | -------------------------------------------------------------------------------- /services/twitter-to-kafka-service/src/main/java/com/microservices/transformer/TwitterStatusToAvroTransformer.java: -------------------------------------------------------------------------------- 1 | package com.microservices.transformer; 2 | 3 | import com.microservices.kafka.avro.model.TwitterAvroModel; 4 | import org.springframework.stereotype.Component; 5 | import twitter4j.Status; 6 | 7 | @Component 8 | public class TwitterStatusToAvroTransformer { 9 | 10 | public TwitterAvroModel getTwitterAvroModelFromStatus(Status status){ 11 | return TwitterAvroModel 12 | .newBuilder() 13 | .setId(status.getId()) 14 | .setUserId(status.getUser().getId()) 15 | .setText(status.getText()) 16 | .setCreatedAt(status.getCreatedAt().getTime()) 17 | .build(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /services/twitter-to-kafka-service/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | application: 3 | name: twitter-to-kafka-service 4 | profiles: 5 | active: twitter_to_kafka 6 | config: 7 | import: 'configserver:' 8 | cloud: 9 | config: 10 | name: twitter-to-kafka-service,config-client 11 | username: spring_cloud_user 12 | password: '1234' 13 | 14 | log: 15 | app-name: twitter-to-kafka-service -------------------------------------------------------------------------------- /services/twitter-to-kafka-service/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /services/twitter-to-kafka-service/src/test/java/com/microservices/TwitterToKafkaServiceApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.microservices; 2 | 3 | 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | 7 | @SpringBootTest 8 | public class TwitterToKafkaServiceApplicationTests 9 | { 10 | @Test 11 | public void contextLoad() {} 12 | } 13 | --------------------------------------------------------------------------------