├── tweet-map-frontend ├── src │ ├── assets │ │ ├── .gitkeep │ │ └── TweetMap-Architecture.png │ ├── app │ │ ├── trend │ │ │ ├── trend.component.css │ │ │ ├── trend.component.html │ │ │ ├── trend.component.ts │ │ │ └── trend.component.spec.ts │ │ ├── topic-list │ │ │ ├── topic-list.component.css │ │ │ ├── topic-list.component.html │ │ │ ├── topic-list.component.ts │ │ │ └── topic-list.component.spec.ts │ │ ├── map │ │ │ ├── visualization │ │ │ │ ├── visualization.component.css │ │ │ │ ├── visualization.component.spec.ts │ │ │ │ ├── visualization.component.ts │ │ │ │ └── visualization.component.html │ │ │ ├── map-routing.module.ts │ │ │ └── map.module.ts │ │ ├── header │ │ │ ├── header.component.css │ │ │ ├── header.component.spec.ts │ │ │ ├── header.component.html │ │ │ └── header.component.ts │ │ ├── footer │ │ │ ├── footer.component.css │ │ │ ├── footer.component.html │ │ │ ├── footer.component.ts │ │ │ └── footer.component.spec.ts │ │ ├── app.component.css │ │ ├── app.component.ts │ │ ├── app-routing.module.ts │ │ ├── app.module.ts │ │ ├── app.component.spec.ts │ │ └── app.component.html │ ├── environments │ │ ├── environment.prod.ts │ │ └── environment.ts │ ├── styles.css │ ├── favicon.ico │ ├── index.html │ ├── main.ts │ ├── test.ts │ └── polyfills.ts ├── e2e │ ├── src │ │ ├── app.po.ts │ │ └── app.e2e-spec.ts │ ├── tsconfig.json │ └── protractor.conf.js ├── .editorconfig ├── tsconfig.app.json ├── tsconfig.spec.json ├── .browserslistrc ├── .gitignore ├── tsconfig.json ├── README.md ├── package.json ├── karma.conf.js ├── tslint.json └── angular.json ├── gradle.properties ├── HeatMap.png ├── tweetmap.pptx ├── ScaledCircle.png ├── TweetMap-Architecture.png ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── rest-api-server ├── src │ ├── main │ │ ├── .DS_Store │ │ ├── resources │ │ │ └── application.yaml │ │ └── java │ │ │ └── io │ │ │ └── zmyzheng │ │ │ └── restapi │ │ │ ├── RestApiApplication.java │ │ │ ├── domain │ │ │ ├── TopicStatistic.java │ │ │ ├── TopicTrend.java │ │ │ ├── HotTopicsTrend.java │ │ │ └── Tweet.java │ │ │ ├── api │ │ │ ├── model │ │ │ │ ├── TopicStatisticDTO.java │ │ │ │ ├── TopicTrendDTO.java │ │ │ │ ├── HotTopicsTrendDTO.java │ │ │ │ ├── TweetDTO.java │ │ │ │ ├── TopicTrendFilter.java │ │ │ │ ├── TopicStatisticFilter.java │ │ │ │ ├── TweetFilter.java │ │ │ │ └── HotTopicsTrendFilter.java │ │ │ ├── mapping │ │ │ │ ├── TweetMapper.java │ │ │ │ ├── HotTopicsTrendMapper.java │ │ │ │ ├── TopicTrendMapper.java │ │ │ │ └── TopicStatisticMapper.java │ │ │ └── controller │ │ │ │ ├── TweetController.java │ │ │ │ └── TopicController.java │ │ │ ├── repository │ │ │ ├── TweetRepository.java │ │ │ └── EsOperationRepository.java │ │ │ ├── service │ │ │ ├── TweetService.java │ │ │ ├── TopicService.java │ │ │ ├── TweetServiceImpl.java │ │ │ └── TopicServiceImpl.java │ │ │ ├── bootstrap │ │ │ └── Bootstrap.java │ │ │ └── config │ │ │ └── swagger │ │ │ └── OpenApiConfig.java │ └── test │ │ └── java │ │ └── io │ │ └── zmyzheng │ │ └── restapi │ │ └── RestApiApplicationTests.java ├── Readme.md └── build.gradle ├── flink-processor ├── src │ └── main │ │ ├── resources │ │ ├── application.properties │ │ └── logback.xml │ │ └── java │ │ └── io │ │ └── zmyzheng │ │ └── processor │ │ ├── model │ │ ├── UniqueEntity.java │ │ └── Tweet.java │ │ ├── StreamProcessor.java │ │ ├── StreamProcessingDriver.java │ │ ├── Application.java │ │ └── impl │ │ ├── TweetKafkaEsProcessor.java │ │ └── KafkaEsProcessor.java └── build.gradle ├── tweet-collector ├── src │ └── main │ │ ├── resources │ │ ├── application.properties │ │ └── logback.xml │ │ └── java │ │ └── io │ │ └── zmyzheng │ │ └── collector │ │ ├── Sinkable.java │ │ ├── SocialMediaCollector.java │ │ ├── SocialMediaCollectionDriver.java │ │ └── implementation │ │ ├── KafkaSink.java │ │ └── TweetCollector.java ├── Dockerfile └── build.gradle ├── settings.gradle ├── .gitignore ├── DevOps └── tweet-collector.yaml ├── README.md ├── gradlew.bat ├── gradlew └── LICENSE /tweet-map-frontend/src/assets/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | dockerRepo=docker.io/zmyzheng -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/trend/trend.component.css: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/topic-list/topic-list.component.css: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/map/visualization/visualization.component.css: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /HeatMap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zmyzheng/TweetMap/HEAD/HeatMap.png -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/trend/trend.component.html: -------------------------------------------------------------------------------- 1 |

trend works!

2 | -------------------------------------------------------------------------------- /tweetmap.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zmyzheng/TweetMap/HEAD/tweetmap.pptx -------------------------------------------------------------------------------- /ScaledCircle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zmyzheng/TweetMap/HEAD/ScaledCircle.png -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/topic-list/topic-list.component.html: -------------------------------------------------------------------------------- 1 |

topic-list works!

2 | -------------------------------------------------------------------------------- /TweetMap-Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zmyzheng/TweetMap/HEAD/TweetMap-Architecture.png -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/header/header.component.css: -------------------------------------------------------------------------------- 1 | nav { 2 | background-color: darkseagreen; 3 | } 4 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/footer/footer.component.css: -------------------------------------------------------------------------------- 1 | footer { 2 | background-color: darkseagreen; 3 | } 4 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/environments/environment.prod.ts: -------------------------------------------------------------------------------- 1 | export const environment = { 2 | production: true 3 | }; 4 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/styles.css: -------------------------------------------------------------------------------- 1 | /* You can add global styles to this file, and also import other style files */ 2 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zmyzheng/TweetMap/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /rest-api-server/src/main/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zmyzheng/TweetMap/HEAD/rest-api-server/src/main/.DS_Store -------------------------------------------------------------------------------- /tweet-map-frontend/src/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zmyzheng/TweetMap/HEAD/tweet-map-frontend/src/favicon.ico -------------------------------------------------------------------------------- /rest-api-server/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | elasticsearch: 3 | rest: 4 | uris: http://localhost:9200 5 | -------------------------------------------------------------------------------- /flink-processor/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | 2 | kafka.brokerList: 3 | kafka.topic: tweets 4 | elasticsearch.url: 5 | elasticsearch.indexName: 6 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/assets/TweetMap-Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zmyzheng/TweetMap/HEAD/tweet-map-frontend/src/assets/TweetMap-Architecture.png -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/app.component.css: -------------------------------------------------------------------------------- 1 | svg#clouds { 2 | position: fixed; 3 | bottom: -160px; 4 | left: -230px; 5 | z-index: -10; 6 | width: 1920px; 7 | } 8 | -------------------------------------------------------------------------------- /tweet-collector/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | twitter.apiKey: 2 | twitter.apiSecret: 3 | twitter.token: 4 | twitter.secret: 5 | 6 | kafka.brokerList: 7 | kafka.topic: tweets 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /flink-processor/src/main/java/io/zmyzheng/processor/model/UniqueEntity.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.processor.model; 2 | 3 | /** 4 | * @Author Mingyang Zheng 5 | * @Date 2021-01-10 19:05 6 | * @Version 3.0.0 7 | */ 8 | public interface UniqueEntity { 9 | T getUniqueKey(); 10 | } 11 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/app.component.ts: -------------------------------------------------------------------------------- 1 | import { Component } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-root', 5 | templateUrl: './app.component.html', 6 | styleUrls: ['./app.component.css'] 7 | }) 8 | export class AppComponent { 9 | title = 'tweet-map-frontend'; 10 | } 11 | -------------------------------------------------------------------------------- /tweet-collector/src/main/java/io/zmyzheng/collector/Sinkable.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.collector; 2 | 3 | /** 4 | * @Author Mingyang Zheng 5 | * @Date 2021-01-02 19:30 6 | * @Version 3.0.0 7 | */ 8 | public interface Sinkable { 9 | void connect(); 10 | void send(T data); 11 | void close(); 12 | } 13 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/footer/footer.component.html: -------------------------------------------------------------------------------- 1 |
2 | © 2021 Copyright: zmyzheng 3 |
4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /rest-api-server/src/test/java/io/zmyzheng/restapi/RestApiApplicationTests.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class RestApiApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | import java.time.format.DateTimeFormatter 2 | System.setProperty("buildTimestamp", LocalDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss"))) 3 | 4 | 5 | rootProject.name = 'TweetMap' 6 | 7 | include 'tweet-collector' 8 | 9 | 10 | include 'flink-processor' 11 | include 'rest-api-server' 12 | 13 | -------------------------------------------------------------------------------- /tweet-collector/src/main/java/io/zmyzheng/collector/SocialMediaCollector.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.collector; 2 | 3 | /** 4 | * @Author Mingyang Zheng 5 | * @Date 2021-01-02 19:34 6 | * @Version 3.0.0 7 | */ 8 | public interface SocialMediaCollector { 9 | void start(); 10 | 11 | T collect(); 12 | 13 | void stop(); 14 | } 15 | -------------------------------------------------------------------------------- /tweet-map-frontend/e2e/src/app.po.ts: -------------------------------------------------------------------------------- 1 | import { browser, by, element } from 'protractor'; 2 | 3 | export class AppPage { 4 | async navigateTo(): Promise { 5 | return browser.get(browser.baseUrl); 6 | } 7 | 8 | async getTitleText(): Promise { 9 | return element(by.css('app-root .content span')).getText(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tweet-map-frontend/e2e/tsconfig.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "extends": "../tsconfig.json", 4 | "compilerOptions": { 5 | "outDir": "../out-tsc/e2e", 6 | "module": "commonjs", 7 | "target": "es2018", 8 | "types": [ 9 | "jasmine", 10 | "node" 11 | ] 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tweet-map-frontend/.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor configuration, see https://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 2 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.ts] 12 | quote_type = single 13 | 14 | [*.md] 15 | max_line_length = off 16 | trim_trailing_whitespace = false 17 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/trend/trend.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-trend', 5 | templateUrl: './trend.component.html', 6 | styleUrls: ['./trend.component.css'] 7 | }) 8 | export class TrendComponent implements OnInit { 9 | 10 | constructor() { } 11 | 12 | ngOnInit(): void { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | TweetMapFrontend 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /tweet-map-frontend/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "extends": "./tsconfig.json", 4 | "compilerOptions": { 5 | "outDir": "./out-tsc/app", 6 | "types": [] 7 | }, 8 | "files": [ 9 | "src/main.ts", 10 | "src/polyfills.ts" 11 | ], 12 | "include": [ 13 | "src/**/*.d.ts" 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/footer/footer.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-footer', 5 | templateUrl: './footer.component.html', 6 | styleUrls: ['./footer.component.css'] 7 | }) 8 | export class FooterComponent implements OnInit { 9 | 10 | constructor() { } 11 | 12 | ngOnInit(): void { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /flink-processor/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /rest-api-server/Readme.md: -------------------------------------------------------------------------------- 1 | 这里选择把DTO和domain object的转换放在API层有两个原因： 2 | 1. 当一个serviceA依赖另一个serviceB的结果时，如果把转换放在service层，那serviceA拿到的只能是转换后的结果，可能会取不到一些不想暴露给外部 3 | 的field。比如serviceA用serviceB的内部field做join，然后把join的结果返回出去。这种情况下serviceB只是一个internal service，不需要暴露给外部， 4 | 所以根本不需要DTO，如果非得暴露DTO，反而可能使serviceA拿不到想要join的内部field。 5 | 6 | 2. 如果把DTO放在service层，那API层和service层都需要保存对外的model，service保存DTO，API保存request response等，不易管理 -------------------------------------------------------------------------------- /tweet-collector/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/topic-list/topic-list.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-topic-list', 5 | templateUrl: './topic-list.component.html', 6 | styleUrls: ['./topic-list.component.css'] 7 | }) 8 | export class TopicListComponent implements OnInit { 9 | 10 | constructor() { } 11 | 12 | ngOnInit(): void { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/RestApiApplication.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class RestApiApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(RestApiApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/main.ts: -------------------------------------------------------------------------------- 1 | import { enableProdMode } from '@angular/core'; 2 | import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; 3 | 4 | import { AppModule } from './app/app.module'; 5 | import { environment } from './environments/environment'; 6 | 7 | if (environment.production) { 8 | enableProdMode(); 9 | } 10 | 11 | platformBrowserDynamic().bootstrapModule(AppModule) 12 | .catch(err => console.error(err)); 13 | -------------------------------------------------------------------------------- /tweet-map-frontend/tsconfig.spec.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "extends": "./tsconfig.json", 4 | "compilerOptions": { 5 | "outDir": "./out-tsc/spec", 6 | "types": [ 7 | "jasmine" 8 | ] 9 | }, 10 | "files": [ 11 | "src/test.ts", 12 | "src/polyfills.ts" 13 | ], 14 | "include": [ 15 | "src/**/*.spec.ts", 16 | "src/**/*.d.ts" 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/domain/TopicStatistic.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.domain; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | 7 | /** 8 | * @Author Mingyang Zheng 9 | * @Date 2021-01-16 22:22 10 | * @Version 3.0.0 11 | */ 12 | 13 | @Data 14 | @NoArgsConstructor 15 | @AllArgsConstructor 16 | public class TopicStatistic { 17 | private String topicName; 18 | private long count; 19 | } 20 | -------------------------------------------------------------------------------- /flink-processor/src/main/java/io/zmyzheng/processor/StreamProcessor.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.processor; 2 | 3 | 4 | /** 5 | * @Author Mingyang Zheng 6 | * @Date 2021-01-05 21:57 7 | * @Version 3.0.0 8 | */ 9 | public interface StreamProcessor { 10 | 11 | void configureStreamExecutionEnvironment(); 12 | 13 | void addSource(); 14 | 15 | void defineProcessingLogic(); 16 | 17 | void addSink() throws Exception; 18 | 19 | void process() throws Exception; 20 | 21 | 22 | } 23 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/model/TopicStatisticDTO.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | 7 | /** 8 | * @Author Mingyang Zheng 9 | * @Date 2021-01-16 22:25 10 | * @Version 3.0.0 11 | */ 12 | 13 | @Data 14 | @NoArgsConstructor 15 | @AllArgsConstructor 16 | public class TopicStatisticDTO { 17 | private String topicName; 18 | private long count; 19 | } 20 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/domain/TopicTrend.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.domain; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | 7 | import java.util.Date; 8 | 9 | /** 10 | * @Author Mingyang Zheng 11 | * @Date 2021-01-17 16:06 12 | * @Version 3.0.0 13 | */ 14 | 15 | @Data 16 | @NoArgsConstructor 17 | @AllArgsConstructor 18 | public class TopicTrend { 19 | private Date date; 20 | private long count; 21 | } 22 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/model/TopicTrendDTO.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | 7 | import java.util.Date; 8 | 9 | /** 10 | * @Author Mingyang Zheng 11 | * @Date 2021-01-17 17:04 12 | * @Version 3.0.0 13 | */ 14 | 15 | @Data 16 | @NoArgsConstructor 17 | @AllArgsConstructor 18 | public class TopicTrendDTO { 19 | private Date date; 20 | private long count; 21 | } 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | build 3 | out 4 | 5 | 6 | HELP.md 7 | .gradle 8 | build/ 9 | !gradle/wrapper/gradle-wrapper.jar 10 | !**/src/main/** 11 | !**/src/test/** 12 | 13 | ### STS ### 14 | .apt_generated 15 | .classpath 16 | .factorypath 17 | .project 18 | .settings 19 | .springBeans 20 | .sts4-cache 21 | 22 | ### IntelliJ IDEA ### 23 | .idea 24 | *.iws 25 | *.iml 26 | *.ipr 27 | out/ 28 | 29 | ### NetBeans ### 30 | /nbproject/private/ 31 | /nbbuild/ 32 | /dist/ 33 | /nbdist/ 34 | /.nb-gradle/ 35 | 36 | ### VS Code ### 37 | .vscode/ 38 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/domain/HotTopicsTrend.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.domain; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | 7 | import java.util.Date; 8 | import java.util.List; 9 | 10 | /** 11 | * @Author Mingyang Zheng 12 | * @Date 2021-01-17 19:02 13 | * @Version 3.0.0 14 | */ 15 | 16 | @Data 17 | @NoArgsConstructor 18 | @AllArgsConstructor 19 | public class HotTopicsTrend { 20 | private Date date; 21 | private List hotTopics; 22 | } 23 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/map/map-routing.module.ts: -------------------------------------------------------------------------------- 1 | import { NgModule } from '@angular/core'; 2 | import { RouterModule, Routes } from '@angular/router'; 3 | import {VisualizationComponent} from './visualization/visualization.component'; 4 | 5 | 6 | 7 | const routes: Routes = [ 8 | { path: '', redirectTo: 'visualization', pathMatch: 'full' }, 9 | { path: 'visualization', component: VisualizationComponent } 10 | ]; 11 | 12 | @NgModule({ 13 | imports: [RouterModule.forChild(routes)], 14 | exports: [RouterModule] 15 | }) 16 | export class MapRoutingModule { } 17 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/model/HotTopicsTrendDTO.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | 7 | import java.util.Date; 8 | import java.util.List; 9 | 10 | /** 11 | * @Author Mingyang Zheng 12 | * @Date 2021-01-17 20:28 13 | * @Version 3.0.0 14 | */ 15 | 16 | @Data 17 | @NoArgsConstructor 18 | @AllArgsConstructor 19 | public class HotTopicsTrendDTO { 20 | private Date date; 21 | private List hotTopics; 22 | } 23 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/repository/TweetRepository.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.repository; 2 | 3 | import io.zmyzheng.restapi.domain.Tweet; 4 | import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; 5 | import org.springframework.stereotype.Repository; 6 | 7 | import java.util.List; 8 | 9 | /** 10 | * @Author: Mingyang Zheng 11 | * @Date: 2020-02-17 00:51 12 | */ 13 | 14 | @Repository 15 | public interface TweetRepository extends ElasticsearchRepository { 16 | List findAll(); 17 | 18 | } 19 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/service/TweetService.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.service; 2 | 3 | import io.zmyzheng.restapi.domain.Tweet; 4 | import org.springframework.data.elasticsearch.core.geo.GeoPoint; 5 | 6 | import java.util.Date; 7 | import java.util.List; 8 | 9 | /** 10 | * @Author: Mingyang Zheng 11 | * @Date: 2020-02-17 17:54 12 | * @Version 3.0.0 13 | */ 14 | public interface TweetService { 15 | 16 | List getTweets(); 17 | 18 | List filterTweets(Date timeFrom, Date timeTo, List selectedTags, GeoPoint center, String radius); 19 | 20 | 21 | 22 | 23 | } 24 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/mapping/TweetMapper.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.mapping; 2 | 3 | import io.zmyzheng.restapi.api.model.TweetDTO; 4 | import io.zmyzheng.restapi.domain.Tweet; 5 | import org.mapstruct.Mapper; 6 | import org.mapstruct.factory.Mappers; 7 | 8 | import java.util.List; 9 | 10 | /** 11 | * @Author Mingyang Zheng 12 | * @Date 2021-01-10 23:58 13 | * @Version 3.0.0 14 | */ 15 | 16 | @Mapper 17 | public interface TweetMapper { 18 | TweetMapper INSTANCE = Mappers.getMapper(TweetMapper.class); 19 | 20 | TweetDTO convert(Tweet tweet); 21 | List convert(List tweets); 22 | } 23 | -------------------------------------------------------------------------------- /flink-processor/src/main/java/io/zmyzheng/processor/StreamProcessingDriver.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.processor; 2 | 3 | /** 4 | * @Author Mingyang Zheng 5 | * @Date 2021-01-05 23:47 6 | * @Version 3.0.0 7 | */ 8 | public class StreamProcessingDriver { 9 | 10 | private StreamProcessor processor; 11 | 12 | public StreamProcessingDriver(StreamProcessor processor) { 13 | this.processor = processor; 14 | } 15 | 16 | public void run() throws Exception { 17 | this.processor.addSource(); 18 | this.processor.defineProcessingLogic(); 19 | this.processor.addSink(); 20 | this.processor.process(); 21 | 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/model/TweetDTO.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | import org.springframework.data.elasticsearch.core.geo.GeoPoint; 7 | 8 | import java.util.Date; 9 | import java.util.List; 10 | 11 | /** 12 | * @Author Mingyang Zheng 13 | * @Date 2021-01-10 23:55 14 | * @Version 3.0.0 15 | */ 16 | 17 | @Data 18 | @NoArgsConstructor 19 | @AllArgsConstructor 20 | public class TweetDTO { 21 | 22 | private String id; 23 | private Date timestamp; 24 | private List hashTags; 25 | private GeoPoint coordinate; 26 | 27 | } 28 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/mapping/HotTopicsTrendMapper.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.mapping; 2 | 3 | import io.zmyzheng.restapi.api.model.HotTopicsTrendDTO; 4 | import io.zmyzheng.restapi.domain.HotTopicsTrend; 5 | import org.mapstruct.Mapper; 6 | import org.mapstruct.factory.Mappers; 7 | 8 | import java.util.List; 9 | 10 | /** 11 | * @Author Mingyang Zheng 12 | * @Date 2021-01-17 20:30 13 | * @Version 3.0.0 14 | */ 15 | 16 | @Mapper 17 | public interface HotTopicsTrendMapper { 18 | 19 | HotTopicsTrendMapper INSTANCE = Mappers.getMapper(HotTopicsTrendMapper.class); 20 | 21 | List convert(List topicStatistic); 22 | } 23 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/mapping/TopicTrendMapper.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.mapping; 2 | 3 | 4 | import io.zmyzheng.restapi.api.model.TopicTrendDTO; 5 | import io.zmyzheng.restapi.domain.TopicTrend; 6 | import org.mapstruct.Mapper; 7 | import org.mapstruct.Mapping; 8 | import org.mapstruct.factory.Mappers; 9 | 10 | import java.util.List; 11 | 12 | /** 13 | * @Author Mingyang Zheng 14 | * @Date 2021-01-17 17:11 15 | * @Version 3.0.0 16 | */ 17 | 18 | @Mapper 19 | public interface TopicTrendMapper { 20 | TopicTrendMapper INSTANCE = Mappers.getMapper(TopicTrendMapper.class); 21 | 22 | List convert(List topicTrends); 23 | } 24 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/model/TopicTrendFilter.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | import org.springframework.data.elasticsearch.core.geo.GeoPoint; 7 | 8 | import java.util.Date; 9 | 10 | /** 11 | * @Author Mingyang Zheng 12 | * @Date 2021-01-17 17:01 13 | * @Version 3.0.0 14 | */ 15 | 16 | @Data 17 | @NoArgsConstructor 18 | @AllArgsConstructor 19 | public class TopicTrendFilter { 20 | 21 | private String calendarInterval; 22 | 23 | private Date timeFrom; 24 | private Date timeTo; 25 | private GeoPoint center; 26 | private String radius; 27 | } 28 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/model/TopicStatisticFilter.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | import org.springframework.data.elasticsearch.core.geo.GeoPoint; 7 | 8 | import java.util.Date; 9 | 10 | /** 11 | * @Author: Mingyang Zheng 12 | * @Date: 2020-02-23 13:46 13 | * @Version 3.0.0 14 | */ 15 | 16 | @Data 17 | @NoArgsConstructor 18 | @AllArgsConstructor 19 | public class TopicStatisticFilter { 20 | 21 | private int topN; 22 | 23 | private Date timeFrom; 24 | private Date timeTo; 25 | private GeoPoint center; 26 | private String radius; 27 | 28 | 29 | } 30 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/model/TweetFilter.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | import org.springframework.data.elasticsearch.core.geo.GeoPoint; 7 | 8 | import java.util.Date; 9 | import java.util.List; 10 | 11 | /** 12 | * @Author Mingyang Zheng 13 | * @Date 2021-01-12 23:01 14 | * @Version 3.0.0 15 | */ 16 | 17 | @Data 18 | @NoArgsConstructor 19 | @AllArgsConstructor 20 | public class TweetFilter { 21 | private Date timeFrom; 22 | private Date timeTo; 23 | private List selectedTags; 24 | private GeoPoint center; 25 | private String radius; 26 | } 27 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/mapping/TopicStatisticMapper.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.mapping; 2 | 3 | import io.zmyzheng.restapi.api.model.TopicStatisticDTO; 4 | import io.zmyzheng.restapi.domain.TopicStatistic; 5 | import org.mapstruct.Mapper; 6 | import org.mapstruct.Mapping; 7 | import org.mapstruct.factory.Mappers; 8 | 9 | import java.util.List; 10 | 11 | /** 12 | * @Author Mingyang Zheng 13 | * @Date 2021-01-16 22:26 14 | * @Version 3.0.0 15 | */ 16 | @Mapper 17 | public interface TopicStatisticMapper { 18 | TopicStatisticMapper INSTANCE = Mappers.getMapper(TopicStatisticMapper.class); 19 | 20 | List convert(List topicStatistic); 21 | } 22 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/app-routing.module.ts: -------------------------------------------------------------------------------- 1 | import { NgModule } from '@angular/core'; 2 | import { RouterModule, Routes } from '@angular/router'; 3 | import {TopicListComponent} from './topic-list/topic-list.component'; 4 | import {TrendComponent} from './trend/trend.component'; 5 | 6 | const routes: Routes = [ 7 | { path: '', redirectTo: 'map', pathMatch: 'full' }, 8 | 9 | { path: 'topics', component: TopicListComponent }, 10 | { path: 'trend', component: TrendComponent }, 11 | { path: 'map', loadChildren: () => import('./map/map.module').then(m => m.MapModule) }, 12 | ]; 13 | 14 | @NgModule({ 15 | imports: [RouterModule.forRoot(routes)], 16 | exports: [RouterModule] 17 | }) 18 | export class AppRoutingModule { } 19 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/model/HotTopicsTrendFilter.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | import org.springframework.data.elasticsearch.core.geo.GeoPoint; 7 | 8 | import java.util.Date; 9 | 10 | /** 11 | * @Author Mingyang Zheng 12 | * @Date 2021-01-17 20:25 13 | * @Version 3.0.0 14 | */ 15 | 16 | @Data 17 | @NoArgsConstructor 18 | @AllArgsConstructor 19 | public class HotTopicsTrendFilter { 20 | 21 | private String calendarInterval; 22 | private int topN; 23 | 24 | private Date timeFrom; 25 | private Date timeTo; 26 | private GeoPoint center; 27 | private String radius; 28 | } 29 | -------------------------------------------------------------------------------- /flink-processor/src/main/java/io/zmyzheng/processor/model/Tweet.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.processor.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | import org.elasticsearch.common.geo.GeoPoint; 7 | 8 | import java.util.List; 9 | 10 | /** 11 | * @Author: Mingyang Zheng 12 | * @Date: 2020-02-11 21:31 13 | */ 14 | 15 | @Data 16 | @NoArgsConstructor 17 | @AllArgsConstructor 18 | public class Tweet implements UniqueEntity { 19 | 20 | private String id; 21 | 22 | private long timestamp; 23 | 24 | private List hashTags; 25 | 26 | private GeoPoint coordinate; 27 | 28 | @Override 29 | public String getUniqueKey() { 30 | return getId(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/environments/environment.ts: -------------------------------------------------------------------------------- 1 | // This file can be replaced during build by using the `fileReplacements` array. 2 | // `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. 3 | // The list of file replacements can be found in `angular.json`. 4 | 5 | export const environment = { 6 | production: false 7 | }; 8 | 9 | /* 10 | * For easier debugging in development mode, you can import the following file 11 | * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`. 12 | * 13 | * This import should be commented out in production mode because it will have a negative impact 14 | * on performance if an error is thrown. 15 | */ 16 | // import 'zone.js/dist/zone-error'; // Included with Angular CLI. 17 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/trend/trend.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { TrendComponent } from './trend.component'; 4 | 5 | describe('TrendComponent', () => { 6 | let component: TrendComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async () => { 10 | await TestBed.configureTestingModule({ 11 | declarations: [ TrendComponent ] 12 | }) 13 | .compileComponents(); 14 | }); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(TrendComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/footer/footer.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { FooterComponent } from './footer.component'; 4 | 5 | describe('FooterComponent', () => { 6 | let component: FooterComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async () => { 10 | await TestBed.configureTestingModule({ 11 | declarations: [ FooterComponent ] 12 | }) 13 | .compileComponents(); 14 | }); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(FooterComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/header/header.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { HeaderComponent } from './header.component'; 4 | 5 | describe('HeaderComponent', () => { 6 | let component: HeaderComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async () => { 10 | await TestBed.configureTestingModule({ 11 | declarations: [ HeaderComponent ] 12 | }) 13 | .compileComponents(); 14 | }); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(HeaderComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /tweet-map-frontend/.browserslistrc: -------------------------------------------------------------------------------- 1 | # This file is used by the build system to adjust CSS and JS output to support the specified browsers below. 2 | # For additional information regarding the format and rule options, please see: 3 | # https://github.com/browserslist/browserslist#queries 4 | 5 | # For the full list of supported browsers by the Angular framework, please see: 6 | # https://angular.io/guide/browser-support 7 | 8 | # You can see what browsers were selected by your queries by running: 9 | # npx browserslist 10 | 11 | last 1 Chrome version 12 | last 1 Firefox version 13 | last 2 Edge major versions 14 | last 2 Safari major versions 15 | last 2 iOS major versions 16 | Firefox ESR 17 | not IE 11 # Angular supports IE 11 only as an opt-in. To opt-in, remove the 'not' prefix on this line. 18 | -------------------------------------------------------------------------------- /tweet-map-frontend/e2e/src/app.e2e-spec.ts: -------------------------------------------------------------------------------- 1 | import { browser, logging } from 'protractor'; 2 | import { AppPage } from './app.po'; 3 | 4 | describe('workspace-project App', () => { 5 | let page: AppPage; 6 | 7 | beforeEach(() => { 8 | page = new AppPage(); 9 | }); 10 | 11 | it('should display welcome message', async () => { 12 | await page.navigateTo(); 13 | expect(await page.getTitleText()).toEqual('tweet-map-frontend app is running!'); 14 | }); 15 | 16 | afterEach(async () => { 17 | // Assert that there are no errors emitted from the browser 18 | const logs = await browser.manage().logs().get(logging.Type.BROWSER); 19 | expect(logs).not.toContain(jasmine.objectContaining({ 20 | level: logging.Level.SEVERE, 21 | } as logging.Entry)); 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/topic-list/topic-list.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { TopicListComponent } from './topic-list.component'; 4 | 5 | describe('TopicListComponent', () => { 6 | let component: TopicListComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async () => { 10 | await TestBed.configureTestingModule({ 11 | declarations: [ TopicListComponent ] 12 | }) 13 | .compileComponents(); 14 | }); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(TopicListComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/map/visualization/visualization.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { ComponentFixture, TestBed } from '@angular/core/testing'; 2 | 3 | import { VisualizationComponent } from './visualization.component'; 4 | 5 | describe('VisualizationComponent', () => { 6 | let component: VisualizationComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(async () => { 10 | await TestBed.configureTestingModule({ 11 | declarations: [ VisualizationComponent ] 12 | }) 13 | .compileComponents(); 14 | }); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(VisualizationComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /tweet-map-frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # See http://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # compiled output 4 | /dist 5 | /tmp 6 | /out-tsc 7 | # Only exists if Bazel was run 8 | /bazel-out 9 | 10 | # dependencies 11 | /node_modules 12 | 13 | # profiling files 14 | chrome-profiler-events*.json 15 | speed-measure-plugin*.json 16 | 17 | # IDEs and editors 18 | /.idea 19 | .project 20 | .classpath 21 | .c9/ 22 | *.launch 23 | .settings/ 24 | *.sublime-workspace 25 | 26 | # IDE - VSCode 27 | .vscode/* 28 | !.vscode/settings.json 29 | !.vscode/tasks.json 30 | !.vscode/launch.json 31 | !.vscode/extensions.json 32 | .history/* 33 | 34 | # misc 35 | /.sass-cache 36 | /connect.lock 37 | /coverage 38 | /libpeerconnection.log 39 | npm-debug.log 40 | yarn-error.log 41 | testem.log 42 | /typings 43 | 44 | # System Files 45 | .DS_Store 46 | Thumbs.db 47 | -------------------------------------------------------------------------------- /tweet-collector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:16-jdk-alpine 2 | 3 | RUN apk update && apk add --no-cache bash 4 | 5 | # add snappy dependency in docker image for kafka 6 | RUN apk add --no-cache libc6-compat 7 | RUN ln -s /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2 8 | 9 | MAINTAINER zmyzheng.io 10 | 11 | ARG JAR_FILE 12 | ARG APP_NAME 13 | 14 | ENV USER_NAME zmyzheng 15 | ENV APP_HOME /home/$USER_NAME/TweetMap/${APP_NAME} 16 | 17 | 18 | RUN addgroup -S appgroup && adduser -S $USER_NAME -G appgroup 19 | 20 | USER $USER_NAME 21 | 22 | RUN mkdir -p $APP_HOME 23 | WORKDIR $APP_HOME 24 | RUN mkdir config 25 | RUN mkdir logs 26 | COPY ${JAR_FILE} ${JAR_FILE} 27 | RUN ln -s ${JAR_FILE} app-server.jar 28 | 29 | ENTRYPOINT ["java", "-jar", "app-server.jar"] 30 | #ENTRYPOINT ["/usr/bin/java", "-jar", "app-server.jar", ">/dev/null", "2>&1"] 31 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/map/map.module.ts: -------------------------------------------------------------------------------- 1 | import { NgModule } from '@angular/core'; 2 | import { CommonModule } from '@angular/common'; 3 | 4 | import { MapRoutingModule } from './map-routing.module'; 5 | import { VisualizationComponent } from './visualization/visualization.component'; 6 | import {ReactiveFormsModule} from '@angular/forms'; 7 | import {NgbModule} from '@ng-bootstrap/ng-bootstrap'; 8 | import {GoogleMapsModule} from '@angular/google-maps'; 9 | import {HttpClientJsonpModule, HttpClientModule} from '@angular/common/http'; 10 | 11 | 12 | 13 | 14 | @NgModule({ 15 | declarations: [VisualizationComponent], 16 | imports: [ 17 | CommonModule, 18 | ReactiveFormsModule, 19 | MapRoutingModule, 20 | NgbModule, 21 | GoogleMapsModule, 22 | HttpClientModule, 23 | HttpClientJsonpModule, 24 | ] 25 | }) 26 | export class MapModule { } 27 | -------------------------------------------------------------------------------- /DevOps/tweet-collector.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: tweet-collector-deployment 5 | labels: 6 | app: tweet-collector 7 | type: collector 8 | spec: 9 | template: 10 | metadata: 11 | name: tweet-collector-pod 12 | labels: 13 | app: tweet-collector 14 | type: collector 15 | spec: 16 | containers: 17 | - name: tweet-collector 18 | image: zmyzheng/tweet-collector:1.1-SNAPSHOT 19 | # ports: 20 | # - containerPort: 8443 21 | # args: ["--kafka.bootstrapServers=b-3.fs-ec-msk-cluster-vpc.n6h4ok.c2.kafka.us-west-2.amazonaws.com:9092,b-1.fs-ec-msk-cluster-vpc.n6h4ok.c2.kafka.us-west-2.amazonaws.com:9092,b-2.fs-ec-msk-cluster-vpc.n6h4ok.c2.kafka.us-west-2.amazonaws.com:9092"] 22 | replicas: 1 23 | selector: 24 | matchLabels: 25 | type: collector -------------------------------------------------------------------------------- /tweet-map-frontend/src/test.ts: -------------------------------------------------------------------------------- 1 | // This file is required by karma.conf.js and loads recursively all the .spec and framework files 2 | 3 | import 'zone.js/dist/zone-testing'; 4 | import { getTestBed } from '@angular/core/testing'; 5 | import { 6 | BrowserDynamicTestingModule, 7 | platformBrowserDynamicTesting 8 | } from '@angular/platform-browser-dynamic/testing'; 9 | 10 | declare const require: { 11 | context(path: string, deep?: boolean, filter?: RegExp): { 12 | keys(): string[]; 13 | (id: string): T; 14 | }; 15 | }; 16 | 17 | // First, initialize the Angular testing environment. 18 | getTestBed().initTestEnvironment( 19 | BrowserDynamicTestingModule, 20 | platformBrowserDynamicTesting() 21 | ); 22 | // Then we find all the tests. 23 | const context = require.context('./', true, /\.spec\.ts$/); 24 | // And load the modules. 25 | context.keys().map(context); 26 | -------------------------------------------------------------------------------- /tweet-map-frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "compileOnSave": false, 4 | "compilerOptions": { 5 | "baseUrl": "./", 6 | "outDir": "./dist/out-tsc", 7 | "forceConsistentCasingInFileNames": true, 8 | "strict": true, 9 | "noImplicitReturns": true, 10 | "noFallthroughCasesInSwitch": true, 11 | "sourceMap": true, 12 | "declaration": false, 13 | "downlevelIteration": true, 14 | "experimentalDecorators": true, 15 | "moduleResolution": "node", 16 | "importHelpers": true, 17 | "target": "es2015", 18 | "module": "es2020", 19 | "lib": [ 20 | "es2018", 21 | "dom" 22 | ] 23 | }, 24 | "angularCompilerOptions": { 25 | "enableI18nLegacyMessageIdFormat": false, 26 | "strictInjectionParameters": true, 27 | "strictInputAccessModifiers": true, 28 | "strictTemplates": true 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/service/TopicService.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.service; 2 | 3 | import io.zmyzheng.restapi.domain.HotTopicsTrend; 4 | import io.zmyzheng.restapi.domain.TopicStatistic; 5 | import io.zmyzheng.restapi.domain.TopicTrend; 6 | import org.springframework.data.elasticsearch.core.geo.GeoPoint; 7 | 8 | import java.util.Date; 9 | import java.util.List; 10 | 11 | /** 12 | * @Author Mingyang Zheng 13 | * @Date 2021-01-17 18:32 14 | * @Version 3.0.0 15 | */ 16 | 17 | public interface TopicService { 18 | List filterTopics(Date timeFrom, Date timeTo, GeoPoint center, String radius, int topN); 19 | 20 | List getTopicTrend(String topicName, String calendarInterval, Date timeFrom, Date timeTo, GeoPoint center, String radius); 21 | 22 | List getHotTopicsTrend(String calendarInterval, int topN, Date timeFrom, Date timeTo, GeoPoint center, String radius); 23 | 24 | 25 | } 26 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/bootstrap/Bootstrap.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.bootstrap; 2 | 3 | import io.zmyzheng.restapi.repository.TweetRepository; 4 | import org.springframework.boot.CommandLineRunner; 5 | import org.springframework.stereotype.Component; 6 | 7 | /** 8 | * @Author: Mingyang Zheng 9 | * @Date: 2020-02-17 01:06 10 | */ 11 | @Component 12 | public class Bootstrap implements CommandLineRunner { 13 | 14 | private final TweetRepository tweetRepository; 15 | 16 | public Bootstrap(TweetRepository tweetRepository) { 17 | this.tweetRepository = tweetRepository; 18 | } 19 | @Override 20 | public void run(String... args) throws Exception { 21 | // Tweet tweet = new Tweet(null, new Date(), Arrays.asList("love", "peace"), Arrays.asList(-48.87868, -23.97924)); 22 | // this.tweetRepository.save(tweet); 23 | 24 | System.out.println(this.tweetRepository.findById("1CYSVXABUOm3GmEwuy9F")); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /flink-processor/src/main/java/io/zmyzheng/processor/Application.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.processor; 2 | 3 | import io.zmyzheng.processor.impl.TweetKafkaEsProcessor; 4 | 5 | /** 6 | * @Author Mingyang Zheng 7 | * @Date 2021-01-05 23:50 8 | * @Version 3.0.0 9 | */ 10 | public class Application { 11 | 12 | public static void main(String[] args) throws Exception { 13 | 14 | 15 | String kafkaBootstrapServers = System.getProperty("kafka.brokerList","application.properties"); 16 | String kafkaTopic = System.getProperty("kafka.topic","application.properties"); 17 | String esUrl = System.getProperty("elasticsearch.url","application.properties"); 18 | String esIndexName = System.getProperty("elasticsearch.indexName","application.properties"); 19 | 20 | 21 | StreamProcessor processor = new TweetKafkaEsProcessor(kafkaBootstrapServers, kafkaTopic, esUrl, esIndexName); 22 | StreamProcessingDriver driver = new StreamProcessingDriver(processor); 23 | driver.run(); 24 | 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/app.module.ts: -------------------------------------------------------------------------------- 1 | import { NgModule } from '@angular/core'; 2 | import { BrowserModule } from '@angular/platform-browser'; 3 | 4 | import { AppRoutingModule } from './app-routing.module'; 5 | import { AppComponent } from './app.component'; 6 | import { NgbModule } from '@ng-bootstrap/ng-bootstrap'; 7 | import { HeaderComponent } from './header/header.component'; 8 | import { FooterComponent } from './footer/footer.component'; 9 | import { TopicListComponent } from './topic-list/topic-list.component'; 10 | import { TrendComponent } from './trend/trend.component'; 11 | 12 | 13 | import {HttpClientModule} from '@angular/common/http'; 14 | 15 | 16 | @NgModule({ 17 | declarations: [ 18 | AppComponent, 19 | HeaderComponent, 20 | FooterComponent, 21 | 22 | TopicListComponent, 23 | TrendComponent 24 | ], 25 | imports: [ 26 | BrowserModule, 27 | HttpClientModule, 28 | 29 | AppRoutingModule, 30 | NgbModule, 31 | 32 | ], 33 | providers: [], 34 | bootstrap: [AppComponent] 35 | }) 36 | export class AppModule { } 37 | -------------------------------------------------------------------------------- /tweet-map-frontend/e2e/protractor.conf.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | // Protractor configuration file, see link for more information 3 | // https://github.com/angular/protractor/blob/master/lib/config.ts 4 | 5 | const { SpecReporter, StacktraceOption } = require('jasmine-spec-reporter'); 6 | 7 | /** 8 | * @type { import("protractor").Config } 9 | */ 10 | exports.config = { 11 | allScriptsTimeout: 11000, 12 | specs: [ 13 | './src/**/*.e2e-spec.ts' 14 | ], 15 | capabilities: { 16 | browserName: 'chrome' 17 | }, 18 | directConnect: true, 19 | SELENIUM_PROMISE_MANAGER: false, 20 | baseUrl: 'http://localhost:4200/', 21 | framework: 'jasmine', 22 | jasmineNodeOpts: { 23 | showColors: true, 24 | defaultTimeoutInterval: 30000, 25 | print: function() {} 26 | }, 27 | onPrepare() { 28 | require('ts-node').register({ 29 | project: require('path').join(__dirname, './tsconfig.json') 30 | }); 31 | jasmine.getEnv().addReporter(new SpecReporter({ 32 | spec: { 33 | displayStacktrace: StacktraceOption.PRETTY 34 | } 35 | })); 36 | } 37 | }; -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/header/header.component.html: -------------------------------------------------------------------------------- 1 | 24 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/config/swagger/OpenApiConfig.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.config.swagger; 2 | 3 | import io.swagger.v3.oas.models.OpenAPI; 4 | import io.swagger.v3.oas.models.info.Info; 5 | import io.swagger.v3.oas.models.info.License; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | 10 | /** 11 | * @Author Mingyang Zheng 12 | * @Date 2021-01-18 17:24 13 | * @Version 3.0.0 14 | */ 15 | 16 | @Configuration 17 | public class OpenApiConfig { 18 | 19 | @Bean 20 | public OpenAPI customOpenAPI(@Value("${application-description}") String appDesciption) { 21 | return new OpenAPI() 22 | .info(new Info() 23 | .title("rest-api-server") 24 | .version("") 25 | .description("") 26 | .termsOfService("") 27 | .license(new License().name("Apache 2.0").url("http://springdoc.org")) 28 | ); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /tweet-map-frontend/README.md: -------------------------------------------------------------------------------- 1 | # TweetMapFrontend 2 | 3 | This project was generated with [Angular CLI](https://github.com/angular/angular-cli) version 11.1.1. 4 | 5 | ## Development server 6 | 7 | Run `ng serve` for a dev server. Navigate to `http://localhost:4200/`. The app will automatically reload if you change any of the source files. 8 | 9 | ## Code scaffolding 10 | 11 | Run `ng generate component component-name` to generate a new component. You can also use `ng generate directive|pipe|service|class|guard|interface|enum|module`. 12 | 13 | ## Build 14 | 15 | Run `ng build` to build the project. The build artifacts will be stored in the `dist/` directory. Use the `--prod` flag for a production build. 16 | 17 | ## Running unit tests 18 | 19 | Run `ng test` to execute the unit tests via [Karma](https://karma-runner.github.io). 20 | 21 | ## Running end-to-end tests 22 | 23 | Run `ng e2e` to execute the end-to-end tests via [Protractor](http://www.protractortest.org/). 24 | 25 | ## Further help 26 | 27 | To get more help on the Angular CLI use `ng help` or go check out the [Angular CLI Overview and Command Reference](https://angular.io/cli) page. 28 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/domain/Tweet.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.domain; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | 7 | import org.springframework.data.annotation.Id; 8 | import org.springframework.data.elasticsearch.annotations.Document; 9 | import org.springframework.data.elasticsearch.annotations.Field; 10 | import org.springframework.data.elasticsearch.annotations.FieldType; 11 | import org.springframework.data.elasticsearch.annotations.GeoPointField; 12 | import org.springframework.data.elasticsearch.core.geo.GeoPoint; 13 | 14 | 15 | import java.util.Date; 16 | import java.util.List; 17 | 18 | /** 19 | * @Author: Mingyang Zheng 20 | * @Date: 2020-02-17 00:29 21 | * @Version 3.0.0 22 | */ 23 | @Data 24 | @NoArgsConstructor 25 | @AllArgsConstructor 26 | @Document(indexName = "tweets") 27 | public class Tweet { 28 | @Id 29 | private String id; 30 | 31 | @Field(name = "timestamp", type = FieldType.Date) 32 | private Date timestamp; 33 | 34 | @Field(name = "hashTags", type = FieldType.Keyword) 35 | private List hashTags; 36 | 37 | @GeoPointField 38 | private GeoPoint coordinate; 39 | 40 | 41 | 42 | } 43 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/app.component.spec.ts: -------------------------------------------------------------------------------- 1 | import { TestBed } from '@angular/core/testing'; 2 | import { RouterTestingModule } from '@angular/router/testing'; 3 | import { AppComponent } from './app.component'; 4 | 5 | describe('AppComponent', () => { 6 | beforeEach(async () => { 7 | await TestBed.configureTestingModule({ 8 | imports: [ 9 | RouterTestingModule 10 | ], 11 | declarations: [ 12 | AppComponent 13 | ], 14 | }).compileComponents(); 15 | }); 16 | 17 | it('should create the app', () => { 18 | const fixture = TestBed.createComponent(AppComponent); 19 | const app = fixture.componentInstance; 20 | expect(app).toBeTruthy(); 21 | }); 22 | 23 | it(`should have as title 'tweet-map-frontend'`, () => { 24 | const fixture = TestBed.createComponent(AppComponent); 25 | const app = fixture.componentInstance; 26 | expect(app.title).toEqual('tweet-map-frontend'); 27 | }); 28 | 29 | it('should render title', () => { 30 | const fixture = TestBed.createComponent(AppComponent); 31 | fixture.detectChanges(); 32 | const compiled = fixture.nativeElement; 33 | expect(compiled.querySelector('.content span').textContent).toContain('tweet-map-frontend app is running!'); 34 | }); 35 | }); 36 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/app.component.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/service/TweetServiceImpl.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.service; 2 | 3 | import io.zmyzheng.restapi.domain.Tweet; 4 | import io.zmyzheng.restapi.repository.EsOperationRepository; 5 | import io.zmyzheng.restapi.repository.TweetRepository; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.springframework.data.elasticsearch.core.geo.GeoPoint; 8 | import org.springframework.stereotype.Service; 9 | 10 | import java.util.Date; 11 | import java.util.List; 12 | 13 | /** 14 | * @Author: Mingyang Zheng 15 | * @Date: 2020-02-22 19:31 16 | * @Version 3.0.0 17 | */ 18 | @Service 19 | @Slf4j 20 | public class TweetServiceImpl implements TweetService { 21 | 22 | private final TweetRepository tweetRepository; 23 | private final EsOperationRepository esOperationRepository; 24 | 25 | public TweetServiceImpl(TweetRepository tweetRepository, EsOperationRepository esOperationRepository) { 26 | this.tweetRepository = tweetRepository; 27 | this.esOperationRepository = esOperationRepository; 28 | } 29 | 30 | @Override 31 | public List getTweets() { 32 | return this.tweetRepository.findAll(); 33 | } 34 | 35 | @Override 36 | public List filterTweets(Date timeFrom, Date timeTo, List selectedTags, GeoPoint center, String radius) { 37 | return this.esOperationRepository.filterTweets(timeFrom, timeTo, selectedTags, center, radius); 38 | } 39 | 40 | 41 | 42 | 43 | } 44 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/header/header.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | import {NavigationEnd, Router} from '@angular/router'; 3 | import {filter} from 'rxjs/operators'; 4 | 5 | @Component({ 6 | selector: 'app-header', 7 | templateUrl: './header.component.html', 8 | styleUrls: ['./header.component.css'] 9 | }) 10 | export class HeaderComponent implements OnInit { 11 | 12 | tabs = [ 13 | { tittle: 'Map', link: '/map' }, 14 | { tittle: 'Topics', link: '/topics' }, 15 | { tittle: 'Trend', link: '/trend' } 16 | ]; 17 | 18 | // 记录当前的URL，使得无论是后退，跳转还是直接在地址栏输入地址，都能高亮当前的导航栏tab。与header.component.html中的activeId配合使用 19 | path = ''; 20 | 21 | // Step 1: 22 | // Create a property to track whether the menu is open. 23 | // Start with the menu collapsed so that it does not 24 | // appear initially when the page loads on a small screen! 25 | // https://ng-bootstrap.github.io/#/components/collapse/examples#navbar 26 | isMenuCollapsed = true; 27 | 28 | constructor(public router: Router) { 29 | // this is a workaround to get the current path. ActivedRoute is not working. 30 | // Discussion: https://github.com/angular/angular/issues/11023 31 | // https://github.com/angular/angular/issues/12623 32 | this.router.events.pipe( 33 | filter(event => event instanceof NavigationEnd ) 34 | ) 35 | .subscribe(event => this.path = (event as NavigationEnd).url); 36 | } 37 | 38 | 39 | ngOnInit(): void { 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /flink-processor/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id 'application' 4 | id "com.github.johnrengelman.shadow" version "6.1.0" 5 | id "com.gorylenko.gradle-git-properties" version "2.2.4" // auto generate git.properties and visible at /actuator/info 6 | } 7 | 8 | sourceCompatibility = '11' 9 | targetCompatibility = '11' 10 | 11 | mainClassName = 'io.zmyzheng.processor.Application' 12 | 13 | ext { 14 | flinkVersion = '1.12.0' 15 | scalaBinaryVersion = '2.11' 16 | 17 | } 18 | 19 | 20 | 21 | dependencies { 22 | 23 | compileOnly 'org.projectlombok:lombok:1.18.16' 24 | annotationProcessor 'org.projectlombok:lombok:1.18.16' 25 | testCompileOnly 'org.projectlombok:lombok:1.18.16' 26 | testAnnotationProcessor 'org.projectlombok:lombok:1.18.16' 27 | 28 | compileOnly "org.apache.flink:flink-java:${flinkVersion}" 29 | compileOnly "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}" 30 | 31 | implementation "org.apache.flink:flink-connector-kafka_${scalaBinaryVersion}:${flinkVersion}" 32 | 33 | implementation group: 'org.apache.flink', name: 'flink-connector-elasticsearch7_2.11', version: '1.12.0' 34 | 35 | implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' 36 | 37 | // testCompile group: 'junit', name: 'junit', version: '4.12' 38 | } 39 | 40 | shadowJar { 41 | transform(com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer) { 42 | resource = 'reference.conf' 43 | } 44 | } -------------------------------------------------------------------------------- /tweet-map-frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tweet-map-frontend", 3 | "version": "0.0.0", 4 | "scripts": { 5 | "ng": "ng", 6 | "start": "ng serve", 7 | "build": "ng build", 8 | "test": "ng test", 9 | "lint": "ng lint", 10 | "e2e": "ng e2e" 11 | }, 12 | "private": true, 13 | "dependencies": { 14 | "@angular/animations": "~11.1.0", 15 | "@angular/common": "~11.1.0", 16 | "@angular/compiler": "~11.1.0", 17 | "@angular/core": "~11.1.0", 18 | "@angular/forms": "~11.1.0", 19 | "@angular/google-maps": "^11.2.0", 20 | "@angular/localize": "~11.1.0", 21 | "@angular/platform-browser": "~11.1.0", 22 | "@angular/platform-browser-dynamic": "~11.1.0", 23 | "@angular/router": "~11.1.0", 24 | "@ng-bootstrap/ng-bootstrap": "^9.0.1", 25 | "bootstrap": "^4.5.0", 26 | "rxjs": "~6.6.0", 27 | "tslib": "^2.0.0", 28 | "zone.js": "~0.11.3" 29 | }, 30 | "devDependencies": { 31 | "@angular-devkit/build-angular": "~0.1101.1", 32 | "@angular/cli": "~11.1.1", 33 | "@angular/compiler-cli": "~11.1.0", 34 | "@types/jasmine": "~3.6.0", 35 | "@types/node": "^12.11.1", 36 | "codelyzer": "^6.0.0", 37 | "jasmine-core": "~3.6.0", 38 | "jasmine-spec-reporter": "~5.0.0", 39 | "karma": "~5.2.0", 40 | "karma-chrome-launcher": "~3.1.0", 41 | "karma-coverage": "~2.0.3", 42 | "karma-jasmine": "~4.0.0", 43 | "karma-jasmine-html-reporter": "^1.5.0", 44 | "protractor": "~7.0.0", 45 | "ts-node": "~8.3.0", 46 | "tslint": "~6.1.0", 47 | "typescript": "~4.1.2" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /tweet-map-frontend/karma.conf.js: -------------------------------------------------------------------------------- 1 | // Karma configuration file, see link for more information 2 | // https://karma-runner.github.io/1.0/config/configuration-file.html 3 | 4 | module.exports = function (config) { 5 | config.set({ 6 | basePath: '', 7 | frameworks: ['jasmine', '@angular-devkit/build-angular'], 8 | plugins: [ 9 | require('karma-jasmine'), 10 | require('karma-chrome-launcher'), 11 | require('karma-jasmine-html-reporter'), 12 | require('karma-coverage'), 13 | require('@angular-devkit/build-angular/plugins/karma') 14 | ], 15 | client: { 16 | jasmine: { 17 | // you can add configuration options for Jasmine here 18 | // the possible options are listed at https://jasmine.github.io/api/edge/Configuration.html 19 | // for example, you can disable the random execution with `random: false` 20 | // or set a specific seed with `seed: 4321` 21 | }, 22 | clearContext: false // leave Jasmine Spec Runner output visible in browser 23 | }, 24 | jasmineHtmlReporter: { 25 | suppressAll: true // removes the duplicated traces 26 | }, 27 | coverageReporter: { 28 | dir: require('path').join(__dirname, './coverage/tweet-map-frontend'), 29 | subdir: '.', 30 | reporters: [ 31 | { type: 'html' }, 32 | { type: 'text-summary' } 33 | ] 34 | }, 35 | reporters: ['progress', 'kjhtml'], 36 | port: 9876, 37 | colors: true, 38 | logLevel: config.LOG_INFO, 39 | autoWatch: true, 40 | browsers: ['Chrome'], 41 | singleRun: false, 42 | restartOnFileChange: true 43 | }); 44 | }; 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tweet Map with Trends 2 | 3 | 1. Build a `java` Tweet Collector to collect real-time tweets with Twitter Streaming API 4 | 2. Push collected tweets to `Kafka` cluster 5 | 3. Utilize `Apache Flink` Streaming to process (parse, filter and tranform) tweets 6 | 4. Ingest processed tweets to `Elasticsearch` for data persistance and index 7 | 5. Develop `SpringBoot` `RESTful API server` to query tweets from Elasticsearch 8 | 6. Visualize real time tweet trends with Frontend `Angular` web application 9 | 7. Show Geographical Tweet Heat Map with `Kibana` 10 | 8. Create `Docker` image and deploy `microservices` to `Kubernetes` cluster 11 | 12 | ![](TweetMap-Architecture.png) 13 | 14 | ![](ScaledCircle.png) 15 | ![](HeatMap.png) 16 | 17 | 18 | ## Project structure 19 | - **tweet-collector**: collect real-time tweets with Twitter Streaming API and push to Kafka cluster. 20 | - To support other social media, implement *SocialMediaCollector* interface. 21 | - To support other message queues, implement *Sinkable* interface. 22 | 23 | - **rest-api-server**: A RESTful API server querying tweets from Elasticsearch using SpringBoot framework. 24 | - **flink-processor**: Streaming process tweets and ingest into Elasticsearch. 25 | - **frontend-website**: Visualize Tweet Map with Angular framework. (developing) 26 | 27 | ## Build 28 | 29 | JDK version: 30 | 1. *master* branch and *dev-jdk11* branch: JDK 11 31 | 2. *dev-jdk8* branch: JDK 8 32 | 33 | ./gradlew :tweet-collector:clean :tweet-collector:build :tweet-collector:dockerPush 34 | 35 | java -jar tweet-collector/build/libs/tweet-collector-1.1-SNAPSHOT.jar 36 | 37 | ./gradlew :flink-processor:clean :flink-processor:build 38 | 39 | 40 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/controller/TweetController.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.controller; 2 | 3 | import io.zmyzheng.restapi.api.model.TweetDTO; 4 | import io.zmyzheng.restapi.api.mapping.TweetMapper; 5 | import io.zmyzheng.restapi.api.model.TweetFilter; 6 | import io.zmyzheng.restapi.service.TweetService; 7 | import lombok.extern.slf4j.Slf4j; 8 | import org.springframework.http.HttpStatus; 9 | import org.springframework.web.bind.annotation.*; 10 | 11 | import java.util.List; 12 | 13 | /** 14 | * @Author: Mingyang Zheng 15 | * @Date: 2020-02-17 17:53 16 | * @Version 3.0.0 17 | */ 18 | 19 | @Slf4j 20 | @RestController 21 | @RequestMapping(TweetController.BASE_URL) 22 | public class TweetController { 23 | public static final String BASE_URL = "api/v3/tweets"; 24 | 25 | private final TweetService tweetService; 26 | private final TweetMapper tweetMapper; 27 | 28 | public TweetController(TweetService tweetService, TweetMapper tweetMapper) { 29 | this.tweetService = tweetService; 30 | this.tweetMapper = tweetMapper; 31 | } 32 | 33 | @GetMapping 34 | @ResponseStatus(HttpStatus.OK) 35 | public List getTweets() { 36 | return this.tweetMapper.convert(this.tweetService.getTweets()); 37 | } 38 | 39 | @PostMapping("/filter") 40 | @ResponseStatus(HttpStatus.OK) 41 | public List filterTweets(@RequestBody TweetFilter tweetFilter) { 42 | return this.tweetMapper 43 | .convert(this.tweetService.filterTweets(tweetFilter.getTimeFrom(), tweetFilter.getTimeTo(), tweetFilter.getSelectedTags(), tweetFilter.getCenter(), tweetFilter.getRadius())); 44 | } 45 | 46 | 47 | 48 | 49 | } 50 | -------------------------------------------------------------------------------- /tweet-collector/src/main/java/io/zmyzheng/collector/SocialMediaCollectionDriver.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.collector; 2 | 3 | import io.zmyzheng.collector.implementation.KafkaSink; 4 | import io.zmyzheng.collector.implementation.TweetCollector; 5 | import lombok.extern.slf4j.Slf4j; 6 | 7 | /** 8 | * @Author Mingyang Zheng 9 | * @Date 2021-01-02 20:34 10 | * @Version 3.0.0 11 | */ 12 | @Slf4j 13 | public class SocialMediaCollectionDriver { 14 | 15 | public static void main(String[] args) { 16 | 17 | String apiKey = System.getProperty("twitter.apiKey","application.properties"); 18 | String apiSecret = System.getProperty("twitter.apiSecret","application.properties"); 19 | String token = System.getProperty("twitter.token","application.properties"); 20 | String secret = System.getProperty("twitter.secret","application.properties"); 21 | 22 | String brokerList = System.getProperty("kafka.brokerList","application.properties"); 23 | String topic = System.getProperty("kafka.topic","application.properties"); 24 | 25 | 26 | 27 | SocialMediaCollector collector = new TweetCollector(apiKey, apiSecret, token, secret); 28 | 29 | Sinkable sink = new KafkaSink(brokerList, topic); 30 | 31 | collector.start(); 32 | sink.connect(); 33 | 34 | Runtime.getRuntime().addShutdownHook(new Thread(() -> { 35 | 36 | log.info("shutting down application..."); 37 | collector.stop(); 38 | sink.close(); 39 | log.info("shut down application: done!"); 40 | })); 41 | 42 | while (true) { 43 | String message = collector.collect(); 44 | sink.send(message); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/service/TopicServiceImpl.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.service; 2 | 3 | import io.zmyzheng.restapi.domain.HotTopicsTrend; 4 | import io.zmyzheng.restapi.domain.TopicStatistic; 5 | import io.zmyzheng.restapi.domain.TopicTrend; 6 | import io.zmyzheng.restapi.repository.EsOperationRepository; 7 | import lombok.extern.slf4j.Slf4j; 8 | import org.springframework.data.elasticsearch.core.geo.GeoPoint; 9 | import org.springframework.stereotype.Service; 10 | 11 | import java.util.Date; 12 | import java.util.List; 13 | 14 | /** 15 | * @Author Mingyang Zheng 16 | * @Date 2021-01-17 18:34 17 | * @Version 3.0.0 18 | */ 19 | 20 | @Service 21 | @Slf4j 22 | public class TopicServiceImpl implements TopicService { 23 | 24 | private final EsOperationRepository esOperationRepository; 25 | 26 | public TopicServiceImpl(EsOperationRepository esOperationRepository) { 27 | this.esOperationRepository = esOperationRepository; 28 | } 29 | 30 | @Override 31 | public List filterTopics(Date timeFrom, Date timeTo, GeoPoint center, String radius, int topN) { 32 | return this.esOperationRepository.filterTopics(timeFrom, timeTo, center, radius, topN); 33 | } 34 | 35 | @Override 36 | public List getTopicTrend(String topicName, String calendarInterval, Date timeFrom, Date timeTo, GeoPoint center, String radius) { 37 | return this.esOperationRepository.getTopicTrend(topicName, calendarInterval, timeFrom, timeTo, center, radius); 38 | } 39 | 40 | @Override 41 | public List getHotTopicsTrend(String calendarInterval, int topN, Date timeFrom, Date timeTo, GeoPoint center, String radius) { 42 | return this.esOperationRepository.getHotTopicsTrend(calendarInterval, topN, timeFrom, timeTo, center, radius); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /tweet-collector/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id 'application' 4 | id "com.github.johnrengelman.shadow" version "6.1.0" 5 | id "com.palantir.docker" version "0.25.0" 6 | id "com.gorylenko.gradle-git-properties" version "2.2.4" // auto generate git.properties and visible at /actuator/info 7 | } 8 | 9 | sourceCompatibility = '11' 10 | targetCompatibility = '11' 11 | 12 | 13 | 14 | mainClassName = 'io.zmyzheng.collector.SocialMediaCollectionDriver' 15 | 16 | dependencies { 17 | compileOnly 'org.projectlombok:lombok:1.18.16' 18 | annotationProcessor 'org.projectlombok:lombok:1.18.16' 19 | testCompileOnly 'org.projectlombok:lombok:1.18.16' 20 | testAnnotationProcessor 'org.projectlombok:lombok:1.18.16' 21 | // implementation group: 'org.yaml', name: 'snakeyaml', version: '1.25' 22 | implementation group: 'org.apache.kafka', name: 'kafka-clients', version: '2.6.0' 23 | implementation group: 'com.twitter', name: 'hbc-core', version: '2.2.0' 24 | 25 | // https://mvnrepository.com/artifact/ch.qos.logback/logback-classic 26 | implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' 27 | 28 | 29 | 30 | 31 | // testCompile group: 'junit', name: 'junit', version: '4.12' 32 | } 33 | 34 | shadowJar { 35 | mergeServiceFiles() 36 | // classifier = null 37 | zip64 = true 38 | } 39 | 40 | //jar { 41 | // manifest { 42 | // attributes 'Main-Class':'io.zmyzheng.SocialMediaCollectionDriver' 43 | // } 44 | // setArchiveClassifier(System.getProperty("buildTimestamp")) 45 | //} 46 | 47 | 48 | // dockerRepo is defined in gradle.properties 49 | docker { 50 | dependsOn jar 51 | name "${dockerRepo}/${jar.getArchiveBaseName().get()}:${jar.getArchiveVersion().get()}" //-${jar.getArchiveClassifier().get()} 52 | dockerfile file('Dockerfile') 53 | files jar.archivePath 54 | buildArgs(['JAR_FILE': "${jar.getArchiveFileName().get()}", 'APP_NAME': "${jar.getArchiveBaseName().get()}"]) 55 | pull false 56 | noCache true 57 | } 58 | -------------------------------------------------------------------------------- /rest-api-server/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'org.springframework.boot' version '2.4.1' 3 | id 'io.spring.dependency-management' version '1.0.10.RELEASE' 4 | id 'java' 5 | id "com.palantir.docker" version "0.25.0" 6 | id "com.gorylenko.gradle-git-properties" version "2.2.4" // auto generate git.properties and visible at /actuator/info 7 | } 8 | 9 | 10 | sourceCompatibility = '11' 11 | 12 | configurations { 13 | compileOnly { 14 | extendsFrom annotationProcessor 15 | } 16 | } 17 | 18 | 19 | 20 | ext { 21 | set('springBootAdminVersion', "2.3.1") 22 | set('mapstructVersion', "1.4.1.Final") 23 | } 24 | 25 | dependencies { 26 | implementation 'org.springframework.boot:spring-boot-starter-actuator' 27 | implementation 'org.springframework.boot:spring-boot-starter-data-elasticsearch' 28 | implementation 'org.springframework.boot:spring-boot-starter-web' 29 | implementation 'de.codecentric:spring-boot-admin-starter-client' 30 | compileOnly 'org.projectlombok:lombok' 31 | annotationProcessor 'org.projectlombok:lombok' 32 | testImplementation 'org.springframework.boot:spring-boot-starter-test' 33 | 34 | 35 | implementation "org.mapstruct:mapstruct:${mapstructVersion}" 36 | annotationProcessor "org.mapstruct:mapstruct-processor:${mapstructVersion}" 37 | // If you are using mapstruct in test code 38 | testAnnotationProcessor "org.mapstruct:mapstruct-processor:${mapstructVersion}" 39 | 40 | 41 | implementation 'org.springdoc:springdoc-openapi-ui:1.5.2' 42 | } 43 | 44 | dependencyManagement { 45 | imports { 46 | mavenBom "de.codecentric:spring-boot-admin-dependencies:${springBootAdminVersion}" 47 | } 48 | } 49 | 50 | test { 51 | useJUnitPlatform() 52 | } 53 | 54 | springBoot { 55 | buildInfo() // generate build info ate /actuator/info 56 | } 57 | 58 | // dockerRepo is defined in gradle.properties 59 | docker { 60 | dependsOn jar 61 | name "${dockerRepo}/${jar.getArchiveBaseName().get()}:${jar.getArchiveVersion().get()}" //-${jar.getArchiveClassifier().get()} 62 | dockerfile file('Dockerfile') 63 | files jar.archivePath 64 | buildArgs(['JAR_FILE': "${jar.getArchiveFileName().get()}", 'APP_NAME': "${jar.getArchiveBaseName().get()}"]) 65 | pull false 66 | noCache true 67 | } 68 | 69 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/map/visualization/visualization.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | import {FormBuilder} from '@angular/forms'; 3 | import {HttpClient} from '@angular/common/http'; 4 | import { Observable, of } from 'rxjs'; 5 | import { catchError, map } from 'rxjs/operators'; 6 | 7 | 8 | @Component({ 9 | selector: 'app-visualization', 10 | templateUrl: './visualization.component.html', 11 | styleUrls: ['./visualization.component.css'] 12 | }) 13 | export class VisualizationComponent implements OnInit { 14 | 15 | tweetFilterForm = this.formBuilder.group({ 16 | timeFrom: null, 17 | timeTo: null, 18 | selectedTags: [], 19 | center: this.formBuilder.group({ 20 | lat: null, 21 | lon: null 22 | }), 23 | radius: -1 24 | }); 25 | 26 | apiLoaded: Observable; 27 | center: google.maps.LatLngLiteral = {lat: 37.774546, lng: -122.433523}; 28 | zoom = 1; 29 | currentPosition: google.maps.LatLngLiteral = {lat: 37.774546, lng: -122.433523}; 30 | 31 | // heatmapOptions = {radius: 5}; 32 | // heatmapData = [ 33 | // {lat: 37.782, lng: -122.447}, 34 | // {lat: 37.782, lng: -122.445}, 35 | // {lat: 37.782, lng: -122.443}, 36 | // {lat: 37.782, lng: -122.441}, 37 | // {lat: 37.782, lng: -122.439}, 38 | // {lat: 37.782, lng: -122.437}, 39 | // {lat: 37.782, lng: -122.435}, 40 | // {lat: 37.785, lng: -122.447}, 41 | // {lat: 37.785, lng: -122.445}, 42 | // {lat: 37.785, lng: -122.443}, 43 | // {lat: 37.785, lng: -122.441}, 44 | // {lat: 37.785, lng: -122.439}, 45 | // {lat: 37.785, lng: -122.437}, 46 | // {lat: 37.785, lng: -122.435} 47 | // ]; 48 | 49 | 50 | constructor(private formBuilder: FormBuilder, private httpClient: HttpClient) { 51 | this.apiLoaded = httpClient.jsonp('https://maps.googleapis.com/maps/api/js?key=YOUR_API_KEY&libraries=visualization', 'callback') 52 | .pipe( 53 | map(() => true), 54 | catchError(() => of(false)), 55 | ); 56 | } 57 | 58 | ngOnInit(): void { 59 | } 60 | 61 | onSubmit(): void { 62 | console.warn(this.tweetFilterForm.value); 63 | } 64 | 65 | onClickMap(event: google.maps.MapMouseEvent): void { 66 | this.center = event.latLng.toJSON(); 67 | } 68 | 69 | onMapMouseMove(event: google.maps.MapMouseEvent): void { 70 | this.currentPosition = event.latLng.toJSON(); 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/app/map/visualization/visualization.component.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
Tweet Filter
5 |
6 |
7 |
8 | 9 | 10 |
11 | 12 |
13 | 14 | 15 |
16 |
17 | 18 | 19 |
20 | 21 | 27 |
28 | 29 |
30 | 31 |
32 |
33 | 34 |
35 |
36 | 37 |
38 |
39 |
40 | 41 |
42 | 43 | 44 |
45 | 46 | 47 |
48 |
49 | 50 |
51 |
52 | 56 | 57 | 58 |
59 |
60 | 61 |
62 |
Latitude: {{currentPosition?.lat}}
63 |
Longitude: {{currentPosition?.lng}}
64 |
65 | 66 |
67 | 68 |
69 | 70 | -------------------------------------------------------------------------------- /flink-processor/src/main/java/io/zmyzheng/processor/impl/TweetKafkaEsProcessor.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.processor.impl; 2 | 3 | import io.zmyzheng.processor.model.Tweet; 4 | import org.apache.flink.api.common.functions.FilterFunction; 5 | import org.apache.flink.api.common.functions.MapFunction; 6 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; 7 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; 8 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; 9 | import org.apache.flink.streaming.api.datastream.DataStream; 10 | import org.elasticsearch.common.geo.GeoPoint; 11 | 12 | import java.util.ArrayList; 13 | import java.util.Arrays; 14 | import java.util.List; 15 | 16 | import static java.util.Arrays.*; 17 | 18 | /** 19 | * @Author Mingyang Zheng 20 | * @Date 2021-01-05 23:30 21 | * @Version 3.0.0 22 | */ 23 | public class TweetKafkaEsProcessor extends KafkaEsProcessor { 24 | public TweetKafkaEsProcessor(String kafkaBootstrapServers, String kafkaTopic, String esUrl, String esIndexName) { 25 | super(kafkaBootstrapServers, kafkaTopic, esUrl, esIndexName); 26 | } 27 | 28 | 29 | @Override 30 | public DataStream defineProcessingLogic(DataStream sourceDataStream) { 31 | DataStream tweetDataStream = sourceDataStream.map(new MapFunction() { 32 | @Override 33 | public Tweet map(String value) { 34 | ObjectMapper objectMapper = new ObjectMapper(); 35 | try { 36 | JsonNode node = objectMapper.readTree(value); 37 | Tweet tweet = new Tweet(); 38 | tweet.setId(node.get("id_str").asText()); 39 | tweet.setTimestamp(Long.parseLong(node.get("timestamp_ms").asText())); 40 | 41 | ArrayNode arrayNode = (ArrayNode) node.get("coordinates").get("coordinates"); 42 | tweet.setCoordinate(new GeoPoint(arrayNode.get(1).asDouble(), arrayNode.get(0).asDouble())); 43 | 44 | arrayNode = (ArrayNode) node.get("entities").get("hashtags"); 45 | List tags = new ArrayList<>(); 46 | for (JsonNode item : arrayNode) { 47 | tags.add(item.get("text").asText()); 48 | } 49 | tweet.setHashTags(tags); 50 | return tweet; 51 | } catch (Exception e) { 52 | return null; 53 | } 54 | } 55 | }); 56 | 57 | DataStream validTweets = tweetDataStream.filter(new FilterFunction() { 58 | @Override 59 | public boolean filter(Tweet tweet) { 60 | return tweet != null && tweet.getHashTags().size() != 0; 61 | } 62 | }); 63 | return validTweets; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /tweet-collector/src/main/java/io/zmyzheng/collector/implementation/KafkaSink.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.collector.implementation; 2 | 3 | 4 | import io.zmyzheng.collector.Sinkable; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.apache.kafka.clients.producer.*; 7 | import org.apache.kafka.common.serialization.StringSerializer; 8 | 9 | import java.util.Properties; 10 | 11 | /** 12 | * @Author Mingyang Zheng 13 | * @Date 2021-01-02 19:48 14 | * @Version 3.0.0 15 | */ 16 | @Slf4j 17 | public class KafkaSink implements Sinkable { 18 | 19 | private String topic; 20 | private String brokerList; 21 | 22 | private KafkaProducer producer; 23 | 24 | public KafkaSink(String brokerList, String topic) { 25 | this.topic = topic; 26 | this.brokerList = brokerList; 27 | } 28 | 29 | @Override 30 | public void connect() { 31 | // create Producer properties 32 | Properties properties = new Properties(); 33 | properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, this.brokerList); 34 | properties.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 35 | properties.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 36 | 37 | // create safe Producer 38 | properties.setProperty(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true"); 39 | properties.setProperty(ProducerConfig.ACKS_CONFIG, "all"); 40 | properties.setProperty(ProducerConfig.RETRIES_CONFIG, Integer.toString(Integer.MAX_VALUE)); 41 | properties.setProperty(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "5"); // kafka 2.0 >= 1.1 so we can keep this as 5. Use 1 otherwise. 42 | 43 | // high throughput producer (at the expense of a bit of latency and CPU usage) 44 | properties.setProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG, "snappy"); 45 | properties.setProperty(ProducerConfig.LINGER_MS_CONFIG, "20"); 46 | properties.setProperty(ProducerConfig.BATCH_SIZE_CONFIG, Integer.toString(32*1024)); // 32 KB batch size 47 | 48 | // create the producer 49 | this.producer = new KafkaProducer(properties); 50 | 51 | } 52 | 53 | @Override 54 | public void send(String data) { 55 | if (data != null){ 56 | // log.debug(msg); 57 | producer.send(new ProducerRecord<>(topic, null, data), new Callback() { 58 | @Override 59 | public void onCompletion(RecordMetadata recordMetadata, Exception e) { 60 | if (e != null) { 61 | log.error("Something bad happened", e); 62 | } 63 | } 64 | }); 65 | } 66 | 67 | } 68 | 69 | @Override 70 | public void close() { 71 | log.info("shutting down Kafka Producer..."); 72 | this.producer.close(); 73 | log.info("Close Kafka Producer: done!"); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/api/controller/TopicController.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.api.controller; 2 | 3 | import io.zmyzheng.restapi.api.mapping.HotTopicsTrendMapper; 4 | import io.zmyzheng.restapi.api.mapping.TopicStatisticMapper; 5 | import io.zmyzheng.restapi.api.mapping.TopicTrendMapper; 6 | import io.zmyzheng.restapi.api.model.*; 7 | import io.zmyzheng.restapi.service.TopicService; 8 | import lombok.extern.slf4j.Slf4j; 9 | import org.springframework.http.HttpStatus; 10 | import org.springframework.web.bind.annotation.*; 11 | 12 | import java.util.List; 13 | 14 | /** 15 | * @Author Mingyang Zheng 16 | * @Date 2021-01-17 18:39 17 | * @Version 3.0.0 18 | */ 19 | 20 | @Slf4j 21 | @RestController 22 | @RequestMapping(TopicController.BASE_URL) 23 | public class TopicController { 24 | public static final String BASE_URL = "api/v3/topics"; 25 | 26 | private final TopicService topicService; 27 | private final TopicStatisticMapper topicStatisticMapper; 28 | private final TopicTrendMapper topicTrendMapper; 29 | private final HotTopicsTrendMapper hotTopicsTrendMapper; 30 | 31 | public TopicController(TopicService topicService, TopicStatisticMapper topicStatisticMapper, TopicTrendMapper topicTrendMapper, HotTopicsTrendMapper hotTopicsTrendMapper) { 32 | this.topicService = topicService; 33 | this.topicStatisticMapper = topicStatisticMapper; 34 | this.topicTrendMapper = topicTrendMapper; 35 | this.hotTopicsTrendMapper = hotTopicsTrendMapper; 36 | } 37 | 38 | @PostMapping("/statistic") 39 | @ResponseStatus(HttpStatus.OK) 40 | public List getTopicStatistic(@RequestBody TopicStatisticFilter topicStatisticFilter) { 41 | return this.topicStatisticMapper 42 | .convert(this.topicService.filterTopics(topicStatisticFilter.getTimeFrom(), topicStatisticFilter.getTimeTo(), topicStatisticFilter.getCenter(), topicStatisticFilter.getRadius(), topicStatisticFilter.getTopN())); 43 | } 44 | 45 | @PostMapping("/{topicName}/trend") 46 | @ResponseStatus(HttpStatus.OK) 47 | public List filterTrendByTopic(@RequestParam String topicName, @RequestBody TopicTrendFilter topicTrendFilter) { 48 | return this.topicTrendMapper 49 | .convert(this.topicService.getTopicTrend(topicName, topicTrendFilter.getCalendarInterval(), topicTrendFilter.getTimeFrom(), topicTrendFilter.getTimeTo(), topicTrendFilter.getCenter(), topicTrendFilter.getRadius())); 50 | } 51 | 52 | @PostMapping("/trend") 53 | @ResponseStatus(HttpStatus.OK) 54 | public List getHotTopicsTrend(@RequestBody HotTopicsTrendFilter hotTopicsTrendFilter) { 55 | return this.hotTopicsTrendMapper 56 | .convert(this.topicService.getHotTopicsTrend(hotTopicsTrendFilter.getCalendarInterval(), hotTopicsTrendFilter.getTopN(), hotTopicsTrendFilter.getTimeFrom(), hotTopicsTrendFilter.getTimeTo(), hotTopicsTrendFilter.getCenter(), hotTopicsTrendFilter.getRadius())); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /tweet-collector/src/main/java/io/zmyzheng/collector/implementation/TweetCollector.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.collector.implementation; 2 | 3 | import com.twitter.hbc.ClientBuilder; 4 | import com.twitter.hbc.core.Client; 5 | import com.twitter.hbc.core.Constants; 6 | import com.twitter.hbc.core.Hosts; 7 | import com.twitter.hbc.core.HttpHosts; 8 | import com.twitter.hbc.core.endpoint.StatusesSampleEndpoint; 9 | import com.twitter.hbc.core.endpoint.StreamingEndpoint; 10 | import com.twitter.hbc.core.processor.StringDelimitedProcessor; 11 | import com.twitter.hbc.httpclient.auth.Authentication; 12 | import com.twitter.hbc.httpclient.auth.OAuth1; 13 | import io.zmyzheng.collector.SocialMediaCollector; 14 | import lombok.extern.slf4j.Slf4j; 15 | 16 | import java.util.concurrent.BlockingQueue; 17 | import java.util.concurrent.LinkedBlockingQueue; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | /** 21 | * @Author Mingyang Zheng 22 | * @Date 2021-01-02 20:12 23 | * @Version 3.0.0 24 | */ 25 | @Slf4j 26 | public class TweetCollector implements SocialMediaCollector { 27 | 28 | private BlockingQueue msgQueue; 29 | 30 | private Client twitterClient; 31 | 32 | public TweetCollector(String apiKey, String apiSecret, String token, String secret) { 33 | /** Set up your blocking queues: Be sure to size these properly based on expected TPS of your stream */ 34 | this.msgQueue = new LinkedBlockingQueue(1000); 35 | this.twitterClient = createTwitterClient(msgQueue, apiKey, apiSecret, token, secret); 36 | } 37 | 38 | private Client createTwitterClient(BlockingQueue msgQueue, String apiKey, String apiSecret, String token, String secret){ 39 | 40 | /** Declare the host you want to connect to, the endpoint, and authentication (basic auth or oauth) */ 41 | Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST); 42 | StreamingEndpoint endpoint = new StatusesSampleEndpoint(); 43 | 44 | // hosebirdEndpoint.trackTerms(terms); 45 | // These secrets should be read from a config file 46 | Authentication hosebirdAuth = new OAuth1(apiKey, apiSecret, token, secret); 47 | 48 | ClientBuilder builder = new ClientBuilder() 49 | .name("Hosebird-Client-01") // optional: mainly for the logs 50 | .hosts(hosebirdHosts) 51 | .authentication(hosebirdAuth) 52 | .endpoint(endpoint) 53 | .processor(new StringDelimitedProcessor(msgQueue)); 54 | 55 | Client hosebirdClient = builder.build(); 56 | return hosebirdClient; 57 | } 58 | 59 | @Override 60 | public void start() { 61 | twitterClient.connect(); 62 | } 63 | 64 | @Override 65 | public String collect() { 66 | String msg = null; 67 | try { 68 | msg = msgQueue.poll(5, TimeUnit.SECONDS); 69 | } catch (InterruptedException e) { 70 | log.error(e.getMessage()); 71 | } 72 | return msg; 73 | } 74 | 75 | @Override 76 | public void stop() { 77 | log.info("shutting down client from twitter..."); 78 | this.twitterClient.stop(); 79 | log.info("shutting down client from twitter: done!"); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /tweet-map-frontend/src/polyfills.ts: -------------------------------------------------------------------------------- 1 | /*************************************************************************************************** 2 | * Load `$localize` onto the global scope - used if i18n tags appear in Angular templates. 3 | */ 4 | import '@angular/localize/init'; 5 | /** 6 | * This file includes polyfills needed by Angular and is loaded before the app. 7 | * You can add your own extra polyfills to this file. 8 | * 9 | * This file is divided into 2 sections: 10 | * 1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers. 11 | * 2. Application imports. Files imported after ZoneJS that should be loaded before your main 12 | * file. 13 | * 14 | * The current setup is for so-called "evergreen" browsers; the last versions of browsers that 15 | * automatically update themselves. This includes Safari >= 10, Chrome >= 55 (including Opera), 16 | * Edge >= 13 on the desktop, and iOS 10 and Chrome on mobile. 17 | * 18 | * Learn more in https://angular.io/guide/browser-support 19 | */ 20 | 21 | /*************************************************************************************************** 22 | * BROWSER POLYFILLS 23 | */ 24 | 25 | /** 26 | * IE11 requires the following for NgClass support on SVG elements 27 | */ 28 | // import 'classlist.js'; // Run `npm install --save classlist.js`. 29 | 30 | /** 31 | * Web Animations `@angular/platform-browser/animations` 32 | * Only required if AnimationBuilder is used within the application and using IE/Edge or Safari. 33 | * Standard animation support in Angular DOES NOT require any polyfills (as of Angular 6.0). 34 | */ 35 | // import 'web-animations-js'; // Run `npm install --save web-animations-js`. 36 | 37 | /** 38 | * By default, zone.js will patch all possible macroTask and DomEvents 39 | * user can disable parts of macroTask/DomEvents patch by setting following flags 40 | * because those flags need to be set before `zone.js` being loaded, and webpack 41 | * will put import in the top of bundle, so user need to create a separate file 42 | * in this directory (for example: zone-flags.ts), and put the following flags 43 | * into that file, and then add the following code before importing zone.js. 44 | * import './zone-flags'; 45 | * 46 | * The flags allowed in zone-flags.ts are listed here. 47 | * 48 | * The following flags will work for all browsers. 49 | * 50 | * (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame 51 | * (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick 52 | * (window as any).__zone_symbol__UNPATCHED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames 53 | * 54 | * in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js 55 | * with the following flag, it will bypass `zone.js` patch for IE/Edge 56 | * 57 | * (window as any).__Zone_enable_cross_context_check = true; 58 | * 59 | */ 60 | 61 | /*************************************************************************************************** 62 | * Zone JS is required by default for Angular itself. 63 | */ 64 | import 'zone.js/dist/zone'; // Included with Angular CLI. 65 | 66 | 67 | /*************************************************************************************************** 68 | * APPLICATION IMPORTS 69 | */ 70 | -------------------------------------------------------------------------------- /tweet-map-frontend/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "tslint:recommended", 3 | "rulesDirectory": [ 4 | "codelyzer" 5 | ], 6 | "rules": { 7 | "align": { 8 | "options": [ 9 | "parameters", 10 | "statements" 11 | ] 12 | }, 13 | "array-type": false, 14 | "arrow-return-shorthand": true, 15 | "curly": true, 16 | "deprecation": { 17 | "severity": "warning" 18 | }, 19 | "eofline": true, 20 | "import-blacklist": [ 21 | true, 22 | "rxjs/Rx" 23 | ], 24 | "import-spacing": true, 25 | "indent": { 26 | "options": [ 27 | "spaces" 28 | ] 29 | }, 30 | "max-classes-per-file": false, 31 | "max-line-length": [ 32 | true, 33 | 140 34 | ], 35 | "member-ordering": [ 36 | true, 37 | { 38 | "order": [ 39 | "static-field", 40 | "instance-field", 41 | "static-method", 42 | "instance-method" 43 | ] 44 | } 45 | ], 46 | "no-console": [ 47 | true, 48 | "debug", 49 | "info", 50 | "time", 51 | "timeEnd", 52 | "trace" 53 | ], 54 | "no-empty": false, 55 | "no-inferrable-types": [ 56 | true, 57 | "ignore-params" 58 | ], 59 | "no-non-null-assertion": true, 60 | "no-redundant-jsdoc": true, 61 | "no-switch-case-fall-through": true, 62 | "no-var-requires": false, 63 | "object-literal-key-quotes": [ 64 | true, 65 | "as-needed" 66 | ], 67 | "quotemark": [ 68 | true, 69 | "single" 70 | ], 71 | "semicolon": { 72 | "options": [ 73 | "always" 74 | ] 75 | }, 76 | "space-before-function-paren": { 77 | "options": { 78 | "anonymous": "never", 79 | "asyncArrow": "always", 80 | "constructor": "never", 81 | "method": "never", 82 | "named": "never" 83 | } 84 | }, 85 | "typedef": [ 86 | true, 87 | "call-signature" 88 | ], 89 | "typedef-whitespace": { 90 | "options": [ 91 | { 92 | "call-signature": "nospace", 93 | "index-signature": "nospace", 94 | "parameter": "nospace", 95 | "property-declaration": "nospace", 96 | "variable-declaration": "nospace" 97 | }, 98 | { 99 | "call-signature": "onespace", 100 | "index-signature": "onespace", 101 | "parameter": "onespace", 102 | "property-declaration": "onespace", 103 | "variable-declaration": "onespace" 104 | } 105 | ] 106 | }, 107 | "variable-name": { 108 | "options": [ 109 | "ban-keywords", 110 | "check-format", 111 | "allow-pascal-case" 112 | ] 113 | }, 114 | "whitespace": { 115 | "options": [ 116 | "check-branch", 117 | "check-decl", 118 | "check-operator", 119 | "check-separator", 120 | "check-type", 121 | "check-typecast" 122 | ] 123 | }, 124 | "component-class-suffix": true, 125 | "contextual-lifecycle": true, 126 | "directive-class-suffix": true, 127 | "no-conflicting-lifecycle": true, 128 | "no-host-metadata-property": true, 129 | "no-input-rename": true, 130 | "no-inputs-metadata-property": true, 131 | "no-output-native": true, 132 | "no-output-on-prefix": true, 133 | "no-output-rename": true, 134 | "no-outputs-metadata-property": true, 135 | "template-banana-in-box": true, 136 | "template-no-negated-async": true, 137 | "use-lifecycle-interface": true, 138 | "use-pipe-transform-interface": true, 139 | "directive-selector": [ 140 | true, 141 | "attribute", 142 | "app", 143 | "camelCase" 144 | ], 145 | "component-selector": [ 146 | true, 147 | "element", 148 | "app", 149 | "kebab-case" 150 | ] 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /tweet-map-frontend/angular.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "./node_modules/@angular/cli/lib/config/schema.json", 3 | "version": 1, 4 | "newProjectRoot": "projects", 5 | "projects": { 6 | "tweet-map-frontend": { 7 | "projectType": "application", 8 | "schematics": { 9 | "@schematics/angular:application": { 10 | "strict": true 11 | } 12 | }, 13 | "root": "", 14 | "sourceRoot": "src", 15 | "prefix": "app", 16 | "architect": { 17 | "build": { 18 | "builder": "@angular-devkit/build-angular:browser", 19 | "options": { 20 | "outputPath": "dist/tweet-map-frontend", 21 | "index": "src/index.html", 22 | "main": "src/main.ts", 23 | "polyfills": "src/polyfills.ts", 24 | "tsConfig": "tsconfig.app.json", 25 | "aot": true, 26 | "assets": [ 27 | "src/favicon.ico", 28 | "src/assets" 29 | ], 30 | "styles": [ 31 | "node_modules/bootstrap/dist/css/bootstrap.min.css", 32 | "src/styles.css" 33 | ], 34 | "scripts": [] 35 | }, 36 | "configurations": { 37 | "production": { 38 | "fileReplacements": [ 39 | { 40 | "replace": "src/environments/environment.ts", 41 | "with": "src/environments/environment.prod.ts" 42 | } 43 | ], 44 | "optimization": true, 45 | "outputHashing": "all", 46 | "sourceMap": false, 47 | "namedChunks": false, 48 | "extractLicenses": true, 49 | "vendorChunk": false, 50 | "buildOptimizer": true, 51 | "budgets": [ 52 | { 53 | "type": "initial", 54 | "maximumWarning": "500kb", 55 | "maximumError": "1mb" 56 | }, 57 | { 58 | "type": "anyComponentStyle", 59 | "maximumWarning": "2kb", 60 | "maximumError": "4kb" 61 | } 62 | ] 63 | } 64 | } 65 | }, 66 | "serve": { 67 | "builder": "@angular-devkit/build-angular:dev-server", 68 | "options": { 69 | "browserTarget": "tweet-map-frontend:build" 70 | }, 71 | "configurations": { 72 | "production": { 73 | "browserTarget": "tweet-map-frontend:build:production" 74 | } 75 | } 76 | }, 77 | "extract-i18n": { 78 | "builder": "@angular-devkit/build-angular:extract-i18n", 79 | "options": { 80 | "browserTarget": "tweet-map-frontend:build" 81 | } 82 | }, 83 | "test": { 84 | "builder": "@angular-devkit/build-angular:karma", 85 | "options": { 86 | "main": "src/test.ts", 87 | "polyfills": "src/polyfills.ts", 88 | "tsConfig": "tsconfig.spec.json", 89 | "karmaConfig": "karma.conf.js", 90 | "assets": [ 91 | "src/favicon.ico", 92 | "src/assets" 93 | ], 94 | "styles": [ 95 | "src/styles.css" 96 | ], 97 | "scripts": [] 98 | } 99 | }, 100 | "lint": { 101 | "builder": "@angular-devkit/build-angular:tslint", 102 | "options": { 103 | "tsConfig": [ 104 | "tsconfig.app.json", 105 | "tsconfig.spec.json", 106 | "e2e/tsconfig.json" 107 | ], 108 | "exclude": [ 109 | "**/node_modules/**" 110 | ] 111 | } 112 | }, 113 | "e2e": { 114 | "builder": "@angular-devkit/build-angular:protractor", 115 | "options": { 116 | "protractorConfig": "e2e/protractor.conf.js", 117 | "devServerTarget": "tweet-map-frontend:serve" 118 | }, 119 | "configurations": { 120 | "production": { 121 | "devServerTarget": "tweet-map-frontend:serve:production" 122 | } 123 | } 124 | } 125 | } 126 | } 127 | }, 128 | "defaultProject": "tweet-map-frontend" 129 | } 130 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | 86 | # Determine the Java command to use to start the JVM. 87 | if [ -n "$JAVA_HOME" ] ; then 88 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 89 | # IBM's JDK on AIX uses strange locations for the executables 90 | JAVACMD="$JAVA_HOME/jre/sh/java" 91 | else 92 | JAVACMD="$JAVA_HOME/bin/java" 93 | fi 94 | if [ ! -x "$JAVACMD" ] ; then 95 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 96 | 97 | Please set the JAVA_HOME variable in your environment to match the 98 | location of your Java installation." 99 | fi 100 | else 101 | JAVACMD="java" 102 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 103 | 104 | Please set the JAVA_HOME variable in your environment to match the 105 | location of your Java installation." 106 | fi 107 | 108 | # Increase the maximum file descriptors if we can. 109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 110 | MAX_FD_LIMIT=`ulimit -H -n` 111 | if [ $? -eq 0 ] ; then 112 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 113 | MAX_FD="$MAX_FD_LIMIT" 114 | fi 115 | ulimit -n $MAX_FD 116 | if [ $? -ne 0 ] ; then 117 | warn "Could not set maximum file descriptor limit: $MAX_FD" 118 | fi 119 | else 120 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 121 | fi 122 | fi 123 | 124 | # For Darwin, add options to specify how the application appears in the dock 125 | if $darwin; then 126 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 127 | fi 128 | 129 | # For Cygwin or MSYS, switch paths to Windows format before running java 130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 131 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 132 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 133 | 134 | JAVACMD=`cygpath --unix "$JAVACMD"` 135 | 136 | # We build the pattern for arguments to be converted via cygpath 137 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 138 | SEP="" 139 | for dir in $ROOTDIRSRAW ; do 140 | ROOTDIRS="$ROOTDIRS$SEP$dir" 141 | SEP="|" 142 | done 143 | OURCYGPATTERN="(^($ROOTDIRS))" 144 | # Add a user-defined pattern to the cygpath arguments 145 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 146 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 147 | fi 148 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 149 | i=0 150 | for arg in "$@" ; do 151 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 152 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 153 | 154 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 155 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 156 | else 157 | eval `echo args$i`="\"$arg\"" 158 | fi 159 | i=`expr $i + 1` 160 | done 161 | case $i in 162 | 0) set -- ;; 163 | 1) set -- "$args0" ;; 164 | 2) set -- "$args0" "$args1" ;; 165 | 3) set -- "$args0" "$args1" "$args2" ;; 166 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;; 167 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 168 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 169 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 170 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 171 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 172 | esac 173 | fi 174 | 175 | # Escape application args 176 | save () { 177 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 178 | echo " " 179 | } 180 | APP_ARGS=`save "$@"` 181 | 182 | # Collect all arguments for the java command, following the shell quoting and substitution rules 183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 184 | 185 | exec "$JAVACMD" "$@" 186 | -------------------------------------------------------------------------------- /rest-api-server/src/main/java/io/zmyzheng/restapi/repository/EsOperationRepository.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.restapi.repository; 2 | 3 | import io.zmyzheng.restapi.domain.HotTopicsTrend; 4 | import io.zmyzheng.restapi.domain.TopicStatistic; 5 | import io.zmyzheng.restapi.domain.TopicTrend; 6 | import io.zmyzheng.restapi.domain.Tweet; 7 | import lombok.extern.slf4j.Slf4j; 8 | import org.elasticsearch.index.query.BoolQueryBuilder; 9 | import org.elasticsearch.index.query.QueryBuilders; 10 | import org.elasticsearch.search.aggregations.AggregationBuilders; 11 | import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; 12 | import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; 13 | import org.elasticsearch.search.aggregations.bucket.terms.Terms; 14 | import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; 15 | import org.springframework.data.elasticsearch.core.SearchHit; 16 | import org.springframework.data.elasticsearch.core.geo.GeoPoint; 17 | import org.springframework.data.elasticsearch.core.query.*; 18 | import org.springframework.stereotype.Repository; 19 | 20 | import java.time.Instant; 21 | import java.util.Date; 22 | import java.util.List; 23 | import java.util.stream.Collectors; 24 | 25 | /** 26 | * @Author: Mingyang Zheng 27 | * @Date: 2020-02-23 14:09 28 | * @Version 3.0.0 29 | * 30 | * @Description: this class defines operations that are not covered in standard Spring Data Repositories 31 | */ 32 | 33 | @Slf4j 34 | @Repository 35 | public class EsOperationRepository { 36 | 37 | private final ElasticsearchRestTemplate elasticsearchRestTemplate; 38 | 39 | public EsOperationRepository(ElasticsearchRestTemplate elasticsearchRestTemplate) { 40 | this.elasticsearchRestTemplate = elasticsearchRestTemplate; 41 | } 42 | 43 | private Criteria createFilterCriteria(Date timeFrom, Date timeTo, List selectedTags, GeoPoint center, String radius) { 44 | Criteria criteria = new Criteria("timestamp").greaterThanEqual(timeFrom).lessThanEqual(timeTo); 45 | if (selectedTags != null) { 46 | criteria.and("hashTags").in(selectedTags); 47 | } 48 | if (center != null) { 49 | criteria.and("coordinate").within(center, radius); 50 | } 51 | return criteria; 52 | } 53 | 54 | public List filterTweets(Date timeFrom, Date timeTo, List selectedTags, GeoPoint center, String radius) { 55 | Criteria criteria = createFilterCriteria(timeFrom, timeTo, selectedTags, center, radius); 56 | Query query = new CriteriaQuery(criteria); 57 | return this.elasticsearchRestTemplate.search(query, Tweet.class) 58 | .get() 59 | .map(SearchHit::getContent) 60 | .collect(Collectors.toList()); 61 | } 62 | 63 | 64 | public List filterTopics(Date timeFrom, Date timeTo, GeoPoint center, String radius, int topN) { 65 | String aggregationName = "topics"; 66 | BoolQueryBuilder builder = QueryBuilders.boolQuery() 67 | .filter(QueryBuilders.rangeQuery("timestamp").gte(timeFrom).lte(timeTo)); 68 | 69 | if (center != null) { 70 | builder.must(QueryBuilders.geoDistanceQuery("coordinate").distance(radius)); 71 | } 72 | 73 | Query query = new NativeSearchQueryBuilder() 74 | .withQuery(builder) 75 | .addAggregation(AggregationBuilders.terms(aggregationName).field("hashTags").size(topN)) // By default, the buckets are ordered by their doc_count descending. https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html#search-aggregations-bucket-terms-aggregation-order 76 | .build(); 77 | 78 | return this.elasticsearchRestTemplate.search(query, Tweet.class) 79 | .getAggregations() 80 | .get(aggregationName) 81 | .getBuckets() 82 | .stream() 83 | .map(bucket -> new TopicStatistic(bucket.getKeyAsString(), bucket.getDocCount())) 84 | .collect(Collectors.toList()); 85 | } 86 | 87 | 88 | public List getTopicTrend(String topicName, String calendarInterval, Date timeFrom, Date timeTo, GeoPoint center, String radius) { 89 | String aggregationName = "topicTrend"; 90 | BoolQueryBuilder builder = QueryBuilders.boolQuery() 91 | .filter(QueryBuilders.rangeQuery("timestamp").gte(timeFrom).lte(timeTo)) 92 | .filter(QueryBuilders.termQuery("hashTags", topicName)); 93 | 94 | if (center != null) { 95 | builder.must(QueryBuilders.geoDistanceQuery("coordinate").distance(radius)); 96 | } 97 | 98 | Query query = new NativeSearchQueryBuilder() 99 | .withQuery(builder) 100 | .addAggregation(AggregationBuilders.dateHistogram(aggregationName).field("timestamp").calendarInterval(new DateHistogramInterval(calendarInterval))) 101 | .build(); 102 | return this.elasticsearchRestTemplate.search(query, Tweet.class) 103 | .getAggregations() 104 | .get(aggregationName) 105 | .getBuckets() 106 | .stream() 107 | .map(bucket -> new TopicTrend(new Date(((Instant) bucket.getKey()).toEpochMilli()), bucket.getDocCount())) 108 | .collect(Collectors.toList()); 109 | } 110 | 111 | 112 | public List getHotTopicsTrend(String calendarInterval, int topN, Date timeFrom, Date timeTo, GeoPoint center, String radius) { 113 | String aggregationName = "hotTopicsTrend"; 114 | String subAggregationName = "topics"; 115 | BoolQueryBuilder builder = QueryBuilders.boolQuery() 116 | .filter(QueryBuilders.rangeQuery("timestamp").gte(timeFrom).lte(timeTo)); 117 | 118 | if (center != null) { 119 | builder.must(QueryBuilders.geoDistanceQuery("coordinate").distance(radius)); 120 | } 121 | 122 | Query query = new NativeSearchQueryBuilder() 123 | .withQuery(builder) 124 | .addAggregation(AggregationBuilders.dateHistogram(aggregationName).field("timestamp").calendarInterval(new DateHistogramInterval(calendarInterval)) 125 | .subAggregation(AggregationBuilders.terms(subAggregationName).field("hashTags").size(topN))) 126 | .build(); 127 | 128 | return this.elasticsearchRestTemplate.search(query, Tweet.class) 129 | .getAggregations() 130 | .get(aggregationName) 131 | .getBuckets() 132 | .stream() 133 | .map(bucket -> { 134 | Date date = new Date(((Instant) bucket.getKey()).toEpochMilli()); 135 | List hotTopics = bucket.getAggregations().get(subAggregationName).getBuckets() 136 | .stream().map(subBucket -> new TopicStatistic(subBucket.getKeyAsString(), subBucket.getDocCount())).collect(Collectors.toList()); 137 | return new HotTopicsTrend(date, hotTopics); 138 | }) 139 | .collect(Collectors.toList()); 140 | } 141 | 142 | 143 | 144 | } 145 | -------------------------------------------------------------------------------- /flink-processor/src/main/java/io/zmyzheng/processor/impl/KafkaEsProcessor.java: -------------------------------------------------------------------------------- 1 | package io.zmyzheng.processor.impl; 2 | 3 | import io.zmyzheng.processor.StreamProcessor; 4 | import io.zmyzheng.processor.model.Tweet; 5 | import io.zmyzheng.processor.model.UniqueEntity; 6 | import org.apache.flink.api.common.functions.FilterFunction; 7 | import org.apache.flink.api.common.functions.MapFunction; 8 | import org.apache.flink.api.common.functions.RuntimeContext; 9 | import org.apache.flink.api.common.restartstrategy.RestartStrategies; 10 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 11 | import org.apache.flink.api.common.time.Time; 12 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; 13 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; 14 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; 15 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; 16 | import org.apache.flink.streaming.api.CheckpointingMode; 17 | import org.apache.flink.streaming.api.datastream.DataStream; 18 | import org.apache.flink.streaming.api.environment.CheckpointConfig; 19 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 20 | import org.apache.flink.streaming.connectors.elasticsearch.ActionRequestFailureHandler; 21 | import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction; 22 | import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer; 23 | import org.apache.flink.streaming.connectors.elasticsearch7.ElasticsearchSink; 24 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; 25 | import org.apache.flink.util.ExceptionUtils; 26 | import org.apache.http.HttpHost; 27 | import org.elasticsearch.ElasticsearchParseException; 28 | import org.elasticsearch.action.ActionRequest; 29 | import org.elasticsearch.action.index.IndexRequest; 30 | import org.elasticsearch.client.Requests; 31 | import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; 32 | import org.elasticsearch.common.xcontent.XContentType; 33 | 34 | import java.net.MalformedURLException; 35 | import java.net.URL; 36 | import java.util.ArrayList; 37 | import java.util.List; 38 | import java.util.Properties; 39 | import java.util.concurrent.TimeUnit; 40 | 41 | /** 42 | * @Author Mingyang Zheng 43 | * @Date 2021-01-05 22:18 44 | * @Version 1.0.0 45 | */ 46 | public abstract class KafkaEsProcessor> implements StreamProcessor { 47 | 48 | private String kafkaBootstrapServers; 49 | private String kafkaTopic; 50 | private String esUrl; 51 | private String esIndexName; 52 | 53 | 54 | private StreamExecutionEnvironment env; 55 | private DataStream sourceDataStream; 56 | private DataStream processedDataStream; 57 | 58 | public KafkaEsProcessor(String kafkaBootstrapServers, String kafkaTopic, String esUrl, String esIndexName) { 59 | this.kafkaBootstrapServers = kafkaBootstrapServers; 60 | this.kafkaTopic = kafkaTopic; 61 | this.esUrl = esUrl; 62 | this.esIndexName = esIndexName; 63 | } 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | @Override 74 | public void configureStreamExecutionEnvironment() { 75 | env = StreamExecutionEnvironment.getExecutionEnvironment(); 76 | env.setRestartStrategy(RestartStrategies.fixedDelayRestart( 77 | 3, // number of restart attempts 78 | Time.of(10, TimeUnit.SECONDS) // delay 79 | )); 80 | env.enableCheckpointing(5000); 81 | // advanced options: 82 | // set mode to exactly-once (this is the default) 83 | env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE); 84 | // make sure 500 ms of progress happen between checkpoints 85 | env.getCheckpointConfig().setMinPauseBetweenCheckpoints(500); 86 | // checkpoints have to complete within one minute, or are discarded 87 | env.getCheckpointConfig().setCheckpointTimeout(60000); 88 | // allow only one checkpoint to be in progress at the same time 89 | env.getCheckpointConfig().setMaxConcurrentCheckpoints(1); 90 | // enable externalized checkpoints which are retained after job cancellation 91 | env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); 92 | // allow job recovery fallback to checkpoint when there is a more recent savepoint 93 | // env.getCheckpointConfig().setPreferCheckpointForRecovery(true); // deprecated 94 | } 95 | 96 | @Override 97 | public void addSource() { 98 | Properties properties = new Properties(); 99 | properties.setProperty("bootstrap.servers", kafkaBootstrapServers); 100 | properties.setProperty("group.id", this.getClass().getName()); 101 | this.sourceDataStream = env.addSource(new FlinkKafkaConsumer(kafkaTopic, new SimpleStringSchema(), properties)); 102 | 103 | 104 | } 105 | 106 | @Override 107 | public void defineProcessingLogic() { 108 | DataStream input = this.sourceDataStream; 109 | this.processedDataStream = defineProcessingLogic(input); 110 | 111 | } 112 | 113 | public abstract DataStream defineProcessingLogic(DataStream sourceDataStream); 114 | 115 | 116 | 117 | @Override 118 | public void addSink() throws Exception { 119 | 120 | this.processedDataStream.addSink(createEsSink()); 121 | 122 | } 123 | 124 | private ElasticsearchSink createEsSink() throws MalformedURLException { 125 | URL url = new URL(esUrl); 126 | String hostname = url.getHost(); 127 | int port = url.getPort(); 128 | String protocol = url.getProtocol(); 129 | 130 | List httpHosts = new ArrayList<>(); 131 | httpHosts.add(new HttpHost(hostname, port, protocol)); 132 | 133 | 134 | // use a ElasticsearchSink.Builder to create an ElasticsearchSink 135 | ElasticsearchSink.Builder esSinkBuilder = new ElasticsearchSink.Builder<>( 136 | httpHosts, 137 | new ElasticsearchSinkFunction() { 138 | public IndexRequest createIndexRequest(T element) throws JsonProcessingException { 139 | ObjectMapper objectMapper = new ObjectMapper(); 140 | return Requests.indexRequest() 141 | .index(esIndexName) 142 | .id(element.getUniqueKey()) 143 | .source(objectMapper.writeValueAsBytes(element), XContentType.JSON); 144 | } 145 | 146 | @Override 147 | public void process(T element, RuntimeContext ctx, RequestIndexer indexer) { 148 | 149 | try { 150 | indexer.add(createIndexRequest(element)); 151 | } catch (JsonProcessingException e) { 152 | throw new RuntimeException(e); 153 | } 154 | 155 | } 156 | } 157 | ); 158 | esSinkBuilder.setFailureHandler(new ActionRequestFailureHandler() { 159 | @Override 160 | public void onFailure(ActionRequest action, Throwable failure, int restStatusCode, RequestIndexer indexer) throws Throwable { 161 | if (ExceptionUtils.findThrowable(failure, EsRejectedExecutionException.class).isPresent()) { 162 | // full queue; re-add document for indexing 163 | indexer.add(action); 164 | } else if (ExceptionUtils.findThrowable(failure, ElasticsearchParseException.class).isPresent()) { 165 | // malformed document; simply drop request without failing sink 166 | } else { 167 | // for all other failures, fail the sink 168 | // here the failure is simply rethrown, but users can also choose to throw custom exceptions 169 | throw failure; 170 | } 171 | 172 | } 173 | }); 174 | return esSinkBuilder.build(); 175 | } 176 | 177 | @Override 178 | public void process() throws Exception { 179 | env.execute(); 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------