17 |
--------------------------------------------------------------------------------
/front-end/src/app/app.component.spec.ts:
--------------------------------------------------------------------------------
1 | import { TestBed, async } from '@angular/core/testing';
2 | import { RouterTestingModule } from '@angular/router/testing';
3 | import { AppComponent } from './app.component';
4 |
5 | describe('AppComponent', () => {
6 | beforeEach(async(() => {
7 | TestBed.configureTestingModule({
8 | imports: [
9 | RouterTestingModule
10 | ],
11 | declarations: [
12 | AppComponent
13 | ],
14 | }).compileComponents();
15 | }));
16 |
17 | it('should create the app', () => {
18 | const fixture = TestBed.createComponent(AppComponent);
19 | const app = fixture.debugElement.componentInstance;
20 | expect(app).toBeTruthy();
21 | });
22 |
23 | it(`should have as title 'frontend-search'`, () => {
24 | const fixture = TestBed.createComponent(AppComponent);
25 | const app = fixture.debugElement.componentInstance;
26 | expect(app.title).toEqual('frontend-search');
27 | });
28 |
29 | it('should render title in a h1 tag', () => {
30 | const fixture = TestBed.createComponent(AppComponent);
31 | fixture.detectChanges();
32 | const compiled = fixture.debugElement.nativeElement;
33 | expect(compiled.querySelector('h1').textContent).toContain('Welcome to frontend-search!');
34 | });
35 | });
36 |
--------------------------------------------------------------------------------
/front-end/src/app/app.component.ts:
--------------------------------------------------------------------------------
1 | import { Component } from '@angular/core';
2 |
3 | @Component({
4 | selector: 'app-root',
5 | templateUrl: './app.component.html',
6 | styleUrls: ['./app.component.css']
7 | })
8 | export class AppComponent {
9 | title = 'Full text Search';
10 | description = 'Search for social media and web';
11 | }
12 |
--------------------------------------------------------------------------------
/front-end/src/app/app.module.ts:
--------------------------------------------------------------------------------
1 | import { BrowserModule } from '@angular/platform-browser';
2 | import { NgModule } from '@angular/core';
3 | import { ReactiveFormsModule } from '@angular/forms';
4 | import { HttpClientModule } from '@angular/common/http';
5 | import { AppRoutingModule } from './app-routing.module';
6 | import { AppComponent } from './app.component';
7 | import { SearchComponent } from './search/search.component';
8 | import { SearchService } from './search/search.service';
9 | import { SettingComponent } from './setting/setting.component';
10 |
11 |
12 | @NgModule({
13 | declarations: [
14 | AppComponent,
15 | SearchComponent,
16 | SettingComponent
17 | ],
18 | imports: [
19 | BrowserModule,
20 | ReactiveFormsModule,
21 | HttpClientModule,
22 | AppRoutingModule
23 | ],
24 | providers: [SearchService],
25 | bootstrap: [AppComponent]
26 | })
27 | export class AppModule { }
28 |
--------------------------------------------------------------------------------
/front-end/src/app/search/search.component.css:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SayedBaladoh/Full-Text-Search-using-Docker-Kafka-Elasticsearch-Kibana-Java-Spring-Boot-HBC-Jsoup-Angular/2cc60ae89faf4ed15fccead7c7d776b2a7e29380/front-end/src/app/search/search.component.css
--------------------------------------------------------------------------------
/front-end/src/app/search/search.component.html:
--------------------------------------------------------------------------------
1 |
29 |
30 |
18 |
25 |
26 | {{title}}
19 |{{description}}
20 | 24 |
27 |
28 |
29 |
30 |
32 |
31 | ;
8 |
9 | beforeEach(async(() => {
10 | TestBed.configureTestingModule({
11 | declarations: [ SearchComponent ]
12 | })
13 | .compileComponents();
14 | }));
15 |
16 | beforeEach(() => {
17 | fixture = TestBed.createComponent(SearchComponent);
18 | component = fixture.componentInstance;
19 | fixture.detectChanges();
20 | });
21 |
22 | it('should create', () => {
23 | expect(component).toBeTruthy();
24 | });
25 | });
26 |
--------------------------------------------------------------------------------
/front-end/src/app/search/search.component.ts:
--------------------------------------------------------------------------------
1 | import { Component, OnInit } from '@angular/core';
2 | import { FormControl } from '@angular/forms';
3 |
4 | import { SearchService } from '../search/search.service';
5 | import { pipe } from 'rxjs'
6 | import { debounceTime, distinctUntilChanged, switchMap } from 'rxjs/operators';
7 |
8 | @Component({
9 | selector: 'app-search',
10 | templateUrl: './search.component.html',
11 | styleUrls: ['./search.component.css']
12 | })
13 | export class SearchComponent implements OnInit {
14 |
15 | results: any[] = [];
16 | queryField: FormControl = new FormControl();
17 | searchAt: string;
18 | constructor(private _apiService: SearchService) {
19 | this.searchAt = "tweets";
20 | }
21 |
22 | ngOnInit() {
23 |
24 | this.queryField.valueChanges.pipe(
25 | debounceTime(200),
26 | distinctUntilChanged(),
27 | switchMap((query) => (this.searchAt == 'tweets') ? this._apiService.searchTweets(query) : (this.searchAt == 'web') ? this._apiService.searchArticles(query) : this._apiService.search(query)
28 | )).subscribe((result: any) => { this.results = result });
29 | }
30 |
31 | onItemChange(item) {
32 | console.log("item: " + item)
33 | this.searchAt = item;
34 | let query = this.queryField.value;
35 | if ((this.searchAt == 'tweets'))
36 | this._apiService.searchTweets(query)
37 | .subscribe((result: any) => { this.results = result });
38 | else if ((this.searchAt == 'web'))
39 | this._apiService.searchArticles(query)
40 | .subscribe((result: any) => { this.results = result });
41 | else if ((this.searchAt == 'all'))
42 | this._apiService.search(query)
43 | .subscribe((result: any) => { this.results = result });
44 | }
45 |
46 | /** Get next page of search results */
47 | getResultsPage(pageNo) {
48 | let query = this.queryField.value;
49 | if ((this.searchAt == 'tweets'))
50 | this._apiService.searchTweetsPage(query, pageNo)
51 | .subscribe((result: any) => { this.results = result });
52 | else if ((this.searchAt == 'web'))
53 | this._apiService.searchArticlesPage(query, pageNo)
54 | .subscribe((result: any) => { this.results = result });
55 | else if ((this.searchAt == 'all'))
56 | this._apiService.searchPage(query, pageNo)
57 | .subscribe((result: any) => { this.results = result });
58 | }
59 |
60 |
61 | }
62 |
--------------------------------------------------------------------------------
/front-end/src/app/search/search.service.spec.ts:
--------------------------------------------------------------------------------
1 | import { TestBed } from '@angular/core/testing';
2 |
3 | import { SearchService } from './search.service';
4 |
5 | describe('SearchService', () => {
6 | beforeEach(() => TestBed.configureTestingModule({}));
7 |
8 | it('should be created', () => {
9 | const service: SearchService = TestBed.get(SearchService);
10 | expect(service).toBeTruthy();
11 | });
12 | });
13 |
--------------------------------------------------------------------------------
/front-end/src/app/search/search.service.ts:
--------------------------------------------------------------------------------
1 | import { Injectable } from '@angular/core';
2 | import { HttpClient, HttpHeaders } from '@angular/common/http';
3 |
4 | @Injectable({
5 | providedIn: 'root'
6 | })
7 | export class SearchService {
8 | root: string = 'http://localhost:8082';
9 | baseSearchUrl: string = this.root + '/consumer/api/search?q=';
10 | baseTweetsUrl: string = this.root + '/consumer/api/tweets?q=';
11 | baseArticlesUrl: string = this.root + '/consumer/api/articles?q=';
12 | corsHeaders: HttpHeaders;
13 |
14 | constructor(private http: HttpClient) {
15 | this.corsHeaders = new HttpHeaders({
16 | 'Content-Type': 'application/json',
17 | 'Accept': 'application/json',
18 | 'Access-Control-Allow-Origin': this.root
19 | });
20 | }
21 |
22 | search(queryString: string) {
23 | let _URL = this.baseSearchUrl + queryString;
24 | return this.http.get(_URL, { headers: this.corsHeaders });
25 | }
26 |
27 | searchPage(queryString: string, pageNo: number) {
28 | let _URL = this.baseSearchUrl + queryString + '&page=' + pageNo;;
29 | return this.http.get(_URL, { headers: this.corsHeaders });
30 | }
31 |
32 | searchTweets(queryString: string) {
33 | let _URL = this.baseTweetsUrl + queryString;
34 | return this.http.get(_URL, { headers: this.corsHeaders });
35 | }
36 |
37 | searchTweetsPage(queryString: string, pageNo: number) {
38 | let _URL = this.baseTweetsUrl + queryString + '&page=' + pageNo;
39 | return this.http.get(_URL, { headers: this.corsHeaders });
40 | }
41 |
42 | searchArticles(queryString: string) {
43 | let _URL = this.baseArticlesUrl + queryString;
44 | return this.http.get(_URL, { headers: this.corsHeaders });
45 | }
46 |
47 | searchArticlesPage(queryString: string, pageNo: number) {
48 | let _URL = this.baseArticlesUrl + queryString + '&page=' + pageNo;;
49 | return this.http.get(_URL, { headers: this.corsHeaders });
50 | }
51 |
52 | }
53 |
--------------------------------------------------------------------------------
/front-end/src/app/setting/setting.component.css:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SayedBaladoh/Full-Text-Search-using-Docker-Kafka-Elasticsearch-Kibana-Java-Spring-Boot-HBC-Jsoup-Angular/2cc60ae89faf4ed15fccead7c7d776b2a7e29380/front-end/src/app/setting/setting.component.css
--------------------------------------------------------------------------------
/front-end/src/app/setting/setting.component.html:
--------------------------------------------------------------------------------
1 |
5 |
36 |;
8 |
9 | beforeEach(async(() => {
10 | TestBed.configureTestingModule({
11 | declarations: [ SettingComponent ]
12 | })
13 | .compileComponents();
14 | }));
15 |
16 | beforeEach(() => {
17 | fixture = TestBed.createComponent(SettingComponent);
18 | component = fixture.componentInstance;
19 | fixture.detectChanges();
20 | });
21 |
22 | it('should create', () => {
23 | expect(component).toBeTruthy();
24 | });
25 | });
26 |
--------------------------------------------------------------------------------
/front-end/src/app/setting/setting.component.ts:
--------------------------------------------------------------------------------
1 | import { Component, OnInit } from '@angular/core';
2 | import { FormGroup, FormControl, Validators } from '@angular/forms';
3 | import { SettingService } from '../setting/setting.service';
4 |
5 | @Component({
6 | selector: 'app-setting',
7 | templateUrl: './setting.component.html',
8 | styleUrls: ['./setting.component.css']
9 | })
10 | export class SettingComponent implements OnInit {
11 |
12 | twitterForm: FormGroup;
13 | crawlerForm: FormGroup;
14 | constructor(private _settingService: SettingService) { }
15 |
16 | ngOnInit() {
17 | this.twitterForm = new FormGroup({
18 | q: new FormControl('', Validators.required),
19 | count: new FormControl('')
20 | });
21 |
22 | this.crawlerForm = new FormGroup({
23 | url: new FormControl('http://', Validators.required),
24 | q: new FormControl('', Validators.required),
25 | count: new FormControl(''),
26 | maxPagesToSearch: new FormControl('')
27 | });
28 |
29 | }
30 | isTweeterDone = true;
31 | onSubmitTwitterForm(form: FormGroup) {
32 | // console.log('Valid?', form.valid); // true or false
33 | // console.log('query', form.value.q);
34 | // console.log('Count', form.value.count);
35 | if (form.valid) {
36 | this.isTweeterDone = false;
37 | this._settingService.lanchTweetsStreamer(form.value.q, form.value.count)
38 | .subscribe((result: any) => { this.isTweeterDone = true });
39 | }
40 | }
41 |
42 | isCrawlerDone = true;
43 | onSubmitCrawlerForm(form: FormGroup) {
44 | // console.log('Valid?', form.valid); // true or false
45 | // console.log('URL', form.value.url);
46 | // console.log('Query', form.value.q);
47 | // console.log('Count', form.value.count);
48 | // console.log('maxPagesToSearch', form.value.maxPagesToSearch);
49 | if (form.valid) {
50 | this.isCrawlerDone = false;
51 | this._settingService.lanchCrawlerStreamer(form.value.q, form.value.url, form.value.count, form.value.maxPagesToSearch)
52 | .subscribe((result: any) => { this.isCrawlerDone = true });
53 | }
54 | }
55 |
56 | isDeleteingTweets = false;
57 | deleteTweets() {
58 | this.isDeleteingTweets = true;
59 | this._settingService.deleteAllTweets()
60 | .subscribe(() => {
61 | this.isDeleteingTweets = false;
62 | });
63 | }
64 |
65 | isDeleteingArticles = false;
66 | deleteCrawler() {
67 | this.isDeleteingArticles = true;
68 | this._settingService.deleteAllArticles()
69 | .subscribe(() => {
70 | this.isDeleteingArticles = false;
71 | });
72 | }
73 |
74 | }
75 |
--------------------------------------------------------------------------------
/front-end/src/app/setting/setting.service.spec.ts:
--------------------------------------------------------------------------------
1 | import { TestBed } from '@angular/core/testing';
2 |
3 | import { SettingService } from './setting.service';
4 |
5 | describe('SettingService', () => {
6 | beforeEach(() => TestBed.configureTestingModule({}));
7 |
8 | it('should be created', () => {
9 | const service: SettingService = TestBed.get(SettingService);
10 | expect(service).toBeTruthy();
11 | });
12 | });
13 |
--------------------------------------------------------------------------------
/front-end/src/app/setting/setting.service.ts:
--------------------------------------------------------------------------------
1 | import { Injectable } from '@angular/core';
2 | import { HttpClient, HttpHeaders } from '@angular/common/http';
3 |
4 | @Injectable({
5 | providedIn: 'root'
6 | })
7 | export class SettingService {
8 |
9 | rootProducer: string = 'http://localhost:8081';
10 | baseTwitterUrl: string = this.rootProducer + '/producer/api/stream/twitter?q=';
11 | baseCrawlerUrl: string = this.rootProducer + '/producer/api/stream/crawler?q=';
12 |
13 | rootConsumer: string = 'http://localhost:8082';
14 | baseTweetsUrl: string = this.rootConsumer + '/consumer/api/tweets';
15 | baseArticlesUrl: string = this.rootConsumer + '/consumer/api/articles';
16 |
17 | corsProducerHeaders: HttpHeaders;
18 | corsConsumerHeaders: HttpHeaders;
19 |
20 | constructor(private http: HttpClient) {
21 | this.corsProducerHeaders = new HttpHeaders({
22 | 'Content-Type': 'application/json',
23 | 'Accept': 'application/json',
24 | 'Access-Control-Allow-Origin': this.rootProducer
25 | });
26 |
27 | this.corsConsumerHeaders = new HttpHeaders({
28 | 'Content-Type': 'application/json',
29 | 'Accept': 'application/json',
30 | 'Access-Control-Allow-Origin': this.rootConsumer
31 | });
32 | }
33 |
34 | lanchTweetsStreamer(query: string, count: number) {
35 | // console.log("count: " + count);
36 | let _URL = this.baseTwitterUrl + query + (count ? '&count=' + count : '');
37 | // console.log("_URL: " + _URL);
38 | return this.http.get(_URL, { headers: this.corsProducerHeaders });
39 | }
40 |
41 | lanchCrawlerStreamer(query: string, url: string, count: number, maxPagesToSearch: number) {
42 | // console.log("count: " + count);
43 | let _URL = this.baseCrawlerUrl + query + '&url=' + url + (count ? '&count=' + count : '') + (maxPagesToSearch ? '&maxPagesToSearch=' + maxPagesToSearch : '');
44 | // console.log("_URL: " + _URL);
45 | return this.http.get(_URL, { headers: this.corsProducerHeaders });
46 | }
47 |
48 | deleteAllTweets() {
49 | let _URL = this.baseTweetsUrl + '/all';
50 | // console.log('deleteTweets'+ _URL)
51 | return this.http.delete(_URL, { headers: this.corsConsumerHeaders });
52 |
53 | }
54 |
55 | deleteAllArticles() {
56 | let _URL = this.baseArticlesUrl + '/all';
57 | // console.log('deleteAllArticles'+ _URL)
58 | return this.http.delete(_URL, { headers: this.corsConsumerHeaders });
59 |
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/front-end/src/assets/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SayedBaladoh/Full-Text-Search-using-Docker-Kafka-Elasticsearch-Kibana-Java-Spring-Boot-HBC-Jsoup-Angular/2cc60ae89faf4ed15fccead7c7d776b2a7e29380/front-end/src/assets/.gitkeep
--------------------------------------------------------------------------------
/front-end/src/assets/spinner.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SayedBaladoh/Full-Text-Search-using-Docker-Kafka-Elasticsearch-Kibana-Java-Spring-Boot-HBC-Jsoup-Angular/2cc60ae89faf4ed15fccead7c7d776b2a7e29380/front-end/src/assets/spinner.gif
--------------------------------------------------------------------------------
/front-end/src/assets/spinnerGrey.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SayedBaladoh/Full-Text-Search-using-Docker-Kafka-Elasticsearch-Kibana-Java-Spring-Boot-HBC-Jsoup-Angular/2cc60ae89faf4ed15fccead7c7d776b2a7e29380/front-end/src/assets/spinnerGrey.gif
--------------------------------------------------------------------------------
/front-end/src/environments/environment.prod.ts:
--------------------------------------------------------------------------------
1 | export const environment = {
2 | production: true
3 | };
4 |
--------------------------------------------------------------------------------
/front-end/src/environments/environment.ts:
--------------------------------------------------------------------------------
1 | // This file can be replaced during build by using the `fileReplacements` array.
2 | // `ng build --prod` replaces `environment.ts` with `environment.prod.ts`.
3 | // The list of file replacements can be found in `angular.json`.
4 |
5 | export const environment = {
6 | production: false
7 | };
8 |
9 | /*
10 | * For easier debugging in development mode, you can import the following file
11 | * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`.
12 | *
13 | * This import should be commented out in production mode because it will have a negative impact
14 | * on performance if an error is thrown.
15 | */
16 | // import 'zone.js/dist/zone-error'; // Included with Angular CLI.
17 |
--------------------------------------------------------------------------------
/front-end/src/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SayedBaladoh/Full-Text-Search-using-Docker-Kafka-Elasticsearch-Kibana-Java-Spring-Boot-HBC-Jsoup-Angular/2cc60ae89faf4ed15fccead7c7d776b2a7e29380/front-end/src/favicon.ico
--------------------------------------------------------------------------------
/front-end/src/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | FrontendSearch
6 |
4 | 4.0.0
5 |
6 | org.springframework.boot
7 | spring-boot-starter-parent
8 | 2.1.5.RELEASE
9 |
11 | com.sayedbaladoh.buzzdiggr
12 | producer
13 | 0.0.1-SNAPSHOT
14 | producer
15 | Producer service to collect, clean data collected from different sources, and send it to Kafka producer
16 |
17 |
18 | 1.8
19 |
20 |
21 |
22 |
23 | org.springframework.boot
24 | spring-boot-starter-actuator
25 |
26 |
27 | org.springframework.boot
28 | spring-boot-starter-web
29 |
30 |
31 | org.springframework.kafka
32 | spring-kafka
33 |
34 |
35 |
36 | org.springframework.boot
37 | spring-boot-starter-test
38 | test
39 |
40 |
41 | org.springframework.kafka
42 | spring-kafka-test
43 | test
44 |
45 |
46 |
47 | com.twitter
48 | hbc-core
49 | 2.2.0
50 |
51 |
52 | org.json
53 | json
54 | 20180813
55 |
56 |
57 |
58 | org.jsoup
59 | jsoup
60 | 1.12.1
61 |
62 |
63 |
64 | io.springfox
65 | springfox-swagger2
66 | 2.8.0
67 |
68 |
69 | io.springfox
70 | springfox-swagger-ui
71 | 2.8.0
72 |
73 |
74 | com.google.guava
75 | guava
76 | 23.0
77 |
78 |
79 |
80 |
81 |
82 |
83 | org.springframework.boot
84 | spring-boot-maven-plugin
85 |
86 |
87 |
88 |
89 |
90 |
--------------------------------------------------------------------------------
/producer/src/main/java/com/sayedbaladoh/buzzdiggr/producer/ProducerApplication.java:
--------------------------------------------------------------------------------
1 | package com.sayedbaladoh.buzzdiggr.producer;
2 |
3 | import org.slf4j.Logger;
4 | import org.slf4j.LoggerFactory;
5 | import org.springframework.beans.factory.annotation.Autowired;
6 | import org.springframework.boot.CommandLineRunner;
7 | import org.springframework.boot.SpringApplication;
8 | import org.springframework.boot.autoconfigure.SpringBootApplication;
9 | import org.springframework.boot.context.properties.EnableConfigurationProperties;
10 |
11 | import com.sayedbaladoh.buzzdiggr.producer.property.TwitterProperties;
12 | import com.sayedbaladoh.buzzdiggr.producer.service.crawler.Spider;
13 | import com.sayedbaladoh.buzzdiggr.producer.service.stream.TwitterStream;
14 |
15 | @EnableConfigurationProperties(TwitterProperties.class)
16 | @SpringBootApplication
17 | public class ProducerApplication implements CommandLineRunner {
18 |
19 | private static final Logger logger = LoggerFactory.getLogger(ProducerApplication.class);
20 |
21 | @Autowired
22 | private TwitterStream twitterStreamingService;
23 |
24 | @Autowired
25 | private Spider spiderService;
26 |
27 | public static void main(String[] args) {
28 | SpringApplication.run(ProducerApplication.class, args);
29 | }
30 |
31 | // Test stream Twitter tweets and web Crawler
32 | @Override
33 | public void run(String... strings) throws Exception {
34 | // logger.info("Running Twitter Streaming ...");
35 | // twitterStreamingService.stream(10, "twitterapi", "محمد صلاح", "#Sports");
36 | // logger.info("Running Crawller Streaming ...");
37 | // spiderService.crawl("https://www.shorouknews.com/", "محمد", 15, 50);
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/producer/src/main/java/com/sayedbaladoh/buzzdiggr/producer/config/KakfaProducerConfig.java:
--------------------------------------------------------------------------------
1 | package com.sayedbaladoh.buzzdiggr.producer.config;
2 |
3 | import com.sayedbaladoh.buzzdiggr.producer.model.Tweet;
4 | import com.sayedbaladoh.buzzdiggr.producer.model.Article;
5 | import org.apache.kafka.clients.producer.ProducerConfig;
6 | import org.apache.kafka.common.serialization.StringSerializer;
7 | import org.springframework.beans.factory.annotation.Value;
8 | import org.springframework.context.annotation.Bean;
9 | import org.springframework.context.annotation.Configuration;
10 | import org.springframework.kafka.core.DefaultKafkaProducerFactory;
11 | import org.springframework.kafka.core.KafkaTemplate;
12 | import org.springframework.kafka.core.ProducerFactory;
13 | import org.springframework.kafka.support.serializer.JsonSerializer;
14 |
15 | import java.util.HashMap;
16 | import java.util.Map;
17 |
18 | /**
19 | * Kakfa Producer Configuration
20 | *
21 | * @author SayedBaladoh
22 | *
23 | */
24 | @Configuration
25 | public class KakfaProducerConfig {
26 |
27 | @Value("${kafka.boot.server}")
28 | private String kafkaServer;
29 |
30 | // String Producer Factory
31 | @Bean
32 | public ProducerFactory producerFactory() {
33 |
34 | Map config = new HashMap<>();
35 |
36 | config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServer);
37 | config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
38 | config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
39 |
40 | return new DefaultKafkaProducerFactory<>(config);
41 | }
42 |
43 | // String Kafka Template
44 | @Bean
45 | public KafkaTemplate kafkaTemplate() {
46 | return new KafkaTemplate<>(producerFactory());
47 | }
48 |
49 |
50 | // Tweet Producer Factory
51 | @Bean
52 | public ProducerFactory tweetProducerFactory() {
53 |
54 | Map config = new HashMap<>();
55 |
56 | config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServer);
57 | config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
58 | config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class);
59 |
60 | return new DefaultKafkaProducerFactory<>(config);
61 | }
62 |
63 | // Tweet Kafka Template
64 | @Bean
65 | public KafkaTemplate tweetKafkaTemplate() {
66 | return new KafkaTemplate<>(tweetProducerFactory());
67 | }
68 |
69 | // Article Producer Factory
70 | @Bean
71 | public ProducerFactory articleProducerFactory() {
72 |
73 | Map config = new HashMap<>();
74 |
75 | config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServer);
76 | config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
77 | config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class);
78 |
79 | return new DefaultKafkaProducerFactory<>(config);
80 | }
81 |
82 | // Article Kafka Template
83 | @Bean
84 | public KafkaTemplate articleKafkaTemplate() {
85 | return new KafkaTemplate<>(articleProducerFactory());
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/producer/src/main/java/com/sayedbaladoh/buzzdiggr/producer/config/SwaggerConfig.java:
--------------------------------------------------------------------------------
1 | package com.sayedbaladoh.buzzdiggr.producer.config;
2 |
3 | import org.springframework.context.annotation.Bean;
4 | import org.springframework.context.annotation.Configuration;
5 |
6 | import springfox.documentation.builders.ApiInfoBuilder;
7 | import springfox.documentation.builders.RequestHandlerSelectors;
8 | import springfox.documentation.service.ApiInfo;
9 | import springfox.documentation.service.Contact;
10 | import springfox.documentation.spi.DocumentationType;
11 | import springfox.documentation.spring.web.plugins.Docket;
12 | import springfox.documentation.swagger2.annotations.EnableSwagger2;
13 |
14 | @Configuration
15 | @EnableSwagger2
16 | public class SwaggerConfig {
17 | @Bean
18 | public Docket Api() {
19 | return new Docket(DocumentationType.SWAGGER_2)
20 | .select().apis(RequestHandlerSelectors.basePackage("com.sayedbaladoh.buzzdiggr.producer.controller"))
21 | // .paths(regex("/api.*"))
22 | .build()
23 | .apiInfo(metaData());
24 | }
25 |
26 | private ApiInfo metaData() {
27 | return new ApiInfoBuilder()
28 | .title("Producer Service API")
29 | .description("Producer service to collect, clean data collected from different sources, and send it to Kafka producer")
30 | .version("1.0.0")
31 | // .license("Apache License Version 2.0")
32 | // .licenseUrl("https://www.apache.org/licenses/LICENSE-2.0")
33 | .contact(
34 | new Contact("Sayed Baladoh", "https://www.linkedin.com/in/sayed-baladoh-227aa66b/", "sayedbaladoh@yahoo.com"))
35 | .build();
36 | }
37 |
38 | }
39 |
--------------------------------------------------------------------------------
/producer/src/main/java/com/sayedbaladoh/buzzdiggr/producer/controller/StreamController.java:
--------------------------------------------------------------------------------
1 | package com.sayedbaladoh.buzzdiggr.producer.controller;
2 |
3 | import org.slf4j.Logger;
4 | import org.slf4j.LoggerFactory;
5 | import org.springframework.beans.factory.annotation.Autowired;
6 | import org.springframework.http.ResponseEntity;
7 | import org.springframework.web.bind.annotation.CrossOrigin;
8 | import org.springframework.web.bind.annotation.GetMapping;
9 | import org.springframework.web.bind.annotation.PathVariable;
10 | import org.springframework.web.bind.annotation.RequestMapping;
11 | import org.springframework.web.bind.annotation.RequestParam;
12 | import org.springframework.web.bind.annotation.RestController;
13 |
14 | import com.sayedbaladoh.buzzdiggr.producer.service.crawler.Spider;
15 | import com.sayedbaladoh.buzzdiggr.producer.service.stream.TwitterStream;
16 |
17 | import io.swagger.annotations.Api;
18 | import io.swagger.annotations.ApiOperation;
19 |
20 | /**
21 | *
22 | * Initialize and instantiate Streaming and Crawling API
23 | *
24 | * @author SayedBaladoh
25 | *
26 | */
27 | @RestController
28 | @CrossOrigin(origins = "${frontend.url}")
29 | @RequestMapping("/api/stream")
30 | @Api(value = "Stream", description = "Initialize and instantiate Streaming and Crowling API")
31 | public class StreamController {
32 |
33 | private static final Logger LOGGER = LoggerFactory.getLogger(StreamController.class);
34 |
35 | @Autowired
36 | private TwitterStream twitterStreamingService;
37 |
38 | @Autowired
39 | private Spider spiderService;
40 |
41 | /**
42 | * stream
43 | *
44 | * Start retrieve real-time Tweets
45 | *
46 | * @param count
47 | * - The total number of tweets to stream. Default (100).
48 | * @param q
49 | * - List of search keywords
50 | *
51 | * @return
52 | */
53 | @ApiOperation(value = "Start retrieve real-time Tweets")
54 | @GetMapping(value = "/twitter", params = { "q", "count" })
55 | public ResponseEntity stream(@RequestParam(name = "count") int numberOfTweets,
56 | @RequestParam(name = "q") String[] keywords) {
57 | try {
58 | twitterStreamingService.stream(numberOfTweets, keywords);
59 | } catch (InterruptedException e) {
60 | // e.printStackTrace();
61 | LOGGER.error(e.getMessage());
62 | return ResponseEntity.badRequest().build();
63 | }
64 | return ResponseEntity.ok().build();
65 | }
66 |
67 | /**
68 | * stream
69 | *
70 | * Start retrieve real-time Tweets with default total number of tweets (100)
71 | *
72 | * @param q
73 | * - List of search keywords
74 | *
75 | * @return
76 | */
77 | @ApiOperation(value = "Start retrieve real-time Tweets with default total number of tweets (100)")
78 | @GetMapping(value = "/twitter", params = { "q" })
79 | public ResponseEntity stream(@RequestParam(name = "q") String[] keywords) {
80 | try {
81 | twitterStreamingService.stream(keywords);
82 | } catch (InterruptedException e) {
83 | LOGGER.error(e.getMessage());
84 | return ResponseEntity.badRequest().build();
85 | }
86 | return ResponseEntity.ok().build();
87 | }
88 |
89 | /**
90 | * crawl
91 | *
92 | * launch web crawler
93 | *
94 | * @param url
95 | * - The starting point of the crawler
96 | * @param q
97 | * - The keyword or string that you are searching for
98 | * @param count
99 | * - The limit for retrieving pages
100 | * @param maxPagesToSearch
101 | * - The maximum pages to search
102 | */
103 | @ApiOperation(value = "launch web crawler")
104 | @GetMapping(value = "/crawler", params = { "url", "q", "count", "maxPagesToSearch" })
105 | public ResponseEntity crawl(@RequestParam(name = "url") String url, @RequestParam(name = "q") String keyword,
106 | @RequestParam(name = "count") int pagesLimit,
107 | @RequestParam(name = "maxPagesToSearch") int maxPagesToSearch) {
108 | spiderService.crawl(url, keyword, pagesLimit, maxPagesToSearch);
109 | return ResponseEntity.ok().build();
110 | }
111 |
112 | /**
113 | * crawl
114 | *
115 | * launch web crawler
116 | *
117 | * @param url
118 | * - The starting point of the crawler
119 | * @param q
120 | * - The keyword or string that you are searching for
121 | * @param count
122 | * - The limit for retrieving pages
123 | */
124 | @ApiOperation(value = "launch web crawler")
125 | @GetMapping(value = "/crawler", params = { "url", "q", "count" })
126 | public ResponseEntity crawl(@RequestParam(name = "url") String url, @RequestParam(name = "q") String keyword,
127 | @RequestParam(name = "count") int pagesLimit) {
128 | spiderService.crawl(url, keyword, pagesLimit);
129 | return ResponseEntity.ok().build();
130 | }
131 |
132 | /**
133 | * crawl
134 | *
135 | * launch web crawler
136 | *
137 | * @param url
138 | * - The starting point of the crawler
139 | * @param q
140 | * - The keyword or string that you are searching for
141 | * @param maxPagesToSearch
142 | * - The maximum pages to search
143 | */
144 | @ApiOperation(value = "launch web crawler")
145 | @GetMapping(value = "/crawler", params = { "url", "q", "maxPagesToSearch" })
146 | public ResponseEntity crawlByMax(@RequestParam(name = "url") String url,
147 | @RequestParam(name = "q") String keyword, @RequestParam(name = "maxPagesToSearch") int maxPagesToSearch) {
148 | spiderService.crawlByMax(url, keyword, maxPagesToSearch);
149 | return ResponseEntity.ok().build();
150 | }
151 |
152 | /**
153 | * crawl
154 | *
155 | * launch web crawler
156 | *
157 | * @param url
158 | * - The starting point of the crawler
159 | * @param q
160 | * - The keyword or string that you are searching for
161 | */
162 | @ApiOperation(value = "launch web crawler")
163 | @GetMapping(value = "/crawler", params = { "url", "q" })
164 | public ResponseEntity crawl(@RequestParam(name = "url") String url, @RequestParam(name = "q") String keyword) {
165 | spiderService.crawl(url, keyword);
166 | return ResponseEntity.ok().build();
167 | }
168 | }
169 |
--------------------------------------------------------------------------------
/producer/src/main/java/com/sayedbaladoh/buzzdiggr/producer/model/Article.java:
--------------------------------------------------------------------------------
1 | package com.sayedbaladoh.buzzdiggr.producer.model;
2 |
3 | public class Article {
4 |
5 | private String url;
6 | private String title;
7 | private String text;
8 |
9 | public Article(){
10 |
11 | }
12 |
13 | public Article(String url, String title, String text) {
14 | super();
15 | this.url = url;
16 | this.title = title;
17 | this.text = text;
18 | }
19 |
20 | public String getUrl() {
21 | return url;
22 | }
23 | public void setUrl(String url) {
24 | this.url = url;
25 | }
26 | public String getTitle() {
27 | return title;
28 | }
29 | public void setTitle(String title) {
30 | this.title = title;
31 | }
32 | public String getText() {
33 | return text;
34 | }
35 | public void setText(String text) {
36 | this.text = text;
37 | }
38 | @Override
39 | public String toString() {
40 | return "Article [url=" + url + ", title=" + title + ", text=" + text + "]";
41 | }
42 |
43 | }
44 |
--------------------------------------------------------------------------------
/producer/src/main/java/com/sayedbaladoh/buzzdiggr/producer/model/Tweet.java:
--------------------------------------------------------------------------------
1 | package com.sayedbaladoh.buzzdiggr.producer.model;
2 |
3 | import java.util.Date;
4 |
5 | public class Tweet {
6 |
7 | private String id;
8 | private String text;
9 | private Date date;
10 | private String language;
11 | private String userName;
12 |
13 | public String getId() {
14 | return id;
15 | }
16 |
17 | public void setId(String id) {
18 | this.id = id;
19 | }
20 |
21 | public String getText() {
22 | return text;
23 | }
24 |
25 | public void setText(String text) {
26 | this.text = text;
27 | }
28 |
29 | public Date getDate() {
30 | return date;
31 | }
32 |
33 | public void setDate(Date date) {
34 | this.date = date;
35 | }
36 |
37 | public String getLanguage() {
38 | return language;
39 | }
40 |
41 | public void setLanguage(String language) {
42 | this.language = language;
43 | }
44 |
45 | public String getUserName() {
46 | return userName;
47 | }
48 |
49 | public void setUserName(String userName) {
50 | this.userName = userName;
51 | }
52 |
53 | @Override
54 | public String toString() {
55 | return "Tweet [id=" + id + ", text=" + text + ", date=" + date + ", language=" + language + ", userName="
56 | + userName + "]";
57 | }
58 |
59 | }
60 |
--------------------------------------------------------------------------------
/producer/src/main/java/com/sayedbaladoh/buzzdiggr/producer/property/TwitterProperties.java:
--------------------------------------------------------------------------------
1 | package com.sayedbaladoh.buzzdiggr.producer.property;
2 |
3 | import org.springframework.boot.context.properties.ConfigurationProperties;
4 |
5 | /**
6 | * Twitter Properties specific to Twitter Social properties.
7 | *
8 | * visitedPages;
53 | private List pageList;
54 | // Use a fake USER_AGENT so the web server thinks the robot is a normal web browser.
55 | private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1";
56 |
57 | /**
58 | * Main launching point for the Spider's functionality with default maxPagesToSearch and pagesLimit.
59 | *
60 | * @param url
61 | * - The starting point of the spider
62 | * @param searchTopic
63 | * - The word or string that you are searching for
64 | */
65 | public void crawl(String url, String topic) {
66 |
67 | visitedPages = new HashSet();
68 | pageList = new ArrayList();
69 |
70 | this.startingURL = url;
71 | this.search(url, topic);
72 |
73 | LOGGER.info("**Crawling Done** Visited:'{}' web page(s), Found:'{}' web page(s).", this.visitedPages.size(), this.pageList.size());
74 | }
75 |
76 | /**
77 | * Overload the main launching point for the Spider's functionality with default maxPagesToSearch.
78 | *
79 | * @param url
80 | * - The starting point of the spider
81 | * @param searchTopic
82 | * - The word or string that you are searching for
83 | * @param pagesLimit
84 | * - The limit for retrieving pages
85 | */
86 | public void crawl(String url, String topic, int pagesLimit) {
87 |
88 | visitedPages = new HashSet();
89 | pageList = new ArrayList();
90 |
91 | this.startingURL = url;
92 | if (pagesLimit > 0)
93 | this.pagesLimit = pagesLimit;
94 |
95 | this.search(url, topic);
96 |
97 | LOGGER.info("**Crawling Done** Visited:'{}' web page(s), Found:'{}' web page(s).", this.visitedPages.size(), this.pageList.size());
98 | }
99 |
100 | /**
101 | * Overload the launching point for the Spider's functionality with default pagesLimit.
102 | *
103 | * @param url
104 | * - The starting point of the spider
105 | * @param searchTopic
106 | * - The word or string that you are searching for
107 | * @param maxPagesToSearch
108 | * - The maximum pages to search
109 | */
110 | public void crawlByMax(String url, String topic, int maxPagesToSearch) {
111 |
112 | visitedPages = new HashSet();
113 | pageList = new ArrayList();
114 |
115 | this.startingURL = url;
116 | if (maxPagesToSearch > 0)
117 | this.maxPagesToSearch = maxPagesToSearch;
118 |
119 | this.search(url, topic);
120 |
121 | LOGGER.info("**Crawling Done** Visited:'{}' web page(s), Found:'{}' web page(s).", this.visitedPages.size(), this.pageList.size());
122 | }
123 |
124 | /**
125 | * Overload the main launching point for the Spider's functionality.
126 | *
127 | * @param url
128 | * - The starting point of the spider
129 | * @param searchTopic
130 | * - The word or string that you are searching for
131 | * @param pagesLimit
132 | * - The limit for retrieving pages
133 | * @param maxPagesToSearch
134 | * - The maxmum pages to search
135 | */
136 | public void crawl(String url, String topic, int pagesLimit, int maxPagesToSearch) {
137 |
138 | visitedPages = new HashSet();
139 | pageList = new ArrayList();
140 |
141 | this.startingURL = url;
142 | if (pagesLimit > 0)
143 | this.pagesLimit = pagesLimit;
144 | if (maxPagesToSearch > 0)
145 | this.maxPagesToSearch = maxPagesToSearch;
146 |
147 | this.search(url, topic);
148 |
149 | LOGGER.info("**Crawling Done** Visited:'{}' web page(s), Found:'{}' web page(s).", this.visitedPages.size(), this.pageList.size());
150 | }
151 |
152 | public void search(String url, String topic) {
153 |
154 | this.visitedPages.add(url);
155 | try {
156 | Connection connection = Jsoup.connect(url).userAgent(USER_AGENT);
157 | Document htmlDocument = connection.get();
158 | // 200 is the HTTP OK status code indicating that everything
159 | // is great.
160 | // if (connection.response().statusCode() == 200) {
161 | // System.out.println("\n**Visiting** Received web page at " + url);
162 | // }
163 | if (!connection.response().contentType().contains("text/html")) {
164 | System.out.println("**Failure** Retrieved something other than HTML");
165 | return;
166 | }
167 |
168 | String text = htmlDocument.body().text();
169 | if (text.toLowerCase().contains(topic.toLowerCase())) {
170 | String title = htmlDocument.title();
171 |
172 | LOGGER.info("**Success search** Topic:'{}' is existed at:'{}' with title:'{}'.", topic, url, title);
173 | this.pageList.add(url);
174 |
175 | Article article = new Article(url, title, text);
176 | LOGGER.info("Article: {} ", article);
177 |
178 | //Send article to Kafka
179 | sender.send(articleTopicName, article);
180 |
181 | }
182 |
183 | // Process page links
184 | Elements linksOnPage = htmlDocument.select("a[href]");
185 | for (Element link : linksOnPage) {
186 | String absUrl = link.absUrl("href");
187 | // Check URL is not already visited, search in the same domain, pages limit and max pages to search
188 | if (absUrl.contains(this.startingURL) && !this.visitedPages.contains(absUrl)
189 | && this.pageList.size() < this.pagesLimit && this.visitedPages.size() < this.maxPagesToSearch) {
190 | search(absUrl, topic);
191 | }
192 | }
193 |
194 | } catch (Exception ex) {
195 | // We were not successful in our HTTP request
196 | LOGGER.error(url, " must supply a valid URL");
197 | }
198 | }
199 |
200 | // /**
201 | // * Test Case.
202 | // *
203 | // * @param args
204 | // * - not used
205 | // */
206 | // public static void main(String[] args) {
207 | // Spider spider = new Spider();
208 | //// spider.crawl("https://www.shorouknews.com/", "Mohammad", 15);
209 | // spider.crawl("https://www.shorouknews.com/", "محمد", 15, 50);
210 | // }
211 |
212 | }
213 |
--------------------------------------------------------------------------------
/producer/src/main/java/com/sayedbaladoh/buzzdiggr/producer/service/kafka/Sender.java:
--------------------------------------------------------------------------------
1 | package com.sayedbaladoh.buzzdiggr.producer.service.kafka;
2 |
3 | import org.slf4j.Logger;
4 | import org.slf4j.LoggerFactory;
5 | import org.springframework.beans.factory.annotation.Autowired;
6 | import org.springframework.kafka.core.KafkaTemplate;
7 | import org.springframework.stereotype.Service;
8 |
9 | import com.sayedbaladoh.buzzdiggr.producer.model.Article;
10 | import com.sayedbaladoh.buzzdiggr.producer.model.Tweet;
11 |
12 | /**
13 | * Kakfa Sender Service
14 | *
15 | * @author SayedBaladoh
16 | *
17 | */
18 | @Service
19 | public class Sender {
20 |
21 | private static final Logger LOGGER = LoggerFactory.getLogger(Sender.class);
22 |
23 | @Autowired
24 | private KafkaTemplate kafkaTemplate;
25 |
26 | @Autowired
27 | private KafkaTemplate tweetKafkaTemplate;
28 |
29 | @Autowired
30 | private KafkaTemplate articleKafkaTemplate;
31 |
32 | // Send String data
33 | public void send(String topic, String data) {
34 |
35 | LOGGER.info("sending data='{}' to topic='{}'", data, topic);
36 | kafkaTemplate.send(topic, data);
37 | }
38 |
39 | // Send Tweet object
40 | public void send(String topic, Tweet tweet) {
41 |
42 | LOGGER.info("sending data='{}' to kafka topic='{}'", tweet, topic);
43 | tweetKafkaTemplate.send(topic, tweet);
44 |
45 | }
46 |
47 | // Send Article object
48 | public void send(String topic, Article article) {
49 |
50 | LOGGER.info("sending data='{}' to kafka topic='{}'", article, topic);
51 | articleKafkaTemplate.send(topic, article);
52 |
53 | }
54 |
55 | }
56 |
--------------------------------------------------------------------------------
/producer/src/main/java/com/sayedbaladoh/buzzdiggr/producer/service/stream/TweetHandler.java:
--------------------------------------------------------------------------------
1 | package com.sayedbaladoh.buzzdiggr.producer.service.stream;
2 |
3 | import java.text.ParseException;
4 | import java.text.SimpleDateFormat;
5 | import java.util.Date;
6 |
7 | import org.json.JSONException;
8 | import org.json.JSONObject;
9 | import org.springframework.stereotype.Service;
10 |
11 | import com.sayedbaladoh.buzzdiggr.producer.model.Tweet;
12 | import com.sayedbaladoh.buzzdiggr.producer.util.StringUtils;
13 |
14 | /**
15 | * Tweet Handler
16 | *
17 | * @author SayedBaladoh
18 | *
19 | */
20 | @Service
21 | public class TweetHandler {
22 |
23 | /**
24 | * Parse Tweet from String to Object
25 | *
26 | * @param jsonText
27 | * - Json as a string
28 | * @return
29 | * - Parsed Tweet object
30 | */
31 | public Tweet parseTweet(String jsonText) {
32 |
33 | Tweet tweet = new Tweet();
34 | try {
35 | JSONObject jsonObject = new JSONObject(jsonText);
36 |
37 | tweet.setId(jsonObject.getString("id_str"));
38 | String cleanedtext = StringUtils.cleanText(jsonObject.getString("text"));
39 | tweet.setText(cleanedtext);
40 |
41 | // "EEE MMM d HH:mm:ss Z yyyy"
42 | SimpleDateFormat sdf = new SimpleDateFormat("EEE MMM d HH:mm:ss Z yyyy");
43 | try {
44 | Date date = sdf.parse(jsonObject.getString("created_at"));
45 | tweet.setDate(date);
46 | } catch (ParseException ex) {
47 | ex.printStackTrace();
48 | }
49 |
50 | tweet.setLanguage(jsonObject.getString("lang"));
51 |
52 | JSONObject user = jsonObject.getJSONObject("user");
53 | tweet.setUserName(user.getString("name"));
54 |
55 | // this.place = jsonObject.getString("place");
56 | // System.out.println("Text: " + jsonObject.getString("text"));
57 | // System.out.println("Created_at: " +
58 | // jsonObject.getString("created_at"));
59 | // System.out.println("lang: " + jsonObject.getString("lang"));
60 | // System.out.println("id_str: " + jsonObject.getString("id_str"));
61 | // System.out.println("place: " + jsonObject.getString("place"));
62 | // System.out.println("user name: " + user.getString("name"));
63 | // System.out.println("user profile_image_url: " +
64 | // user.getString("profile_image_url"));
65 | // System.out.println();
66 |
67 | } catch (JSONException ex) {
68 | ex.printStackTrace();
69 | }
70 |
71 | return tweet;
72 | }
73 |
74 | }
--------------------------------------------------------------------------------
/producer/src/main/java/com/sayedbaladoh/buzzdiggr/producer/service/stream/TwitterStream.java:
--------------------------------------------------------------------------------
1 | package com.sayedbaladoh.buzzdiggr.producer.service.stream;
2 |
3 | import java.util.concurrent.BlockingQueue;
4 | import java.util.concurrent.LinkedBlockingQueue;
5 |
6 | import org.slf4j.Logger;
7 | import org.slf4j.LoggerFactory;
8 | import org.springframework.beans.factory.annotation.Autowired;
9 | import org.springframework.beans.factory.annotation.Value;
10 | import org.springframework.stereotype.Service;
11 |
12 | import com.google.common.collect.Lists;
13 | import com.sayedbaladoh.buzzdiggr.producer.model.Tweet;
14 | import com.sayedbaladoh.buzzdiggr.producer.property.TwitterProperties;
15 | import com.sayedbaladoh.buzzdiggr.producer.service.kafka.Sender;
16 | import com.twitter.hbc.ClientBuilder;
17 | import com.twitter.hbc.core.Client;
18 | import com.twitter.hbc.core.Constants;
19 | import com.twitter.hbc.core.endpoint.StatusesFilterEndpoint;
20 | import com.twitter.hbc.core.processor.StringDelimitedProcessor;
21 | import com.twitter.hbc.httpclient.auth.Authentication;
22 | import com.twitter.hbc.httpclient.auth.OAuth1;
23 |
24 | /**
25 | * Twitter Streaming Service
26 | *
27 | * @author SayedBaladoh
28 | *
29 | */
30 | @Service
31 | public class TwitterStream {
32 |
33 | private static final Logger LOGGER = LoggerFactory.getLogger(TwitterStream.class);
34 |
35 | @Autowired
36 | TwitterProperties twitterProperties;
37 |
38 | @Autowired
39 | TweetHandler tweetHandler;
40 |
41 | @Autowired
42 | private Sender sender;
43 |
44 | @Value("${kafka.topic.json.tweet}")
45 | private String tweetTopicName;
46 |
47 | @Value("${social.twitter.count}")
48 | private int count;
49 |
50 | /**
51 | * Stream tweets from twitter with default (100) total number of tweets to
52 | * stream
53 | * @param topics
54 | * - List of topics to search
55 | * @throws InterruptedException
56 | */
57 | public void stream(String... topics) throws InterruptedException {
58 | this.stream(this.count, topics);
59 | }
60 |
61 | /**
62 | * Stream tweets from twitter
63 | *
64 | * @param numberOfTweets
65 | * - Number of tweets
66 | * @param topics
67 | * - List of topics to search
68 | * @throws InterruptedException
69 | */
70 | public void stream(int numberOfTweets, String... topics) throws InterruptedException {
71 |
72 | // Creating Twitter Stream
73 | BlockingQueue queue = new LinkedBlockingQueue(10000);
74 |
75 | StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();
76 |
77 | // Add some track terms
78 | endpoint.trackTerms(Lists.newArrayList(topics));
79 |
80 | Authentication auth = new OAuth1(twitterProperties.getApiKey(), twitterProperties.getApiSecretKey(),
81 | twitterProperties.getAccessToken(), twitterProperties.getAccessTokenSecret());
82 |
83 | // Create a new BasicClient. By default gzip is enabled.
84 | Client client = new ClientBuilder().hosts(Constants.STREAM_HOST).endpoint(endpoint).authentication(auth)
85 | .processor(new StringDelimitedProcessor(queue)).build();
86 |
87 | // Establish a connection
88 | client.connect();
89 |
90 | // Parse and process tweets
91 | for (int msgRead = 0; msgRead < numberOfTweets; msgRead++) {
92 | String msg = queue.take();
93 |
94 | // LOGGER.info("Message: {} ", msg);
95 |
96 | // Parse and Clean Msg
97 | if (msg != null) {
98 | Tweet tweet = tweetHandler.parseTweet(msg);
99 | // LOGGER.info("Text: {} ", tweet.getText());
100 | LOGGER.info("Tweet: {} ", tweet);
101 |
102 | // Send tweet to Kafka
103 | sender.send(tweetTopicName, tweet);
104 | }
105 |
106 | }
107 |
108 | client.stop();
109 |
110 | LOGGER.info("'{}' messages processed!\n", client.getStatsTracker().getNumMessages());
111 |
112 | }
113 |
114 | }
115 |
--------------------------------------------------------------------------------
/producer/src/main/java/com/sayedbaladoh/buzzdiggr/producer/util/StringUtils.java:
--------------------------------------------------------------------------------
1 | package com.sayedbaladoh.buzzdiggr.producer.util;
2 |
3 | public class StringUtils {
4 |
5 | // ToDo : Remove Stop Words, Grammar checking, Spelling correction
6 | /**
7 | * Simple Text Cleaner
8 | *
9 | * @param text
10 | * - Text to clean
11 | * @return
12 | * - The cleaned text
13 | *
14 | * @author SayedBaladoh
15 | */
16 | public static String cleanText(String text) {
17 |
18 | if (text == null && text.equals(""))
19 | return "";
20 | // System.out.println("Dirty text: " + text);
21 | // Delete all usernames mentioned (?:\\s|\\A)@+([A-Za-z0-9-_]+)
22 | // Delete all RT (retweets flags) (RT)
23 | // Delete all hashtags mentioned (?:\\s|\\A)#+([A-Za-z0-9-_]+)
24 | // Delete all URLs (https?://(\\w+\\.)+\\S*)
25 | // Delete all no printable characters and Emojis ([\ud83c\udc00-\ud83c\udfff]|[\ud83d\udc00-\ud83d\udfff]|[\u2600-\u27ff]+)
26 | // Replace all break lines with spaces (\n)
27 | // Replace all double spaces with single spaces " {2,}"
28 | // Remove leading/trailing spaces trim()
29 |
30 | text = text.replaceAll("((?:\\s|\\A)@[A-Za-z0-9-_]+)|(RT)|((?:\\s|\\A)#+([A-Za-z0-9-_]+))|(https?://(\\w+\\.)+\\S*)|([\ud83c\udc00-\ud83c\udfff]|[\ud83d\udc00-\ud83d\udfff]|[\u2600-\u27ff]+)", "").replaceAll("\n", " ").replaceAll(" {2,}", " ").trim();
31 | // System.out.println("Cleaned text: " + text);
32 | return text;
33 | }
34 |
35 | /**
36 | * Test Case.
37 | *
38 | * @param args
39 | * - not used
40 | */
41 | public static void main(String[] args) {
42 | String text = "öäü L'alphabet est génial 😀! I luv صراع الكرة الذهبية 🏆\n\n🇦🇷 (الدوري my <3 iphone @abc12 & you’re awsm #apple. DisplayIsAwesomehttps://www.apple, sooo happppppy 🙂 http://www.apple.com sdas 👽😀☂❤华み원❤";
43 | text="RT @H45HEM: \u0645\u062d\u0645\u062f \u0635\u0644\u0627\u062d \u064a\u0646\u0642\u0630 \u0633\u0645\u0643\u0629 \u0645\u0646 \u0627\u0644\u063a\u0631\u0642! \u0648\u0627\u0644\u0646\u0639\u0645 \u0641\u064a\u0643 \u064a\u0627 \u0641\u062e\u0631 \u0627\u0644\u0639\u0631\u0628. https://t.co/8rGdLuGqKE";
44 | System.out.println("Dirty text: " + text);
45 | System.out.println("------------------------------------------------------------------------------");
46 | System.out.println("Cleaned text: " + cleanText(text));
47 | System.out.println("------------------------------------------------------------------------------");
48 | }
49 |
50 | }
51 |
--------------------------------------------------------------------------------
/producer/src/main/resources/application.properties:
--------------------------------------------------------------------------------
1 | # Server Properties
2 | spring.application.name=producer-service
3 | server.port=8081
4 | server.servlet.context-path=/producer
5 |
6 | # INFO Endpoint Configuration
7 | info.app.name=@project.name@
8 | info.app.description=@project.description@
9 | info.app.version=@project.version@
10 | info.app.encoding=@project.build.sourceEncoding@
11 | info.app.java.version=@java.version@
12 |
13 | #URLs for allow CrossOrigin
14 | frontend.url = http://localhost:4200
--------------------------------------------------------------------------------
/producer/src/main/resources/application.yml:
--------------------------------------------------------------------------------
1 | #Kafka Configs
2 | kafka:
3 | boot:
4 | server: 127.0.0.1:9092
5 | topic:
6 | string:
7 | name: strings
8 | json:
9 | tweet: tweets
10 | article: articles
11 |
12 | # Twitter Social Configs
13 | social:
14 | twitter:
15 | auth:
16 | apiKey: API_KEY
17 | apiSecretKey: API_SECRET_KEY
18 | accessToken: ACCESS_TOKEN
19 | accessTokenSecret: ACCESS_TOKEN_SECRET
20 | count: 100
21 |
22 | # Crawler Configs
23 | crawler:
24 | maxPagesToSearch: 100
25 | pagesLimit: 20
26 |
--------------------------------------------------------------------------------
/producer/src/test/java/com/sayedbaladoh/buzzdiggr/producer/ProducerApplicationTests.java:
--------------------------------------------------------------------------------
1 | package com.sayedbaladoh.buzzdiggr.producer;
2 |
3 | import org.junit.Test;
4 | import org.junit.runner.RunWith;
5 | import org.springframework.boot.test.context.SpringBootTest;
6 | import org.springframework.test.context.junit4.SpringRunner;
7 |
8 | @RunWith(SpringRunner.class)
9 | @SpringBootTest
10 | public class ProducerApplicationTests {
11 |
12 | @Test
13 | public void contextLoads() {
14 | }
15 |
16 | }
17 |
--------------------------------------------------------------------------------
32 |
33 |
34 |
35 |
36 |
48 |
49 |
37 |
40 |
43 |
46 |
47 |
50 |
--------------------------------------------------------------------------------
/front-end/src/app/search/search.component.spec.ts:
--------------------------------------------------------------------------------
1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing';
2 |
3 | import { SearchComponent } from './search.component';
4 |
5 | describe('SearchComponent', () => {
6 | let component: SearchComponent;
7 | let fixture: ComponentFixture
51 |
53 | Not found!
52 |
54 |
64 |
55 |
58 |
59 | {{results.totalElements}} Hits, {{results.totalPages}} Pages
56 |Page: {{results.number+1}}, Displaying Results: ({{results.numberOfElements}})
57 |
60 |
61 |
62 |
63 |
65 |
66 |
101 |
102 |
67 |
100 |
68 |
79 |
80 |
91 |
92 | 69 | {{result.userName}} 70 |
71 |{{result.text}}
72 |73 | {{result.language | uppercase}} 74 |
75 |76 | {{result.date | date:'long'}} 77 |
78 |
93 |
99 | 94 | {{result.title}} 95 |
96 |{{result.text}}
97 | 98 |2 | Settings 3 |
4 |5 |
6 | Streamer Settings 7 |
8 |
9 |
10 | 
11 |
12 |
13 |
11 |
14 |
34 |
35 | Launch real-time tweets Streamer
15 | 32 | 33 |36 |
37 | Crawler Settings 38 |
39 |
40 |
41 |
42 |
43 |
42 |
44 |
--------------------------------------------------------------------------------
/front-end/src/app/setting/setting.component.spec.ts:
--------------------------------------------------------------------------------
1 | import { async, ComponentFixture, TestBed } from '@angular/core/testing';
2 |
3 | import { SettingComponent } from './setting.component';
4 |
5 | describe('SettingComponent', () => {
6 | let component: SettingComponent;
7 | let fixture: ComponentFixtureLaunch web crawler
45 | 71 | 72 |9 | * Properties are configured in the application.properties file. 10 | * Bind all the file Twitter properties to a POJO class 11 | *
12 | * 13 | * @author SayedBaladoh 14 | */ 15 | @ConfigurationProperties(prefix = "social.twitter.auth", ignoreUnknownFields = false) 16 | public class TwitterProperties { 17 | 18 | private String apiKey; 19 | private String apiSecretKey; 20 | private String accessToken; 21 | private String accessTokenSecret; 22 | public String getApiKey() { 23 | return apiKey; 24 | } 25 | public void setApiKey(String apiKey) { 26 | this.apiKey = apiKey; 27 | } 28 | public String getApiSecretKey() { 29 | return apiSecretKey; 30 | } 31 | public void setApiSecretKey(String apiSecretKey) { 32 | this.apiSecretKey = apiSecretKey; 33 | } 34 | public String getAccessToken() { 35 | return accessToken; 36 | } 37 | public void setAccessToken(String accessToken) { 38 | this.accessToken = accessToken; 39 | } 40 | public String getAccessTokenSecret() { 41 | return accessTokenSecret; 42 | } 43 | public void setAccessTokenSecret(String accessTokenSecret) { 44 | this.accessTokenSecret = accessTokenSecret; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /producer/src/main/java/com/sayedbaladoh/buzzdiggr/producer/service/crawler/Spider.java: -------------------------------------------------------------------------------- 1 | package com.sayedbaladoh.buzzdiggr.producer.service.crawler; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashSet; 5 | import java.util.List; 6 | import java.util.Set; 7 | 8 | import org.jsoup.Connection; 9 | import org.jsoup.Jsoup; 10 | import org.jsoup.nodes.Document; 11 | import org.jsoup.nodes.Element; 12 | import org.jsoup.select.Elements; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | import org.springframework.beans.factory.annotation.Autowired; 16 | import org.springframework.beans.factory.annotation.Value; 17 | import org.springframework.stereotype.Service; 18 | 19 | import com.sayedbaladoh.buzzdiggr.producer.model.Article; 20 | import com.sayedbaladoh.buzzdiggr.producer.service.kafka.Sender; 21 | 22 | 23 | /** 24 | * Simple web crawler 25 | * 26 | * @author SayedBaladoh 27 | * 28 | * Adapted from: 29 | * @see How to make a simple web crawler in Java - 30 | * http://www.netinstructions.com/how-to-make-a-simple-web-crawler-in-java/ 31 | * @see Simple web crawler - 32 | * https://www.programcreek.com/java-api-examples/?code=PacktPublishing/Machine-Learning-End-to-Endguide-for-Java-developers/Machine-Learning-End-to-Endguide-for-Java-developers-master/Module%201/JavaforDataScience_Code/chapter02/SimpleWebCrawler.java 33 | * 34 | */ 35 | @Service 36 | public class Spider { 37 | 38 | private static final Logger LOGGER = LoggerFactory.getLogger(Spider.class); 39 | 40 | @Autowired 41 | private Sender sender; 42 | 43 | @Value("${kafka.topic.json.article}") 44 | private String articleTopicName; 45 | 46 | @Value("${crawler.maxPagesToSearch}") 47 | private int maxPagesToSearch; 48 | @Value("${crawler.pagesLimit}") 49 | private int pagesLimit; 50 | 51 | private String startingURL; 52 | private Set