30 |
49 |
31 | 
32 |
33 |
34 |
48 |
35 |
47 |
36 |
46 | {this.props.news.title}
37 |
38 |
45 | {this.props.news.description}
39 |
40 | {this.props.news.source.id != null &&
44 | {this.props.news.source.name}
}
41 | {this.props.news.reason != null && {this.props.news.reason}
}
42 | {this.props.news.time != null && {this.props.news.time}
}
43 | 
6 |
7 | ### News Pipeline
8 | News pipeline is composed by news monitor, web scraper and news deduper, news is sent and received between them by RabbitMQ which decouples these components. The news monitor use [News API](https://newsapi.org) to derive latest news and store news title MD5 digest into Redis to avoid sending same news to the message queue. The web scraper use a third party package [Newspaper](https://newspaper.readthedocs.io/en/latest/) to fetch corresponding news articles from offical news website. News depuper implements TF-IDF to calculate similarity of news to avoid storing same news from different news source into MongoDB. For similar news, only store the one published firstly.
9 |
10 | ### News Topic Modeling
11 | The news topic classification is implemented by Convolutional Neutral Nework(CNN) in TensorFlow and deployed online using the offline trained model. Manually label the news with 17 classes: `Colleges & Schools`, `Environmental`, `World`, `Entertainment`, `Media`, `Politics & Government`, `Regional News`, `Religion`, `Sports`, `Technology`, `Traffic`, `Weather`, `Economic & Corp`, `Advertisements`, `Crime`, `Magazine`, `Other`.
12 |
13 | ### News Recommendation
14 | I built a click log processor to implement a time decay model. If a news topic is clicked, p = (1-α)p + α, if not, p = (1-α)p, Where p is the selection probability, and α is the degree of weight decrease. The result of this is that the nth most recent selection will have a weight of (1-α)^n. Using a coefficient value of 0.05 as an example, the 10th most recent selection would only have half the weight of the most recent. Increasing α would bias towards more recent results more.
15 |
16 | When some news' topic with the most probability for that user, the webpage will show a "Recommend" tag to the user.
17 |
--------------------------------------------------------------------------------
/news_topic_modeling_service/trainer/news_class_trainer.py:
--------------------------------------------------------------------------------
1 | import news_cnn_model
2 | import numpy as np
3 | import os
4 | import pandas as pd
5 | import pickle
6 | import shutil
7 | import tensorflow as tf
8 |
9 | from sklearn import metrics
10 |
11 | learn = tf.contrib.learn
12 |
13 | REMOVE_PREVIOUS_MODEL = True
14 |
15 | MODEL_OUTPUT_DIR = '../model/'
16 | DATA_SET_FILE = '../data/labeled_news.csv'
17 | VARS_FILE = '../model/vars'
18 | VOCAB_PROCESSOR_SAVE_FILE = '../model/vocab_procesor_save_file'
19 | MAX_DOCUMENT_LENGTH = 100
20 | N_CLASSES = 8
21 |
22 | # Training parms
23 | STEPS = 200
24 |
25 | def main(unused_argv):
26 | if REMOVE_PREVIOUS_MODEL:
27 | # Remove old model
28 | print("Removing previous model...")
29 | shutil.rmtree(MODEL_OUTPUT_DIR)
30 | os.mkdir(MODEL_OUTPUT_DIR)
31 |
32 | # Prepare training and testing data
33 | df = pd.read_csv(DATA_SET_FILE, header=None)
34 | train_df = df[0:400]
35 | test_df = df.drop(train_df.index)
36 |
37 | # x - news title, y - class
38 | x_train = train_df[1]
39 | y_train = train_df[0]
40 | x_test = test_df[1]
41 | y_test = test_df[0]
42 |
43 | # Process vocabulary
44 | vocab_processor = learn.preprocessing.VocabularyProcessor(MAX_DOCUMENT_LENGTH)
45 | x_train = np.array(list(vocab_processor.fit_transform(x_train)))
46 | x_test = np.array(list(vocab_processor.transform(x_test)))
47 |
48 | n_words = len(vocab_processor.vocabulary_)
49 | print('Total words: %d' % n_words)
50 |
51 | # Saving n_words and vocab_processor:
52 | with open(VARS_FILE, 'wb') as f: # needs to be opened in binary mode.
53 | pickle.dump(n_words, f)
54 |
55 | vocab_processor.save(VOCAB_PROCESSOR_SAVE_FILE)
56 |
57 | # Build model
58 | classifier = learn.Estimator(
59 | model_fn=news_cnn_model.generate_cnn_model(N_CLASSES, n_words),
60 | model_dir=MODEL_OUTPUT_DIR)
61 |
62 | # Train and predict
63 | classifier.fit(x_train, y_train, steps=STEPS)
64 |
65 | # Evaluate model
66 | y_predicted = [
67 | p['class'] for p in classifier.predict(x_test, as_iterable=True)
68 | ]
69 |
70 | score = metrics.accuracy_score(y_test, y_predicted)
71 | print('Accuracy: {0:f}'.format(score))
72 |
73 | if __name__ == '__main__':
74 | tf.app.run(main=main)
75 |
--------------------------------------------------------------------------------
/web_server/client/src/Signup/SignUpForm.js:
--------------------------------------------------------------------------------
1 | import './SignUpForm.css';
2 |
3 | import { Link } from 'react-router'
4 | import PropTypes from 'prop-types';
5 | import React from 'react';
6 |
7 | const SignUpForm = ({
8 | onSubmit,
9 | onChange,
10 | errors,
11 | user,
12 | }) => (
13 |