├── .DS_Store
├── .gitignore
├── 1_twitter_stream
    ├── status.json
    └── twitter_filter.py
├── 2_sentiment
    └── twitter_filter.py
├── 3_redis
    ├── __pycache__
    │   └── tweet_store.cpython-36.pyc
    ├── dump.rdb
    ├── tweet_store.py
    └── twitter_filter.py
├── 4_design_front_end
    ├── index.html
    ├── pen_paper.jpg
    └── styles.css
├── 5_python_class
    ├── tweet.py
    ├── tweet_store.py
    └── twitter_filter.py
├── 6_flask
    ├── __pycache__
    │   ├── tweet.cpython-36.pyc
    │   └── tweet_store.cpython-36.pyc
    ├── dump.rdb
    ├── static
    │   └── styles.css
    ├── templates
    │   └── index.html
    ├── twatcher.py
    ├── tweet.py
    ├── tweet_store.py
    └── twitter_filter.py
└── README.md


/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/k2datascience/twitter_filter/1c6df786281b8254675a8ff7b3134dfbfdcd573d/.DS_Store


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /config
2 | 


--------------------------------------------------------------------------------
/1_twitter_stream/status.json:
--------------------------------------------------------------------------------
  1 | Status(_api = < tweepy.api.API object at 0x10da339b0 > , _json = {
  2 |   'created_at': 'Wed Oct 24 13:36:16 +0000 2018',
  3 |   'id': 1055090601106989056,
  4 |   'id_str': '1055090601106989056',
  5 |   'text': 'pineapple on pizza https://t.co/Pwr9We9QA2',
  6 |   'display_text_range': [0, 18],
  7 |   'source': '<a href="http://twitter.com/download/iphone" rel="nofollow">Twitter for iPhone</a>',
  8 |   'truncated': False,
  9 |   'in_reply_to_status_id': None,
 10 |   'in_reply_to_status_id_str': None,
 11 |   'in_reply_to_user_id': None,
 12 |   'in_reply_to_user_id_str': None,
 13 |   'in_reply_to_screen_name': None,
 14 |   'user': {
 15 |     'id': 1837578733,
 16 |     'id_str': '1837578733',
 17 |     'name': 'josie',
 18 |     'screen_name': 'JocelynBChu',
 19 |     'location': 'ɹǝuǝʇǝǝʍs',
 20 |     'url': None,
 21 |     'description': 'peachy',
 22 |     'translator_type': 'none',
 23 |     'protected': False,
 24 |     'verified': False,
 25 |     'followers_count': 586,
 26 |     'friends_count': 65,
 27 |     'listed_count': 4,
 28 |     'favourites_count': 3523,
 29 |     'statuses_count': 21091,
 30 |     'created_at': 'Mon Sep 09 01:47:37 +0000 2013',
 31 |     'utc_offset': None,
 32 |     'time_zone': None,
 33 |     'geo_enabled': False,
 34 |     'lang': 'en',
 35 |     'contributors_enabled': False,
 36 |     'is_translator': False,
 37 |     'profile_background_color': '000000',
 38 |     'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',
 39 |     'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',
 40 |     'profile_background_tile': False,
 41 |     'profile_link_color': 'F58EA8',
 42 |     'profile_sidebar_border_color': '000000',
 43 |     'profile_sidebar_fill_color': '000000',
 44 |     'profile_text_color': '000000',
 45 |     'profile_use_background_image': False,
 46 |     'profile_image_url': 'http://pbs.twimg.com/profile_images/1042100383873622016/TjPUseCJ_normal.jpg',
 47 |     'profile_image_url_https': 'https://pbs.twimg.com/profile_images/1042100383873622016/TjPUseCJ_normal.jpg',
 48 |     'profile_banner_url': 'https://pbs.twimg.com/profile_banners/1837578733/1524743204',
 49 |     'default_profile': False,
 50 |     'default_profile_image': False,
 51 |     'following': None,
 52 |     'follow_request_sent': None,
 53 |     'notifications': None
 54 |   },
 55 |   'geo': None,
 56 |   'coordinates': None,
 57 |   'place': None,
 58 |   'contributors': None,
 59 |   'is_quote_status': False,
 60 |   'quote_count': 0,
 61 |   'reply_count': 0,
 62 |   'retweet_count': 0,
 63 |   'favorite_count': 0,
 64 |   'entities': {
 65 |     'hashtags': [],
 66 |     'urls': [],
 67 |     'user_mentions': [],
 68 |     'symbols': [],
 69 |     'media': [{
 70 |       'id': 1055090590751186945,
 71 |       'id_str': '1055090590751186945',
 72 |       'indices': [19, 42],
 73 |       'media_url': 'http://pbs.twimg.com/media/DqRvTVuU4AECE7w.jpg',
 74 |       'media_url_https': 'https://pbs.twimg.com/media/DqRvTVuU4AECE7w.jpg',
 75 |       'url': 'https://t.co/Pwr9We9QA2',
 76 |       'display_url': 'pic.twitter.com/Pwr9We9QA2',
 77 |       'expanded_url': 'https://twitter.com/JocelynBChu/status/1055090601106989056/photo/1',
 78 |       'type': 'photo',
 79 |       'sizes': {
 80 |         'large': {
 81 |           'w': 1536,
 82 |           'h': 2048,
 83 |           'resize': 'fit'
 84 |         },
 85 |         'thumb': {
 86 |           'w': 150,
 87 |           'h': 150,
 88 |           'resize': 'crop'
 89 |         },
 90 |         'medium': {
 91 |           'w': 900,
 92 |           'h': 1200,
 93 |           'resize': 'fit'
 94 |         },
 95 |         'small': {
 96 |           'w': 510,
 97 |           'h': 680,
 98 |           'resize': 'fit'
 99 |         }
100 |       }
101 |     }]
102 |   },
103 |   'extended_entities': {
104 |     'media': [{
105 |       'id': 1055090590751186945,
106 |       'id_str': '1055090590751186945',
107 |       'indices': [19, 42],
108 |       'media_url': 'http://pbs.twimg.com/media/DqRvTVuU4AECE7w.jpg',
109 |       'media_url_https': 'https://pbs.twimg.com/media/DqRvTVuU4AECE7w.jpg',
110 |       'url': 'https://t.co/Pwr9We9QA2',
111 |       'display_url': 'pic.twitter.com/Pwr9We9QA2',
112 |       'expanded_url': 'https://twitter.com/JocelynBChu/status/1055090601106989056/photo/1',
113 |       'type': 'photo',
114 |       'sizes': {
115 |         'large': {
116 |           'w': 1536,
117 |           'h': 2048,
118 |           'resize': 'fit'
119 |         },
120 |         'thumb': {
121 |           'w': 150,
122 |           'h': 150,
123 |           'resize': 'crop'
124 |         },
125 |         'medium': {
126 |           'w': 900,
127 |           'h': 1200,
128 |           'resize': 'fit'
129 |         },
130 |         'small': {
131 |           'w': 510,
132 |           'h': 680,
133 |           'resize': 'fit'
134 |         }
135 |       }
136 |     }]
137 |   },
138 |   'favorited': False,
139 |   'retweeted': False,
140 |   'possibly_sensitive': False,
141 |   'filter_level': 'low',
142 |   'lang': 'en',
143 |   'timestamp_ms': '1540388176596'
144 | }, created_at = datetime.datetime(2018, 10, 24, 13, 36, 16), id = 1055090601106989056, id_str = '1055090601106989056', text = 'pineapple on pizza https://t.co/Pwr9We9QA2', display_text_range = [0, 18], source = 'Twitter for iPhone', source_url = 'http://twitter.com/download/iphone', truncated = False, in_reply_to_status_id = None, in_reply_to_status_id_str = None, in_reply_to_user_id = None, in_reply_to_user_id_str = None, in_reply_to_screen_name = None, author = User(_api = < tweepy.api.API object at 0x10da339b0 > , _json = {
145 |   'id': 1837578733,
146 |   'id_str': '1837578733',
147 |   'name': 'josie',
148 |   'screen_name': 'JocelynBChu',
149 |   'location': 'ɹǝuǝʇǝǝʍs',
150 |   'url': None,
151 |   'description': 'peachy',
152 |   'translator_type': 'none',
153 |   'protected': False,
154 |   'verified': False,
155 |   'followers_count': 586,
156 |   'friends_count': 65,
157 |   'listed_count': 4,
158 |   'favourites_count': 3523,
159 |   'statuses_count': 21091,
160 |   'created_at': 'Mon Sep 09 01:47:37 +0000 2013',
161 |   'utc_offset': None,
162 |   'time_zone': None,
163 |   'geo_enabled': False,
164 |   'lang': 'en',
165 |   'contributors_enabled': False,
166 |   'is_translator': False,
167 |   'profile_background_color': '000000',
168 |   'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',
169 |   'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',
170 |   'profile_background_tile': False,
171 |   'profile_link_color': 'F58EA8',
172 |   'profile_sidebar_border_color': '000000',
173 |   'profile_sidebar_fill_color': '000000',
174 |   'profile_text_color': '000000',
175 |   'profile_use_background_image': False,
176 |   'profile_image_url': 'http://pbs.twimg.com/profile_images/1042100383873622016/TjPUseCJ_normal.jpg',
177 |   'profile_image_url_https': 'https://pbs.twimg.com/profile_images/1042100383873622016/TjPUseCJ_normal.jpg',
178 |   'profile_banner_url': 'https://pbs.twimg.com/profile_banners/1837578733/1524743204',
179 |   'default_profile': False,
180 |   'default_profile_image': False,
181 |   'following': None,
182 |   'follow_request_sent': None,
183 |   'notifications': None
184 | }, id = 1837578733, id_str = '1837578733', name = 'josie', screen_name = 'JocelynBChu', location = 'ɹǝuǝʇǝǝʍs', url = None, description = 'peachy', translator_type = 'none', protected = False, verified = False, followers_count = 586, friends_count = 65, listed_count = 4, favourites_count = 3523, statuses_count = 21091, created_at = datetime.datetime(2013, 9, 9, 1, 47, 37), utc_offset = None, time_zone = None, geo_enabled = False, lang = 'en', contributors_enabled = False, is_translator = False, profile_background_color = '000000', profile_background_image_url = 'http://abs.twimg.com/images/themes/theme1/bg.png', profile_background_image_url_https = 'https://abs.twimg.com/images/themes/theme1/bg.png', profile_background_tile = False, profile_link_color = 'F58EA8', profile_sidebar_border_color = '000000', profile_sidebar_fill_color = '000000', profile_text_color = '000000', profile_use_background_image = False, profile_image_url = 'http://pbs.twimg.com/profile_images/1042100383873622016/TjPUseCJ_normal.jpg', profile_image_url_https = 'https://pbs.twimg.com/profile_images/1042100383873622016/TjPUseCJ_normal.jpg', profile_banner_url = 'https://pbs.twimg.com/profile_banners/1837578733/1524743204', default_profile = False, default_profile_image = False, following = False, follow_request_sent = None, notifications = None), user = User(_api = < tweepy.api.API object at 0x10da339b0 > , _json = {
185 |   'id': 1837578733,
186 |   'id_str': '1837578733',
187 |   'name': 'josie',
188 |   'screen_name': 'JocelynBChu',
189 |   'location': 'ɹǝuǝʇǝǝʍs',
190 |   'url': None,
191 |   'description': 'peachy',
192 |   'translator_type': 'none',
193 |   'protected': False,
194 |   'verified': False,
195 |   'followers_count': 586,
196 |   'friends_count': 65,
197 |   'listed_count': 4,
198 |   'favourites_count': 3523,
199 |   'statuses_count': 21091,
200 |   'created_at': 'Mon Sep 09 01:47:37 +0000 2013',
201 |   'utc_offset': None,
202 |   'time_zone': None,
203 |   'geo_enabled': False,
204 |   'lang': 'en',
205 |   'contributors_enabled': False,
206 |   'is_translator': False,
207 |   'profile_background_color': '000000',
208 |   'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',
209 |   'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',
210 |   'profile_background_tile': False,
211 |   'profile_link_color': 'F58EA8',
212 |   'profile_sidebar_border_color': '000000',
213 |   'profile_sidebar_fill_color': '000000',
214 |   'profile_text_color': '000000',
215 |   'profile_use_background_image': False,
216 |   'profile_image_url': 'http://pbs.twimg.com/profile_images/1042100383873622016/TjPUseCJ_normal.jpg',
217 |   'profile_image_url_https': 'https://pbs.twimg.com/profile_images/1042100383873622016/TjPUseCJ_normal.jpg',
218 |   'profile_banner_url': 'https://pbs.twimg.com/profile_banners/1837578733/1524743204',
219 |   'default_profile': False,
220 |   'default_profile_image': False,
221 |   'following': None,
222 |   'follow_request_sent': None,
223 |   'notifications': None
224 | }, id = 1837578733, id_str = '1837578733', name = 'josie', screen_name = 'JocelynBChu', location = 'ɹǝuǝʇǝǝʍs', url = None, description = 'peachy', translator_type = 'none', protected = False, verified = False, followers_count = 586, friends_count = 65, listed_count = 4, favourites_count = 3523, statuses_count = 21091, created_at = datetime.datetime(2013, 9, 9, 1, 47, 37), utc_offset = None, time_zone = None, geo_enabled = False, lang = 'en', contributors_enabled = False, is_translator = False, profile_background_color = '000000', profile_background_image_url = 'http://abs.twimg.com/images/themes/theme1/bg.png', profile_background_image_url_https = 'https://abs.twimg.com/images/themes/theme1/bg.png', profile_background_tile = False, profile_link_color = 'F58EA8', profile_sidebar_border_color = '000000', profile_sidebar_fill_color = '000000', profile_text_color = '000000', profile_use_background_image = False, profile_image_url = 'http://pbs.twimg.com/profile_images/1042100383873622016/TjPUseCJ_normal.jpg', profile_image_url_https = 'https://pbs.twimg.com/profile_images/1042100383873622016/TjPUseCJ_normal.jpg', profile_banner_url = 'https://pbs.twimg.com/profile_banners/1837578733/1524743204', default_profile = False, default_profile_image = False, following = False, follow_request_sent = None, notifications = None), geo = None, coordinates = None, place = None, contributors = None, is_quote_status = False, quote_count = 0, reply_count = 0, retweet_count = 0, favorite_count = 0, entities = {
225 |   'hashtags': [],
226 |   'urls': [],
227 |   'user_mentions': [],
228 |   'symbols': [],
229 |   'media': [{
230 |     'id': 1055090590751186945,
231 |     'id_str': '1055090590751186945',
232 |     'indices': [19, 42],
233 |     'media_url': 'http://pbs.twimg.com/media/DqRvTVuU4AECE7w.jpg',
234 |     'media_url_https': 'https://pbs.twimg.com/media/DqRvTVuU4AECE7w.jpg',
235 |     'url': 'https://t.co/Pwr9We9QA2',
236 |     'display_url': 'pic.twitter.com/Pwr9We9QA2',
237 |     'expanded_url': 'https://twitter.com/JocelynBChu/status/1055090601106989056/photo/1',
238 |     'type': 'photo',
239 |     'sizes': {
240 |       'large': {
241 |         'w': 1536,
242 |         'h': 2048,
243 |         'resize': 'fit'
244 |       },
245 |       'thumb': {
246 |         'w': 150,
247 |         'h': 150,
248 |         'resize': 'crop'
249 |       },
250 |       'medium': {
251 |         'w': 900,
252 |         'h': 1200,
253 |         'resize': 'fit'
254 |       },
255 |       'small': {
256 |         'w': 510,
257 |         'h': 680,
258 |         'resize': 'fit'
259 |       }
260 |     }
261 |   }]
262 | }, extended_entities = {
263 |   'media': [{
264 |     'id': 1055090590751186945,
265 |     'id_str': '1055090590751186945',
266 |     'indices': [19, 42],
267 |     'media_url': 'http://pbs.twimg.com/media/DqRvTVuU4AECE7w.jpg',
268 |     'media_url_https': 'https://pbs.twimg.com/media/DqRvTVuU4AECE7w.jpg',
269 |     'url': 'https://t.co/Pwr9We9QA2',
270 |     'display_url': 'pic.twitter.com/Pwr9We9QA2',
271 |     'expanded_url': 'https://twitter.com/JocelynBChu/status/1055090601106989056/photo/1',
272 |     'type': 'photo',
273 |     'sizes': {
274 |       'large': {
275 |         'w': 1536,
276 |         'h': 2048,
277 |         'resize': 'fit'
278 |       },
279 |       'thumb': {
280 |         'w': 150,
281 |         'h': 150,
282 |         'resize': 'crop'
283 |       },
284 |       'medium': {
285 |         'w': 900,
286 |         'h': 1200,
287 |         'resize': 'fit'
288 |       },
289 |       'small': {
290 |         'w': 510,
291 |         'h': 680,
292 |         'resize': 'fit'
293 |       }
294 |     }
295 |   }]
296 | }, favorited = False, retweeted = False, possibly_sensitive = False, filter_level = 'low', lang = 'en', timestamp_ms = '1540388176596')
297 | 


--------------------------------------------------------------------------------
/1_twitter_stream/twitter_filter.py:
--------------------------------------------------------------------------------
 1 | import tweepy
 2 | import datetime
 3 | import json
 4 | 
 5 | file_path = '../config/api.json'
 6 | 
 7 | with open(file_path) as f:
 8 |     twitter_api = json.loads(f.read())
 9 | 
10 | consumer_key = twitter_api['consumer_key']
11 | consumer_secret = twitter_api['consumer_secret']
12 | access_token = twitter_api['access_token']
13 | access_token_secret = twitter_api['access_token_secret']
14 | 
15 | auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
16 | auth.set_access_token(access_token, access_token_secret)
17 | 
18 | api = tweepy.API(auth)
19 | 
20 | class StreamListener(tweepy.StreamListener):
21 | 
22 |     def on_status(self, status):
23 | 
24 |         if ('RT @' not in status.text):
25 |             tweet_item = {
26 |                 'id_str': status.id_str,
27 |                 'text': status.text,
28 |                 'username': status.user.screen_name,
29 |                 'name': status.user.name,
30 |                 'profile_image_url': status.user.profile_image_url,
31 |                 'received_at': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
32 |             }
33 | 
34 |             print(tweet_item)
35 | 
36 |     def on_error(self, status_code):
37 |         if status_code == 420:
38 |             return False
39 | 
40 | stream_listener = StreamListener()
41 | stream = tweepy.Stream(auth=api.auth, listener=stream_listener)
42 | stream.filter(track=["@WarbyParker", "@Bonobos", "@Casper", "@Glossier", "@DollarShaveClub", "@Allbirds"])
43 | 


--------------------------------------------------------------------------------
/2_sentiment/twitter_filter.py:
--------------------------------------------------------------------------------
 1 | import tweepy
 2 | import datetime
 3 | from textblob import TextBlob
 4 | import json
 5 | 
 6 | file_path = '../config/api.json'
 7 | 
 8 | with open(file_path) as f:
 9 |     twitter_api = json.loads(f.read())
10 | 
11 | consumer_key = twitter_api['consumer_key']
12 | consumer_secret = twitter_api['consumer_secret']
13 | access_token = twitter_api['access_token']
14 | access_token_secret = twitter_api['access_token_secret']
15 | 
16 | auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
17 | auth.set_access_token(access_token, access_token_secret)
18 | 
19 | api = tweepy.API(auth)
20 | 
21 | class StreamListener(tweepy.StreamListener):
22 | 
23 |     def on_status(self, status):
24 | 
25 |         if ('RT @' not in status.text):
26 |             blob = TextBlob(status.text)
27 |             sent = blob.sentiment
28 |             polarity = sent.polarity
29 |             subjectivity = sent.subjectivity
30 | 
31 |             tweet_item = {
32 |                 'id_str': status.id_str,
33 |                 'text': status.text,
34 |                 'polarity': polarity,
35 |                 'subjectivity': subjectivity,
36 |                 'username': status.user.screen_name,
37 |                 'name': status.user.name,
38 |                 'profile_image_url': status.user.profile_image_url,
39 |                 'received_at': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
40 |             }
41 | 
42 |             print(tweet_item)
43 | 
44 |     def on_error(self, status_code):
45 |         if status_code == 420:
46 |             return False
47 | 
48 | stream_listener = StreamListener()
49 | stream = tweepy.Stream(auth=api.auth, listener=stream_listener)
50 | stream.filter(track=["@WarbyParker", "@Bonobos", "@Casper", "@Glossier", "@DollarShaveClub", "@Allbirds", "pizza"])
51 | 


--------------------------------------------------------------------------------
/3_redis/__pycache__/tweet_store.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/k2datascience/twitter_filter/1c6df786281b8254675a8ff7b3134dfbfdcd573d/3_redis/__pycache__/tweet_store.cpython-36.pyc


--------------------------------------------------------------------------------
/3_redis/dump.rdb:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/k2datascience/twitter_filter/1c6df786281b8254675a8ff7b3134dfbfdcd573d/3_redis/dump.rdb


--------------------------------------------------------------------------------
/3_redis/tweet_store.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import redis
 3 | 
 4 | class TweetStore:
 5 | 
 6 |     # Redis Configuration
 7 |     redis_host = "localhost"
 8 |     redis_port = 6379
 9 |     redis_password = ""
10 | 
11 |     # Tweet Configuration
12 |     redis_key = 'tweets'
13 |     num_tweets = 20
14 | 
15 |     def __init__(self):
16 |         self.db = r = redis.Redis(
17 |             host=self.redis_host,
18 |             port=self.redis_port,
19 |             password=self.redis_password
20 |         )
21 |         self.trim_count = 0
22 | 
23 |     def push(self, data):
24 |         self.db.lpush(self.redis_key, json.dumps(data))
25 |         self.trim_count += 1
26 | 
27 |         # Periodically trim the list so it doesn't grow too large.
28 |         if self.trim_count > 100:
29 |             self.db.ltrim(self.redis_key, 0, self.num_tweets)
30 |             self.trim_count = 0
31 | 
32 |     def tweets(self, limit=15):
33 |         tweets = []
34 | 
35 |         for item in self.db.lrange(self.redis_key, 0, limit-1):
36 |             tweet_obj = json.loads(item)
37 |             tweets.append(tweet_obj)
38 | 
39 |         return tweets
40 | 


--------------------------------------------------------------------------------
/3_redis/twitter_filter.py:
--------------------------------------------------------------------------------
 1 | import tweepy
 2 | import datetime
 3 | from textblob import TextBlob
 4 | from tweet_store import TweetStore
 5 | import json
 6 | 
 7 | file_path = '../config/api.json'
 8 | 
 9 | with open(file_path) as f:
10 |     twitter_api = json.loads(f.read())
11 | 
12 | consumer_key = twitter_api['consumer_key']
13 | consumer_secret = twitter_api['consumer_secret']
14 | access_token = twitter_api['access_token']
15 | access_token_secret = twitter_api['access_token_secret']
16 | 
17 | auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
18 | auth.set_access_token(access_token, access_token_secret)
19 | 
20 | api = tweepy.API(auth)
21 | store = TweetStore()
22 | 
23 | class StreamListener(tweepy.StreamListener):
24 | 
25 |     def on_status(self, status):
26 | 
27 |         if ('RT @' not in status.text):
28 |             blob = TextBlob(status.text)
29 |             sent = blob.sentiment
30 |             polarity = sent.polarity
31 |             subjectivity = sent.subjectivity
32 | 
33 |             tweet_item = {
34 |                 'id_str': status.id_str,
35 |                 'text': status.text,
36 |                 'polarity': polarity,
37 |                 'subjectivity': subjectivity,
38 |                 'username': status.user.screen_name,
39 |                 'name': status.user.name,
40 |                 'profile_image_url': status.user.profile_image_url,
41 |                 'received_at': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
42 |             }
43 | 
44 |             store.push(tweet_item)
45 |             print("Pushed to redis:", tweet_item)
46 | 
47 |     def on_error(self, status_code):
48 |         if status_code == 420:
49 |             return False
50 | 
51 | stream_listener = StreamListener()
52 | stream = tweepy.Stream(auth=api.auth, listener=stream_listener)
53 | stream.filter(track=["@WarbyParker", "@Bonobos", "@Casper", "@Glossier", "@DollarShaveClub", "@Allbirds", "pizza"])
54 | 


--------------------------------------------------------------------------------
/4_design_front_end/index.html:
--------------------------------------------------------------------------------
 1 | <!doctype html>
 2 | <html lang="en">
 3 |   <head>
 4 |     <!-- Required meta tags -->
 5 |     <meta charset="utf-8">
 6 |     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
 7 | 
 8 |     <!-- Bootstrap CSS -->
 9 |     <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
10 | 
11 |     <!-- Custom CSS -->
12 |     <link rel="stylesheet" href="styles.css">
13 | 
14 |     <title>DTC-Twitter Filter</title>
15 |   </head>
16 |   <body>
17 |     <div class="container">
18 | 
19 |       <div class="row mb-5">
20 |         <div class="col text-center">
21 |           <h1 class="mb-3">
22 |             DTC-Twitter Filter
23 |           </h1>
24 |           <h4 class="mb-2">
25 |             <small class="text-muted">
26 |               Monitoring Warby Parker, Bonobos, Casper, Glossier, Dollar Shave Club and Allbirds
27 |             </small>
28 |           </h4>
29 |         </div>
30 |       </div>
31 | 
32 |       <div class="row my-4">
33 |         <div class="col">
34 |           <div class="card positive">
35 |             <div class="card-body">
36 |               <p class="card-text">
37 |                 Just stopped by the new @WarbyParker store and got a fresh pair of spectacles!
38 |               </p>
39 |               <br />
40 |               <div class="card-text">
41 |                 <img alt="janefame15" height="48" src="https://i0.wp.com/2.bp.blogspot.com/-oBk39VtCsEY/WnOO5LVE82I/AAAAAAAADUM/bTQ96PzU9dwfc0GXkyUEz2_ehqS8C8C6ACLcBGAs/s640/0.png?zoom=2.625&resize=382%2C268&ssl=1" width="48"/>
42 |                  janefame15 - (<a href="http://www.twitter.com/janefame15" target="_blank">janefame15</a>)
43 |               </div>
44 |             </div>
45 |           </div>
46 |         </div>
47 |       </div>
48 | 
49 |       <div class="row my-4">
50 |         <div class="col">
51 |           <div class="card">
52 |             <div class="card-body">
53 |               <p class="card-text">
54 |                 Real Estate Announcement: WarbyParker leases storefront in DC Commons.
55 |               </p>
56 |               <br />
57 |               <div class="card-text">
58 |                 <img alt="janefame15" height="48" src="https://i0.wp.com/2.bp.blogspot.com/-oBk39VtCsEY/WnOO5LVE82I/AAAAAAAADUM/bTQ96PzU9dwfc0GXkyUEz2_ehqS8C8C6ACLcBGAs/s640/0.png?zoom=2.625&resize=382%2C268&ssl=1" width="48"/>
59 |                  janefame15 - (<a href="http://www.twitter.com/janefame15" target="_blank">janefame15</a>)
60 |               </div>
61 |             </div>
62 |           </div>
63 |         </div>
64 |       </div>
65 | 
66 |       <div class="row my-4">
67 |         <div class="col">
68 |           <div class="card negative">
69 |             <div class="card-body">
70 |               <p class="card-text">
71 |                 I fell down the stairs and broke my WarbyParker frames. NOOOOO!!!!
72 |               </p>
73 |               <br />
74 |               <div class="card-text">
75 |                 <img alt="janefame15" height="48" src="https://i0.wp.com/2.bp.blogspot.com/-oBk39VtCsEY/WnOO5LVE82I/AAAAAAAADUM/bTQ96PzU9dwfc0GXkyUEz2_ehqS8C8C6ACLcBGAs/s640/0.png?zoom=2.625&resize=382%2C268&ssl=1" width="48"/>
76 |                  janefame15 - (<a href="http://www.twitter.com/janefame15" target="_blank">janefame15</a>)
77 |               </div>
78 |             </div>
79 |           </div>
80 |         </div>
81 |       </div>
82 | 
83 |       <div class="text-center mb-4">
84 |         <a class="btn btn-primary" href="/" role="button">Refresh</a>
85 |       </div>
86 | 
87 | 
88 |     </div>
89 |   </body>
90 | </html>
91 | 


--------------------------------------------------------------------------------
/4_design_front_end/pen_paper.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/k2datascience/twitter_filter/1c6df786281b8254675a8ff7b3134dfbfdcd573d/4_design_front_end/pen_paper.jpg


--------------------------------------------------------------------------------
/4_design_front_end/styles.css:
--------------------------------------------------------------------------------
 1 | body {
 2 |   margin-top: 50px;
 3 | }
 4 | 
 5 | .lrg-heading {
 6 |   font-size: 48px;
 7 | }
 8 | 
 9 | .description {
10 |   font-size: 24px;
11 |   color: gray;
12 | }
13 | 
14 | .brand {
15 |   color: yellow;
16 | }
17 | 
18 | .positive {
19 |   border: 1px solid rgba(0,164,0,.8)
20 | }
21 | 
22 | .negative {
23 |   border: 1px solid rgba(164,0,0,.8)
24 | }
25 | 


--------------------------------------------------------------------------------
/5_python_class/tweet.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | 
 3 | class Tweet:
 4 | 
 5 |     def __init__(self, data):
 6 |         self.data = data
 7 | 
 8 |     def user_link(self):
 9 |         return "http://twitter.com/{}".format(self.data['username'])
10 | 
11 |     def filtered_text(self):
12 |         return self.filter_brands(self.filter_urls(self.data['text']))
13 | 
14 |     def filter_brands(self, text):
15 |         brands = ["@WarbyParker", "@Bonobos", "@Casper", "@Glossier", "@DollarShaveClub", "@Allbirds"]
16 | 
17 |         for brand in brands:
18 |             if (brand in text):
19 |                 text = text.replace(brand, "<mark>{}</mark>".format(brand))
20 |             else:
21 |                 continue
22 | 
23 |         return text
24 | 
25 |     def filter_urls(self, text):
26 |         return re.sub("(https?:\/\/\w+(\.\w+)+(\/[\w\+\-\,\%]+)*(\?[\w\[\]]+(=\w*)?(&\w+(=\w*)?)*)?(#\w+)?)", r'<a href="\1" target="_blank">\1</a>', text)
27 | 


--------------------------------------------------------------------------------
/5_python_class/tweet_store.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import redis
 3 | from tweet import Tweet
 4 | 
 5 | class TweetStore:
 6 | 
 7 |     # Redis Configuration
 8 |     redis_host = "localhost"
 9 |     redis_port = 6379
10 |     redis_password = ""
11 | 
12 |     # Tweet Configuration
13 |     redis_key = 'tweets'
14 |     num_tweets = 20
15 |     trim_threshold = 100
16 | 
17 |     def __init__(self):
18 |         self.db = r = redis.Redis(
19 |             host=self.redis_host,
20 |             port=self.redis_port,
21 |             password=self.redis_password
22 |         )
23 |         self.trim_count = 0
24 | 
25 |     def tweets(self, limit=15):
26 |         tweets = []
27 | 
28 |         for item in self.db.lrange(self.redis_key, 0, limit-1):
29 |             tweet_obj = json.loads(item)
30 |             tweets.append(Tweet(tweet_obj))
31 | 
32 |         return tweets
33 | 
34 |     def push(self, data):
35 |         self.db.lpush(self.redis_key, json.dumps(data))
36 |         self.trim_count += 1
37 | 
38 |         # Periodically trim the list so it doesn't grow too large.
39 |         if self.trim_count > 100:
40 |             self.db.ltrim(self.redis_key, 0, self.num_tweets)
41 |             self.trim_count = 0
42 | 


--------------------------------------------------------------------------------
/5_python_class/twitter_filter.py:
--------------------------------------------------------------------------------
 1 | import tweepy
 2 | import datetime
 3 | from textblob import TextBlob
 4 | from tweet_store import TweetStore
 5 | import json
 6 | 
 7 | file_path = '../config/api.json'
 8 | 
 9 | with open(file_path) as f:
10 |     twitter_api = json.loads(f.read())
11 | 
12 | consumer_key = twitter_api['consumer_key']
13 | consumer_secret = twitter_api['consumer_secret']
14 | access_token = twitter_api['access_token']
15 | access_token_secret = twitter_api['access_token_secret']
16 | 
17 | auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
18 | auth.set_access_token(access_token, access_token_secret)
19 | 
20 | api = tweepy.API(auth)
21 | store = TweetStore()
22 | 
23 | class StreamListener(tweepy.StreamListener):
24 | 
25 |     def on_status(self, status):
26 | 
27 |         if ('RT @' not in status.text):
28 |             blob = TextBlob(status.text)
29 |             sent = blob.sentiment
30 |             polarity = sent.polarity
31 |             subjectivity = sent.subjectivity
32 | 
33 |             tweet_item = {
34 |                 'id_str': status.id_str,
35 |                 'text': status.text,
36 |                 'polarity': polarity,
37 |                 'subjectivity': subjectivity,
38 |                 'username': status.user.screen_name,
39 |                 'name': status.user.name,
40 |                 'profile_image_url': status.user.profile_image_url,
41 |                 'received_at': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
42 |             }
43 | 
44 |             store.push(tweet_item)
45 |             print("Pushed to redis:", tweet_item)
46 | 
47 |     def on_error(self, status_code):
48 |         if status_code == 420:
49 |             return False
50 | 
51 | stream_listener = StreamListener()
52 | stream = tweepy.Stream(auth=api.auth, listener=stream_listener)
53 | stream.filter(track=["@WarbyParker", "@Bonobos", "@Casper", "@Glossier", "@DollarShaveClub", "@Allbirds"])
54 | 


--------------------------------------------------------------------------------
/6_flask/__pycache__/tweet.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/k2datascience/twitter_filter/1c6df786281b8254675a8ff7b3134dfbfdcd573d/6_flask/__pycache__/tweet.cpython-36.pyc


--------------------------------------------------------------------------------
/6_flask/__pycache__/tweet_store.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/k2datascience/twitter_filter/1c6df786281b8254675a8ff7b3134dfbfdcd573d/6_flask/__pycache__/tweet_store.cpython-36.pyc


--------------------------------------------------------------------------------
/6_flask/dump.rdb:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/k2datascience/twitter_filter/1c6df786281b8254675a8ff7b3134dfbfdcd573d/6_flask/dump.rdb


--------------------------------------------------------------------------------
/6_flask/static/styles.css:
--------------------------------------------------------------------------------
 1 | body {
 2 |   margin-top: 50px;
 3 | }
 4 | 
 5 | img {
 6 |   border-radius: 50%;
 7 | }
 8 | 
 9 | .lrg-heading {
10 |   font-size: 48px;
11 | }
12 | 
13 | .description {
14 |   font-size: 24px;
15 |   color: gray;
16 | }
17 | 
18 | .brand {
19 |   color: yellow;
20 | }
21 | 
22 | .positive {
23 |   border: 1px solid rgba(0,164,0,.8)
24 | }
25 | 
26 | .negative {
27 |   border: 1px solid rgba(164,0,0,.8)
28 | }
29 | 


--------------------------------------------------------------------------------
/6_flask/templates/index.html:
--------------------------------------------------------------------------------
 1 | <!doctype html>
 2 | <html lang="en">
 3 |   <head>
 4 |     <!-- Required meta tags -->
 5 |     <meta charset="utf-8">
 6 |     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
 7 | 
 8 |     <!-- Bootstrap CSS -->
 9 |     <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
10 | 
11 |     <!-- Custom CSS -->
12 |     <link rel="stylesheet" href="{{ url_for('static', filename='styles.css') }}">
13 | 
14 |     <title>DTC-Twitter Filter</title>
15 |   </head>
16 |   <body>
17 |     <div class="container">
18 | 
19 |       <div class="row mb-5">
20 |         <div class="col text-center">
21 |           <h1 class="mb-3">
22 |             DTC-Twitter Filter
23 |           </h1>
24 |           <h4 class="mb-2">
25 |             <small class="text-muted">
26 |               Monitoring Warby Parker, Bonobos, Casper, Glossier, Dollar Shave Club and Allbirds
27 |             </small>
28 |           </h4>
29 |         </div>
30 |       </div>
31 | 
32 |       {% for tweet in tweets %}
33 |         <div class="row my-4">
34 |           <div class="col">
35 |             <div class="card {{ 'positive' if tweet.data.polarity > 0 }} {{ 'negative' if tweet.data.polarity < 0 }}">
36 |               <div class="card-body">
37 |                 <p class="card-text">
38 |                   {{ tweet.filtered_text()|safe }}
39 |                 </p>
40 |                 <br />
41 |                 <div class="card-text">
42 |                   <img alt="{{ tweet.data.username }}" height="48" src="{{ tweet.data.profile_image_url }}" width="48" class="mr-3"/>
43 |                   {{ tweet.data.name }} (<a href="{{ tweet.user_link() }}" target="_blank">{{ tweet.data.username }}</a>)
44 |                 </div>
45 |               </div>
46 |             </div>
47 |           </div>
48 |         </div>
49 |       {% endfor %}
50 | 
51 |       <div class="text-center mb-4">
52 |         <a class="btn btn-primary" href="/" role="button">Refresh</a>
53 |       </div>
54 | 
55 | 
56 |     </div>
57 |   </body>
58 | </html>
59 | 


--------------------------------------------------------------------------------
/6_flask/twatcher.py:
--------------------------------------------------------------------------------
 1 | from flask import Flask, render_template
 2 | from tweet_store import TweetStore
 3 | 
 4 | app = Flask(__name__)
 5 | store = TweetStore()
 6 | 
 7 | @app.route('/')
 8 | def index():
 9 |     tweets = store.tweets()
10 |     return render_template('index.html', tweets=tweets)
11 | 
12 | if __name__ == '__main__':
13 |     app.run(debug=True)
14 | 


--------------------------------------------------------------------------------
/6_flask/tweet.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | 
 3 | class Tweet:
 4 | 
 5 |     def __init__(self, data):
 6 |         self.data = data
 7 | 
 8 |     def user_link(self):
 9 |         return "http://twitter.com/{}".format(self.data['username'])
10 | 
11 |     def filtered_text(self):
12 |         return self.filter_brands(self.filter_urls(self.data['text']))
13 | 
14 |     def filter_brands(self, text):
15 |         brands = ["@WarbyParker", "@Bonobos", "@Casper", "@Glossier", "@DollarShaveClub", "@Allbirds"]
16 | 
17 |         for brand in brands:
18 |             if (brand in text):
19 |                 text = text.replace(brand, "<mark>{}</mark>".format(brand))
20 |             else:
21 |                 continue
22 | 
23 |         return text
24 | 
25 |     def filter_urls(self, text):
26 |         return re.sub("(https?:\/\/\w+(\.\w+)+(\/[\w\+\-\,\%]+)*(\?[\w\[\]]+(=\w*)?(&\w+(=\w*)?)*)?(#\w+)?)", r'<a href="\1" target="_blank">\1</a>', text)
27 | 


--------------------------------------------------------------------------------
/6_flask/tweet_store.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import redis
 3 | from tweet import Tweet
 4 | 
 5 | class TweetStore:
 6 | 
 7 |     # Redis Configuration
 8 |     redis_host = "localhost"
 9 |     redis_port = 6379
10 |     redis_password = ""
11 | 
12 |     # Tweet Configuration
13 |     redis_key = 'tweets'
14 |     num_tweets = 20
15 |     trim_threshold = 100
16 | 
17 |     def __init__(self):
18 |         self.db = r = redis.Redis(
19 |             host=self.redis_host,
20 |             port=self.redis_port,
21 |             password=self.redis_password
22 |         )
23 |         self.trim_count = 0
24 | 
25 |     def tweets(self, limit=15):
26 |         tweets = []
27 | 
28 |         for item in self.db.lrange(self.redis_key, 0, limit-1):
29 |             tweet_obj = json.loads(item)
30 |             tweets.append(Tweet(tweet_obj))
31 | 
32 |         return tweets
33 | 
34 |     def push(self, data):
35 |         self.db.lpush(self.redis_key, json.dumps(data))
36 |         self.trim_count += 1
37 | 
38 |         # Periodically trim the list so it doesn't grow too large.
39 |         if self.trim_count > 100:
40 |             self.db.ltrim(self.redis_key, 0, self.num_tweets)
41 |             self.trim_count = 0
42 | 


--------------------------------------------------------------------------------
/6_flask/twitter_filter.py:
--------------------------------------------------------------------------------
 1 | import tweepy
 2 | import datetime
 3 | from textblob import TextBlob
 4 | from tweet_store import TweetStore
 5 | import json
 6 | 
 7 | file_path = '../config/api.json'
 8 | 
 9 | with open(file_path) as f:
10 |     twitter_api = json.loads(f.read())
11 | 
12 | consumer_key = twitter_api['consumer_key']
13 | consumer_secret = twitter_api['consumer_secret']
14 | access_token = twitter_api['access_token']
15 | access_token_secret = twitter_api['access_token_secret']
16 | 
17 | auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
18 | auth.set_access_token(access_token, access_token_secret)
19 | 
20 | api = tweepy.API(auth)
21 | store = TweetStore()
22 | 
23 | class StreamListener(tweepy.StreamListener):
24 | 
25 |     def on_status(self, status):
26 | 
27 |         if ('RT @' not in status.text):
28 |             blob = TextBlob(status.text)
29 |             sent = blob.sentiment
30 |             polarity = sent.polarity
31 |             subjectivity = sent.subjectivity
32 | 
33 |             tweet_item = {
34 |                 'id_str': status.id_str,
35 |                 'text': status.text,
36 |                 'polarity': polarity,
37 |                 'subjectivity': subjectivity,
38 |                 'username': status.user.screen_name,
39 |                 'name': status.user.name,
40 |                 'profile_image_url': status.user.profile_image_url,
41 |                 'received_at': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
42 |             }
43 | 
44 |             store.push(tweet_item)
45 |             print("Pushed to redis:", tweet_item)
46 | 
47 |     def on_error(self, status_code):
48 |         if status_code == 420:
49 |             return False
50 | 
51 | stream_listener = StreamListener()
52 | stream = tweepy.Stream(auth=api.auth, listener=stream_listener)
53 | stream.filter(track=["@WarbyParker", "@Bonobos", "@Casper", "@Glossier", "@DollarShaveClub", "@Allbirds"])
54 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Streaming Twitter Filter
 2 | 
 3 | Companion code for the tutorial series about building a streaming Twitter filter using Python and Redis.
 4 | 
 5 | - [Udemy Course (free - coming soon)]()
 6 | - [YouTube Playlist (coming soon)]()
 7 | - [Medium Article (coming soon)]()
 8 | 
 9 | ## File Organization
10 | 
11 | - **1_twitter_stream**: Contains the Python script to download, filter and extract data from the Twitter Streaming API.
12 | - **2_sentiment**: Contains the updated Python script to calculate and store sentiment scores for each tweet.
13 | - **3_redis**: Contains the additional Python class to interface with a Redis server.
14 | - **4_design_front_end**: Contains the pen and paper mockup as well as the static front-end code.
15 | - **5_python_class**: Contains the Python class to pre-process each tweet prior to hitting the Flask app.
16 | - **6_flask**: Contains the Flask app to read the last 15 tweets from Redis and display them.
17 | 


--------------------------------------------------------------------------------