├── .gitignore ├── README.md ├── requirements.txt ├── save_200_tweets_to_json.py ├── save_ff_timelines_to_json.py ├── save_recent_tweets_to_json.py ├── save_timelines_to_json.py ├── test_tweet_filter.py ├── testdata ├── bad_json_tweets_x3 ├── retweet_x1 └── shears.txt ├── tweet_filter.py ├── tweet_filter_timeline_downloadable.py ├── twitter_crawler.py └── twitter_oauth_settings.sample.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *~ 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | trawler 2 | ======= 3 | 4 | Most of the interesting functionality is in the class 5 | RateLimitedTwitterEndpoint. The class is a wrapper around the (Twython 6 | wrapper around the) Twitter API that handles all of the details of 7 | rate limiting. It also robustly handles errors that occur when the 8 | Twitter servers are temporarily misbehaving. 9 | 10 | Once you've created an instance of RateLimitedTwitterEndpoint, call: 11 | 12 | ````python 13 | endpoint.get_data(twitter_api_parameters) 14 | ```` 15 | 16 | and get_data() will return the data from the Twitter API as soon as 17 | possible without violating the Twitter rate limits (and thus the 18 | TOS). This means that get_data() may block for up to 15 minutes. All 19 | of the classes used by RateLimitedTwitterEndpoint are thread safe. 20 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | chromium-compact-language-detector 2 | twython>=3.1.2 3 | -------------------------------------------------------------------------------- /save_200_tweets_to_json.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | This script downloads all available Tweets for a given list of 5 | usernames. 6 | 7 | The script takes as input a text file which lists one Twitter username 8 | per line of the file. The script creates a [username].tweets file for 9 | each username specified in the directory. 10 | 11 | Your Twitter OAuth credentials should be stored in the file 12 | twitter_oauth_settings.py. 13 | """ 14 | 15 | # Standard Library modules 16 | import argparse 17 | import codecs 18 | import os 19 | import sys 20 | 21 | # Third party modules 22 | from twython import Twython, TwythonError 23 | 24 | # Local modules 25 | from twitter_crawler import (CrawlTwitterTimelines, RateLimitedTwitterEndpoint, 26 | get_console_info_logger, get_screen_names_from_file, save_tweets_to_json_file) 27 | try: 28 | from twitter_oauth_settings import access_token, access_token_secret, consumer_key, consumer_secret 29 | except ImportError: 30 | print "You must create a 'twitter_oauth_settings.py' file with your Twitter API credentials." 31 | print "Please copy over the sample configuration file:" 32 | print " cp twitter_oauth_settings.sample.py twitter_oauth_settings.py" 33 | print "and add your API credentials to the file." 34 | sys.exit() 35 | 36 | 37 | def main(): 38 | # Make stdout output UTF-8, preventing "'ascii' codec can't encode" errors 39 | sys.stdout = codecs.getwriter('utf8')(sys.stdout) 40 | 41 | parser = argparse.ArgumentParser(description="") 42 | parser.add_argument('screen_name_file') 43 | args = parser.parse_args() 44 | 45 | logger = get_console_info_logger() 46 | 47 | ACCESS_TOKEN = Twython(consumer_key, consumer_secret, oauth_version=2).obtain_access_token() 48 | twython = Twython(consumer_key, access_token=ACCESS_TOKEN) 49 | 50 | crawler = RateLimitedTwitterEndpoint(twython, "statuses/user_timeline", logger) 51 | 52 | screen_names = get_screen_names_from_file(args.screen_name_file) 53 | 54 | for screen_name in screen_names: 55 | tweet_filename = "%s.tweets" % screen_name 56 | if os.path.exists(tweet_filename): 57 | logger.info("File '%s' already exists - will not attempt to download Tweets for '%s'" % (tweet_filename, screen_name)) 58 | else: 59 | try: 60 | logger.info("Retrieving Tweets for user '%s'" % screen_name) 61 | tweets = crawler.get_data(screen_name=screen_name, count=200) 62 | except TwythonError as e: 63 | print "TwythonError: %s" % e 64 | if e.error_code == 404: 65 | logger.warn("HTTP 404 error - Most likely, Twitter user '%s' no longer exists" % screen_name) 66 | elif e.error_code == 401: 67 | logger.warn("HTTP 401 error - Most likely, Twitter user '%s' no longer publicly accessible" % screen_name) 68 | else: 69 | # Unhandled exception 70 | raise e 71 | else: 72 | save_tweets_to_json_file(tweets, "%s.tweets" % screen_name) 73 | 74 | 75 | if __name__ == "__main__": 76 | main() 77 | -------------------------------------------------------------------------------- /save_ff_timelines_to_json.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | This script downloads all available Tweets from the one-hop network 5 | for a given list of usernames. For a given Twitter user, only other 6 | users who are both Friends and Followers (e.g. have a reciprocal 7 | relationship) are considered to be part of the one-hop network. 8 | 9 | For each username in the provided list, the script creates a 10 | [username].ff file that contains the friends-and-followers screen 11 | names (one screen name per line) for the specified Twitter user. 12 | 13 | The script creates a [username].tweets file containing the Tweets (one 14 | JSON object per line) for each user who is a friend-and-follower of 15 | one of the provided users.. 16 | 17 | Your Twitter OAuth credentials should be stored in the file 18 | twitter_oauth_settings.py. 19 | """ 20 | 21 | # Standard Library modules 22 | import argparse 23 | import codecs 24 | import os 25 | import sys 26 | 27 | # Third party modules 28 | from twython import Twython, TwythonError 29 | 30 | # Local modules 31 | from twitter_crawler import (CrawlTwitterTimelines, FindFriendFollowers, RateLimitedTwitterEndpoint, 32 | get_console_info_logger, get_screen_names_from_file, 33 | save_screen_names_to_file, save_tweets_to_json_file) 34 | try: 35 | from twitter_oauth_settings import access_token, access_token_secret, consumer_key, consumer_secret 36 | except ImportError: 37 | print "You must create a 'twitter_oauth_settings.py' file with your Twitter API credentials." 38 | print "Please copy over the sample configuration file:" 39 | print " cp twitter_oauth_settings.sample.py twitter_oauth_settings.py" 40 | print "and add your API credentials to the file." 41 | sys.exit() 42 | 43 | 44 | def main(): 45 | # Make stdout output UTF-8, preventing "'ascii' codec can't encode" errors 46 | sys.stdout = codecs.getwriter('utf8')(sys.stdout) 47 | 48 | parser = argparse.ArgumentParser(description="") 49 | parser.add_argument('screen_name_file') 50 | args = parser.parse_args() 51 | 52 | logger = get_console_info_logger() 53 | 54 | ACCESS_TOKEN = Twython(consumer_key, consumer_secret, oauth_version=2).obtain_access_token() 55 | twython = Twython(consumer_key, access_token=ACCESS_TOKEN) 56 | 57 | timeline_crawler = CrawlTwitterTimelines(twython, logger) 58 | ff_finder = FindFriendFollowers(twython, logger) 59 | 60 | screen_names = get_screen_names_from_file(args.screen_name_file) 61 | 62 | for screen_name in screen_names: 63 | ff_screen_names = ff_finder.get_ff_screen_names_for_screen_name(screen_name) 64 | save_screen_names_to_file(ff_screen_names, "%s.ff" % screen_name, logger) 65 | 66 | for ff_screen_name in ff_screen_names: 67 | tweet_filename = "%s.tweets" % ff_screen_name 68 | if os.path.exists(tweet_filename): 69 | logger.info("File '%s' already exists - will not attempt to download Tweets for '%s'" % (tweet_filename, ff_screen_name)) 70 | else: 71 | try: 72 | tweets = timeline_crawler.get_all_timeline_tweets_for_screen_name(ff_screen_name) 73 | except TwythonError as e: 74 | print "TwythonError: %s" % e 75 | if e.error_code == 404: 76 | logger.warn("HTTP 404 error - Most likely, Twitter user '%s' no longer exists" % ff_screen_name) 77 | elif e.error_code == 401: 78 | logger.warn("HTTP 401 error - Most likely, Twitter user '%s' no longer publicly accessible" % ff_screen_name) 79 | else: 80 | # Unhandled exception 81 | raise e 82 | else: 83 | save_tweets_to_json_file(tweets, tweet_filename) 84 | 85 | 86 | 87 | if __name__ == "__main__": 88 | main() 89 | -------------------------------------------------------------------------------- /save_recent_tweets_to_json.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | This script downloads all "new" Tweets for a list of usernames that 5 | have been posted since the last time the users' feeds were crawled. 6 | 7 | The script takes as input: 8 | - a text file which lists one Twitter username per line of the file 9 | - the path to the existing [username].tweets files 10 | - the path where the new [username].tweets files will be stored 11 | 12 | For each username, the script opens the '[old_path]/[username].tweets' 13 | file, determines the most recently downloaded Tweet, and then creates 14 | a new '[new_path]/[username].tweets' file containing any new Tweets 15 | from the user. 16 | 17 | Your Twitter OAuth credentials should be stored in the file 18 | twitter_oauth_settings.py. 19 | """ 20 | 21 | # Standard Library modules 22 | import argparse 23 | import codecs 24 | import json 25 | import os 26 | import sys 27 | 28 | # Third party modules 29 | from twython import Twython, TwythonError 30 | 31 | # Local modules 32 | from twitter_crawler import (CrawlTwitterTimelines, RateLimitedTwitterEndpoint, 33 | get_console_info_logger, get_screen_names_from_file, save_tweets_to_json_file) 34 | try: 35 | from twitter_oauth_settings import access_token, access_token_secret, consumer_key, consumer_secret 36 | except ImportError: 37 | print "You must create a 'twitter_oauth_settings.py' file with your Twitter API credentials." 38 | print "Please copy over the sample configuration file:" 39 | print " cp twitter_oauth_settings.sample.py twitter_oauth_settings.py" 40 | print "and add your API credentials to the file." 41 | sys.exit() 42 | 43 | 44 | def main(): 45 | # Make stdout output UTF-8, preventing "'ascii' codec can't encode" errors 46 | sys.stdout = codecs.getwriter('utf8')(sys.stdout) 47 | 48 | parser = argparse.ArgumentParser(description="") 49 | parser.add_argument('screen_name_file') 50 | parser.add_argument('old_tweet_path') 51 | parser.add_argument('new_tweet_path') 52 | args = parser.parse_args() 53 | 54 | logger = get_console_info_logger() 55 | 56 | ACCESS_TOKEN = Twython(consumer_key, consumer_secret, oauth_version=2).obtain_access_token() 57 | twython = Twython(consumer_key, access_token=ACCESS_TOKEN) 58 | 59 | crawler = CrawlTwitterTimelines(twython, logger) 60 | 61 | screen_names = get_screen_names_from_file(args.screen_name_file) 62 | 63 | for screen_name in screen_names: 64 | old_tweet_filename = os.path.join(args.old_tweet_path, "%s.tweets" % screen_name) 65 | new_tweet_filename = os.path.join(args.new_tweet_path, "%s.tweets" % screen_name) 66 | 67 | if not os.path.exists(old_tweet_filename): 68 | logger.error("Older Tweet file '%s' does not exist - will not attempt to download Tweets for '%s'" % (old_tweet_filename, screen_name)) 69 | continue 70 | if os.path.exists(new_tweet_filename): 71 | logger.info("File '%s' already exists - will not attempt to download Tweets for '%s'" % (new_tweet_filename, screen_name)) 72 | continue 73 | 74 | most_recent_tweet_id = get_most_recent_tweet_id_from_json_tweet_file(old_tweet_filename) 75 | 76 | try: 77 | tweets = crawler.get_all_timeline_tweets_for_screen_name_since(screen_name, most_recent_tweet_id) 78 | except TwythonError as e: 79 | print "TwythonError: %s" % e 80 | if e.error_code == 404: 81 | logger.warn("HTTP 404 error - Most likely, Twitter user '%s' no longer exists" % screen_name) 82 | elif e.error_code == 401: 83 | logger.warn("HTTP 401 error - Most likely, Twitter user '%s' no longer publicly accessible" % screen_name) 84 | else: 85 | # Unhandled exception 86 | raise e 87 | else: 88 | save_tweets_to_json_file(tweets, new_tweet_filename) 89 | 90 | 91 | def get_most_recent_tweet_id_from_json_tweet_file(json_tweet_filename): 92 | """ 93 | Assumes that Tweets in file are ordered newest to oldest 94 | """ 95 | json_tweet_file = codecs.open(json_tweet_filename, "r", encoding="utf-8") 96 | first_tweet_json = json_tweet_file.readline() 97 | first_tweet = json.loads(first_tweet_json) 98 | most_recent_tweet_id = first_tweet['id'] 99 | json_tweet_file.close() 100 | return most_recent_tweet_id 101 | 102 | 103 | if __name__ == "__main__": 104 | main() 105 | -------------------------------------------------------------------------------- /save_timelines_to_json.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | This script downloads all available Tweets for a given list of 5 | usernames. 6 | 7 | The script takes as input a text file which lists one Twitter username 8 | per line of the file. The script creates a [username].tweets file for 9 | each username specified in the directory. 10 | 11 | Your Twitter OAuth credentials should be stored in the file 12 | twitter_oauth_settings.py. 13 | """ 14 | 15 | # Standard Library modules 16 | import argparse 17 | import codecs 18 | import os 19 | import sys 20 | 21 | # Third party modules 22 | from twython import Twython, TwythonError 23 | 24 | # Local modules 25 | from twitter_crawler import (CrawlTwitterTimelines, RateLimitedTwitterEndpoint, 26 | get_console_info_logger, get_screen_names_from_file, save_tweets_to_json_file) 27 | try: 28 | from twitter_oauth_settings import access_token, access_token_secret, consumer_key, consumer_secret 29 | except ImportError: 30 | print "You must create a 'twitter_oauth_settings.py' file with your Twitter API credentials." 31 | print "Please copy over the sample configuration file:" 32 | print " cp twitter_oauth_settings.sample.py twitter_oauth_settings.py" 33 | print "and add your API credentials to the file." 34 | sys.exit() 35 | 36 | 37 | def main(): 38 | # Make stdout output UTF-8, preventing "'ascii' codec can't encode" errors 39 | sys.stdout = codecs.getwriter('utf8')(sys.stdout) 40 | 41 | parser = argparse.ArgumentParser(description="") 42 | parser.add_argument('screen_name_file') 43 | args = parser.parse_args() 44 | 45 | logger = get_console_info_logger() 46 | 47 | ACCESS_TOKEN = Twython(consumer_key, consumer_secret, oauth_version=2).obtain_access_token() 48 | twython = Twython(consumer_key, access_token=ACCESS_TOKEN) 49 | 50 | crawler = CrawlTwitterTimelines(twython, logger) 51 | 52 | screen_names = get_screen_names_from_file(args.screen_name_file) 53 | 54 | for screen_name in screen_names: 55 | tweet_filename = "%s.tweets" % screen_name 56 | if os.path.exists(tweet_filename): 57 | logger.info("File '%s' already exists - will not attempt to download Tweets for '%s'" % (tweet_filename, screen_name)) 58 | else: 59 | try: 60 | tweets = crawler.get_all_timeline_tweets_for_screen_name(screen_name) 61 | except TwythonError as e: 62 | print "TwythonError: %s" % e 63 | if e.error_code == 404: 64 | logger.warn("HTTP 404 error - Most likely, Twitter user '%s' no longer exists" % screen_name) 65 | elif e.error_code == 401: 66 | logger.warn("HTTP 401 error - Most likely, Twitter user '%s' no longer publicly accessible" % screen_name) 67 | else: 68 | # Unhandled exception 69 | raise e 70 | else: 71 | save_tweets_to_json_file(tweets, "%s.tweets" % screen_name) 72 | 73 | 74 | if __name__ == "__main__": 75 | main() 76 | -------------------------------------------------------------------------------- /test_tweet_filter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | """ 5 | 6 | # Standard Library modules 7 | import unittest 8 | 9 | # Local modules 10 | from tweet_filter import * 11 | 12 | 13 | class TestFilterValidJSON(unittest.TestCase): 14 | def test_check_required_fields(self): 15 | json_tweet = '{"id": 1, "id_str": "1", "text":"foo", "user": {"screen_name": "charman"}}' 16 | json_tweet_no_screen_name = '{"id": 1, "id_str": "1", "text":"foo"}' 17 | tweet_json_filter = TweetFilterValidJSON() 18 | 19 | self.assertTrue(tweet_json_filter.filter(json_tweet)) 20 | self.assertFalse(tweet_json_filter.filter(json_tweet_no_screen_name)) 21 | 22 | 23 | class TestFilterNoURLs(unittest.TestCase): 24 | def test_url_filtering(self): 25 | json_tweet_http = '{"id": 1, "id_str": "1", "text":"http://twitter.com"}' 26 | json_tweet_https = '{"id": 2, "id_str": "2", "text":"https://twitter.com"}' 27 | json_tweet_clean = '{"id": 3, "id_str": "3", "text":"no urls here"}' 28 | tweet_no_url_filter = TweetFilterNoURLs() 29 | 30 | self.assertFalse(tweet_no_url_filter.filter(json_tweet_http)) 31 | self.assertFalse(tweet_no_url_filter.filter(json_tweet_https)) 32 | self.assertTrue(tweet_no_url_filter.filter(json_tweet_clean)) 33 | 34 | 35 | class TestFilterOneTweetPerScreenName(unittest.TestCase): 36 | def test_screen_name_filtering(self): 37 | json_tweet_1 = '{"id": 1, "id_str": "1", "user": {"screen_name":"charman"}}' 38 | json_tweet_2 = '{"id": 2, "id_str": "2", "user": {"screen_name":"PHonyDoc"}}' 39 | json_tweet_3 = '{"id": 3, "id_str": "3", "user": {"screen_name":"charman"}}' 40 | json_tweet_4 = '{"id": 4, "id_str": "4", "user": {"screen_name":"PHonyDoc"}}' 41 | screen_name_filter = TweetFilterOneTweetPerScreenName(TweetFilter) 42 | 43 | self.assertTrue(screen_name_filter.filter(json_tweet_1)) 44 | self.assertTrue(screen_name_filter.filter(json_tweet_2)) 45 | self.assertFalse(screen_name_filter.filter(json_tweet_3)) 46 | self.assertFalse(screen_name_filter.filter(json_tweet_4)) 47 | 48 | 49 | class TestFilterReliablyEnglish(unittest.TestCase): 50 | def test_english_filtering(self): 51 | spanish_tweet = '{"id": 1, "id_str": "1", "text":"Muchas de las victimas fueron mostrados con vendajes aplicados a toda prisa"}' 52 | english_tweet = '{"id": 2, "id_str": "2", "text":"The quick brown fox jumped over the lazy sleeping dog"}' 53 | english_filter = TweetFilterReliablyEnglish() 54 | 55 | self.assertFalse(english_filter.filter(spanish_tweet)) 56 | self.assertTrue(english_filter.filter(english_tweet)) 57 | 58 | 59 | class TestFilterTweetIDInSet(unittest.TestCase): 60 | def test_add_tweet_functions(self): 61 | json_tweet_1 = '{"id": 1, "id_str": "1"}' 62 | json_tweet_2 = '{"id": 2, "id_str": "2"}' 63 | json_tweet_3 = '{"id": 3, "id_str": "3"}' 64 | json_tweet_4 = '{"id": 4, "id_str": "4"}' 65 | json_tweet_5 = '{"id": 5, "id_str": "5"}' 66 | tweet_id_filter = TweetFilterTweetIDInSet() 67 | 68 | self.assertFalse(tweet_id_filter.filter(json_tweet_1)) 69 | tweet_id_filter.add_tweet(json_tweet_1) 70 | self.assertTrue(tweet_id_filter.filter(json_tweet_1)) 71 | 72 | tweet_id_filter.add_tweets([json_tweet_2, json_tweet_3]) 73 | self.assertTrue(tweet_id_filter.filter(json_tweet_2)) 74 | self.assertTrue(tweet_id_filter.filter(json_tweet_3)) 75 | 76 | self.assertFalse(tweet_id_filter.filter(json_tweet_4)) 77 | tweet_id_filter.add_tweet_id(4) 78 | self.assertTrue(tweet_id_filter.filter(json_tweet_4)) 79 | 80 | tweet_id_filter.add_tweet_ids([4,5]) 81 | self.assertTrue(tweet_id_filter.filter(json_tweet_5)) 82 | 83 | def test_tweet_id_as_ints(self): 84 | json_tweet_1 = '{"id": 1, "id_str": "1"}' 85 | tweet_id_filter = TweetFilterTweetIDInSet() 86 | self.assertFalse(tweet_id_filter.filter(json_tweet_1)) 87 | tweet_id_filter.add_tweet_id(1) 88 | self.assertTrue(tweet_id_filter.filter(json_tweet_1)) 89 | 90 | def test_tweet_id_as_unicode(self): 91 | json_tweet_1 = '{"id": 1, "id_str": "1"}' 92 | tweet_id_filter = TweetFilterTweetIDInSet() 93 | self.assertFalse(tweet_id_filter.filter(json_tweet_1)) 94 | tweet_id_filter.add_tweet_id(u'1') 95 | self.assertTrue(tweet_id_filter.filter(json_tweet_1)) 96 | 97 | 98 | 99 | class TestFilterTweetIDNotInSet(unittest.TestCase): 100 | def test_add_tweet_functions(self): 101 | json_tweet_1 = '{"id": 1, "id_str": "1"}' 102 | json_tweet_2 = '{"id": 2, "id_str": "2"}' 103 | json_tweet_3 = '{"id": 3, "id_str": "3"}' 104 | json_tweet_4 = '{"id": 4, "id_str": "4"}' 105 | json_tweet_5 = '{"id": 5, "id_str": "5"}' 106 | tweet_id_filter = TweetFilterTweetIDNotInSet() 107 | 108 | self.assertTrue(tweet_id_filter.filter(json_tweet_1)) 109 | tweet_id_filter.add_tweet(json_tweet_1) 110 | self.assertFalse(tweet_id_filter.filter(json_tweet_1)) 111 | 112 | tweet_id_filter.add_tweets([json_tweet_2, json_tweet_3]) 113 | self.assertFalse(tweet_id_filter.filter(json_tweet_2)) 114 | self.assertFalse(tweet_id_filter.filter(json_tweet_3)) 115 | 116 | self.assertTrue(tweet_id_filter.filter(json_tweet_4)) 117 | tweet_id_filter.add_tweet_id(4) 118 | self.assertFalse(tweet_id_filter.filter(json_tweet_4)) 119 | 120 | tweet_id_filter.add_tweet_ids([4,5]) 121 | self.assertFalse(tweet_id_filter.filter(json_tweet_5)) 122 | 123 | def test_tweet_id_as_ints(self): 124 | json_tweet_1 = '{"id": 1, "id_str": "1"}' 125 | tweet_id_filter = TweetFilterTweetIDNotInSet() 126 | self.assertTrue(tweet_id_filter.filter(json_tweet_1)) 127 | tweet_id_filter.add_tweet_id(1) 128 | self.assertFalse(tweet_id_filter.filter(json_tweet_1)) 129 | 130 | def test_tweet_id_as_unicode(self): 131 | json_tweet_1 = '{"id": 1, "id_str": "1"}' 132 | tweet_id_filter = TweetFilterTweetIDNotInSet() 133 | self.assertTrue(tweet_id_filter.filter(json_tweet_1)) 134 | tweet_id_filter.add_tweet_id(u'1') 135 | self.assertFalse(tweet_id_filter.filter(json_tweet_1)) 136 | 137 | 138 | 139 | class TestFilteredTweetReader(unittest.TestCase): 140 | def test_add_filter_when_reader_crated(self): 141 | filtered_reader = FilteredTweetReader([TweetFilterNotARetweet()]) 142 | filtered_reader.open("testdata/shears.txt") 143 | self.assertEqual(total_tweets_passed_through_filters(filtered_reader), 30) 144 | filtered_reader.close() 145 | 146 | def test_add_multiple_filters(self): 147 | filtered_reader = FilteredTweetReader() 148 | filtered_reader.add_filter(TweetFilterNotARetweet()) 149 | filtered_reader.add_filter(TweetFilterFieldMatchesRegEx('text', r'\bmy %s(s|es)?\b' % 'shears')) 150 | filtered_reader.open("testdata/shears.txt") 151 | self.assertEqual(total_tweets_passed_through_filters(filtered_reader), 19) 152 | filtered_reader.close() 153 | 154 | def test_filter_field_matches_regex(self): 155 | filtered_reader = FilteredTweetReader() 156 | regex_filter = TweetFilterFieldMatchesRegEx('text', r'\bmy %s(s|es)?\b' % 'shears') 157 | filtered_reader.add_filter(regex_filter) 158 | filtered_reader.open("testdata/shears.txt") 159 | self.assertEqual(total_tweets_passed_through_filters(filtered_reader), 20) 160 | filtered_reader.close() 161 | 162 | def test_filter_not_a_retweet(self): 163 | filtered_reader = FilteredTweetReader() 164 | filtered_reader.add_filter(TweetFilterNotARetweet()) 165 | 166 | filtered_reader.open("testdata/retweet_x1") 167 | self.assertEqual(total_tweets_passed_through_filters(filtered_reader), 0) 168 | filtered_reader.close() 169 | 170 | filtered_reader.open("testdata/shears.txt") 171 | self.assertEqual(total_tweets_passed_through_filters(filtered_reader), 30) 172 | filtered_reader.close() 173 | 174 | def test_filter_valid_json(self): 175 | # FilteredTweetReader() uses the valid JSON filter by default 176 | filtered_reader = FilteredTweetReader() 177 | filtered_reader.open("testdata/bad_json_tweets_x3") 178 | self.assertEqual(total_tweets_passed_through_filters(filtered_reader), 0) 179 | filtered_reader.close() 180 | 181 | def test_reader_with_no_filters(self): 182 | filtered_reader = FilteredTweetReader() 183 | filtered_reader.open("testdata/shears.txt") 184 | self.assertEqual(total_tweets_passed_through_filters(filtered_reader), 32) 185 | filtered_reader.close() 186 | 187 | def test_filter_raises_exception(self): 188 | filtered_reader = FilteredTweetReader() 189 | filtered_reader.add_filter(TweetFilterAlwaysRaiseException()) 190 | 191 | filtered_reader.open("testdata/shears.txt") 192 | self.assertRaises(Exception, total_tweets_passed_through_filters, filtered_reader) 193 | filtered_reader.close() 194 | 195 | def test_filters_short_circuit_after_first_rejection(self): 196 | filtered_reader = FilteredTweetReader() 197 | # Filteres are applied in order, so TweetFilterAlwaysRaiseException should never be called 198 | filtered_reader.add_filter(TweetFilterAlwaysReject()) 199 | filtered_reader.add_filter(TweetFilterAlwaysRaiseException()) 200 | 201 | filtered_reader.open("testdata/shears.txt") 202 | self.assertEqual(total_tweets_passed_through_filters(filtered_reader), 0) 203 | filtered_reader.close() 204 | 205 | 206 | def total_tweets_passed_through_filters(filtered_reader): 207 | tweets = [] 208 | for tweet in filtered_reader: 209 | tweets.append(tweet) 210 | return len(tweets) 211 | 212 | 213 | class TweetFilterAlwaysRaiseException(TweetFilter): 214 | def filter(self, json_tweet_string): 215 | raise Exception 216 | 217 | 218 | class TweetFilterAlwaysReject(TweetFilter): 219 | def filter(self, json_tweet_string): 220 | return False 221 | 222 | 223 | 224 | if __name__ == '__main__': 225 | unittest.main(buffer=True) 226 | -------------------------------------------------------------------------------- /testdata/bad_json_tweets_x3: -------------------------------------------------------------------------------- 1 | {"created_at":"Wed Oct 02 20:04:41 +0000 2013","id":385495574004641793,"id_str":"385495574004641793","text":"*angkat tangan* RT @DuniaKeras: Adakah yang cantik masih bangun? http:\/\/t.co\/iAEZr5hJGX","source":"\u003ca href=\"http:\/\/ubersocial.com\" rel=\"nofollow\"\u003eUberSocial for BlackBerry\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1274178954,"id_str":"1274178954","name":"Schweinsteiger090493","screen_name":"PutriayuHanifah","location":"Betawi","url":null,"description":"College in @kstaracademy university with my teacher is @setevy_ika and @januar141 ^___^","protected":false,"followers_count":97,"friends_count":147,"listed_count":0,"created_at":"Sun Mar 17 06:17:39 +0000 2013","favourites_count":34,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":3906,"lang":"id","contributors_enabled":false,"is_translator":false,"profile_background_color":"642D8B","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/834961600\/e087a9675d4b1c6cccf7b91848c1e944.jpeg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/834961600\/e087a9675d4b1c6cccf7b91848c1e944.jpeg","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3635211454\/1ea77460dd06ad9d1c8812b90996f1db_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3635211454\/1ea77460dd06ad9d1c8812b90996f1db_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1274178954\/1365196943","profile_link_color":"8C00FF","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"7AC3EE","profile_text_color":"3D1957","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"DuniaKeras","name":"Dunia Keras","id":354168059,"id_str":"354168059","indices":[19,30]}],"media":[{"id":385495573803323392,"id_str":"385495573803323392","indices":[65,87],"media_url":"http:\/\/pbs.twimg.com\/media\/BVmOLpMI{"created_at":"Sat Feb 02 16:47:57 +0000 2013","id":297748200964640768,"id_str":"297748200964640768","text":"Insomnia back to me (\u02d8\u0329\u0329\u0329\u2323\u02d8\u0329\u0329\u0329 )","source":"\u003ca href=\"http:\/\/blackberry.com\/twitter\" rel=\"nofollow\"\u003eTwitter for BlackBerry\u00ae\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":625624674,"id_str":"625624674","name":"Rizka dianasari","screen_name":"riskadiana25","location":"Yogyakarta, Indonesia","url":null,"description":"GOD is everything | family is everything | friendship is everything | economy university islamic of indonesia`12 | may 25th","protected":false,"followers_count":392,"friends_count":423,"listed_count":0,"created_at":"Tue Jul 03 13:11:19 +0000 2012","favourites_count":171,"utc_offset":25200,"time_zone":"Bangkok","geo_enabled":true,"verified":false,"statuses_count":8717,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"642D8B","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/719364530\/aaff1d2aba42fa8b2686ad4036446f4f.jpeg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/719364530\/aaff1d2aba42fa8b2686ad4036446f4f.jpeg","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3121022968\/11e4ee571d6132e0b5975fbffd0c3188_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3121022968\/11e4ee571d6132e0b5975fbffd0c3188_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/625624674\/1353685846","profile_link_color":"FF0000","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"7AC3EE","profile_text_color":"3D1957","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"lang":"en"} 2 | {"created_at":"Mon Feb 04 04:47:01 +0000 2013","id":298291547881996288,"id_str":"298291547881996288","text":"@lord_maddie already?","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":298288483263463424,"in_reply_to_status_id_str":"298288483263463424","in_reply_to_user_id":723201168,"in_reply_to_user_id_str":"723201168","in_reply_to_screen_name":"lord_maddie","user":{"id":60210285,"id_str":"60210285","name":"Hannah Long \u2655","screen_name":"longggtime","location":"Metro Motor City","url":"http:\/\/www.letsdressmedown.tumblr.com","description":"Legal at the casinos. My university fires up. I'm sensibly insensitive. Cheetah print and sparkles make the world go 'round.","protected":false,"followers_count":440,"friends_count":247,"listed_count":1,"created_at":"Sun Jul 26 02:29:02 +0000 2009","favourites_count":894,"utc_offset":-18000,"time_zone":"Eastern Time (US & Canada)","geo_enabled":true,"verified":false,"statuses_count":8562,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"080808","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/747408959\/1c46b2ca63630fbe999c05edc3b2cedf.jpeg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/747408959\/1c46b2ca63630fbe999c05edc3b2cedf.jpeg","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3193747831\/376777726b5895f35373a889b7307367_normal.png","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3193747831\/376777726b5895f35373a889b7307367_normal.png","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/60210285\/1359928223","profile_link_color":"038543","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"F6F6F6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_{"created_at":"Sun Mar 03 06:21:50 +0000 2013","id":308099881665437697,"id_str":"308099881665437697","text":"He's never watched the breakfast club ...........what is wrong with my generation","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":390312869,"id_str":"390312869","name":"Marian Nur","screen_name":"mari_nur","location":"Nation's Capital ","url":null,"description":"Selling illegally imported Green Tea in order to pay my university tuition to make it in this capatilist society (economist in the making)insta\/kik:mari_nur","protected":false,"followers_count":388,"friends_count":376,"listed_count":1,"created_at":"Thu Oct 13 19:52:16 +0000 2011","favourites_count":77,"utc_offset":-18000,"time_zone":"Quito","geo_enabled":true,"verified":false,"statuses_count":13489,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/783374195\/747030206eab852cfb6dd10e464d956e.jpeg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/783374195\/747030206eab852cfb6dd10e464d956e.jpeg","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3140131716\/f76a05be014e71c7aea373302c535136_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3140131716\/f76a05be014e71c7aea373302c535136_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/390312869\/1358915521","profile_link_color":"0084B4","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium"} 3 | {"created_at":"Wed Sep 04 18:40:44 +0000 2013","id":375327587041816576,"id_str":"375327587041816576","text":"\u201c@jaidah_jai: Love my university seminar teacher, she's real lol\u201d who do you have?","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":375327501117296640,"in_reply_to_status_id_str":"375327501117296640","in_reply_to_user_id":238286082,"in_reply_to_user_id_str":"238286082","in_reply_to_screen_name":"jaidah_jai","user":{"id":1166748224,"id_str":"1166748224","name":"Tiffanyyy","screen_name":"HavocDinero","location":"","u{"created_at":"Sat May 04 21:11:18 +0000 2013","id":330791770923347968,"id_str":"330791770923347968","text":"Happening tomorrow in Chapelfield Gardens btw https:\/\/t.co\/Vc83S0JTIu #MayDay #Norwich","source":"web","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":877974456,"id_str":"877974456","name":"Alex Francis","screen_name":"FrancoBollywood","location":"Norwich","url":"http:\/\/facebook.com\/AJHFdesign","description":"Designer \/ Artist \/ Filmmaker, living and studying in Norwich. \r\n\r\n#ProudtobeaSocialist + a feminist. \r\n\r\nMy university blog: http:\/\/ajhfnuca.blogspot.co.uk","protected":false,"followers_count":105,"friends_count":243,"listed_count":0,"created_at":"Sat Oct 13 14:42:16 +0000 2012","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":529,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3443631247\/b1be676d2777278adb8b0bcd8d91d85c_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3443631247\/b1be676d2777278adb8b0bcd8d91d85c_normal.jpeg","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"MayDay","indices":[70,77]},{"text":"Norwich","indices":[78,86]}],"symbols":[],"urls":[{"url":"https:\/\/t.co\/Vc83S0JTIu","expanded_url":"https:\/\/fbcdn-sphotos-g-a.akamaihd.net\/hphotos-ak-frc3\/485281_368161483292920_155963762_n.jpg","display_url":"fbcdn-sphotos-g-a.akamaihd.net\/hphotos-ak-frc\u2026","indices":[46,69]}],"user_mentions":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"medium","lang":"en"} 4 | -------------------------------------------------------------------------------- /testdata/retweet_x1: -------------------------------------------------------------------------------- 1 | {"created_at":"Sun Sep 01 08:28:37 +0000 2013","id":374086379254587393,"id_str":"374086379254587393","text":"RT @MrMichaelSpicer: When I take my apron off, I do it aggressively so it looks like I'm a chef quitting his job. It's fun! Keeps the tears\u2026","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":67263530,"id_str":"67263530","name":"Marianne Levy","screen_name":"MarianneLevy","location":"London","url":"http:\/\/www.mariannelevy.com","description":"I write Ellie May books. Portrait by @alipyeillo","protected":false,"followers_count":817,"friends_count":164,"listed_count":13,"created_at":"Thu Aug 20 08:23:14 +0000 2009","favourites_count":44,"utc_offset":3600,"time_zone":"London","geo_enabled":false,"verified":false,"statuses_count":12548,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"3482AD","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/30907358\/gems.jpg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/30907358\/gems.jpg","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000294699553\/350a633f8d905512304ee7866c57d230_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000294699553\/350a633f8d905512304ee7866c57d230_normal.jpeg","profile_link_color":"0084B4","profile_sidebar_border_color":"BDDCAD","profile_sidebar_fill_color":"DDFFCC","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Sun Sep 01 08:28:25 +0000 2013","id":374086328172175361,"id_str":"374086328172175361","text":"When I take my apron off, I do it aggressively so it looks like I'm a chef quitting his job. It's fun! Keeps the tears at bay anyway.","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":93222172,"id_str":"93222172","name":"Michael Spicer","screen_name":"MrMichaelSpicer","location":"Kent, London","url":"http:\/\/lolwagon.tumblr.com\/","description":"An impressively consistent disappointment.","protected":false,"followers_count":15955,"friends_count":843,"listed_count":213,"created_at":"Sat Nov 28 17:19:58 +0000 2009","favourites_count":185,"utc_offset":3600,"time_zone":"London","geo_enabled":false,"verified":false,"statuses_count":28250,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"709397","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme6\/bg.gif","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme6\/bg.gif","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000241570013\/063df73dc840c425eb76df9c37d665ab_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000241570013\/063df73dc840c425eb76df9c37d665ab_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/93222172\/1357725331","profile_link_color":"661E1E","profile_sidebar_border_color":"86A4A6","profile_sidebar_fill_color":"A0C5C7","profile_text_color":"302D2A","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":1,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"lang":"en"},"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"MrMichaelSpicer","name":"Michael Spicer","id":93222172,"id_str":"93222172","indices":[3,19]}]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 2 | -------------------------------------------------------------------------------- /testdata/shears.txt: -------------------------------------------------------------------------------- 1 | {"created_at":"Thu May 02 18:43:59 +0000 2013","id":330029921764253696,"id_str":"330029921764253696","text":"Love the feel of my shears sliding thru hair makes me feel alive hope u luv it cut my beautiful one @iHATEmrtampa #addicted2hair","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":915753656,"id_str":"915753656","name":"Jacque Roker Asberry","screen_name":"lovehairJacque","location":"San Antonio ","url":null,"description":"Love doing hair raising my kids and my fun job @AE live to love -love to live oh yea love being a BMS mommy","protected":false,"followers_count":22,"friends_count":75,"listed_count":0,"created_at":"Tue Oct 30 23:51:26 +0000 2012","favourites_count":86,"utc_offset":-18000,"time_zone":"Eastern Time (US & Canada)","geo_enabled":false,"verified":false,"statuses_count":128,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"642D8B","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme10\/bg.gif","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme10\/bg.gif","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/2786831753\/a2b215368cd60c347a6c039e8be54cee_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/2786831753\/a2b215368cd60c347a6c039e8be54cee_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/915753656\/1365821121","profile_link_color":"FF0000","profile_sidebar_border_color":"65B0DA","profile_sidebar_fill_color":"7AC3EE","profile_text_color":"3D1957","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"addicted2hair","indices":[114,128]}],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"iHATEmrtampa","name":"Sherri Allgood","id":1283503873,"id_str":"1283503873","indices":[100,113]}]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 2 | {"created_at":"Tue Jul 02 02:21:39 +0000 2013","id":351888369275510785,"id_str":"351888369275510785","text":"@ashears_ hahaha I was in a rush \ud83d\ude02","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":351880745704370176,"in_reply_to_status_id_str":"351880745704370176","in_reply_to_user_id":713518861,"in_reply_to_user_id_str":"713518861","in_reply_to_screen_name":"ashears_","user":{"id":579313230,"id_str":"579313230","name":"Maddie","screen_name":"mjvukelic10","location":"","url":null,"description":"Amy shears is my god","protected":false,"followers_count":66,"friends_count":40,"listed_count":0,"created_at":"Sun May 13 19:46:38 +0000 2012","favourites_count":109,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":120,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3733913060\/206686482099b71bb05eef30f6974f0a_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3733913060\/206686482099b71bb05eef30f6974f0a_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/579313230\/1372576780","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"ashears_","name":"Amy","id":713518861,"id_str":"713518861","indices":[0,9]}]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"nl"} 3 | {"created_at":"Tue Sep 03 22:21:16 +0000 2013","id":375020698223517696,"id_str":"375020698223517696","text":"Got my shears today!! Part one and a half of my full kit (: already cut myself on them LOL. http:\/\/t.co\/u0saWyvYzD","source":"\u003ca href=\"http:\/\/instagram.com\" rel=\"nofollow\"\u003eInstagram\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":427972336,"id_str":"427972336","name":"Tanner Mills","screen_name":"Supa_Tannah","location":"Kapolei, HI","url":"http:\/\/www.facebook.com\/Squanto.Baby","description":"I'm actually not that interesting. I don't tweet meaningful things, I just say what comes to my twisted mind. I'm crazy.","protected":false,"followers_count":231,"friends_count":310,"listed_count":1,"created_at":"Sun Dec 04 05:46:33 +0000 2011","favourites_count":1008,"utc_offset":-36000,"time_zone":"Hawaii","geo_enabled":true,"verified":false,"statuses_count":9542,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"012A38","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme15\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme15\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/344513261581076213\/5812c66f070ce1b9095c11ac05a513b9_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/344513261581076213\/5812c66f070ce1b9095c11ac05a513b9_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/427972336\/1369298391","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[{"url":"http:\/\/t.co\/u0saWyvYzD","expanded_url":"http:\/\/instagram.com\/p\/d0JeMuQNgz\/","display_url":"instagram.com\/p\/d0JeMuQNgz\/","indices":[92,114]}],"user_mentions":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"medium","lang":"en"} 4 | {"created_at":"Wed Aug 07 00:46:39 +0000 2013","id":364910424900108289,"id_str":"364910424900108289","text":"Tattoo idea to go next to my shears ! What do you guys think ? http:\/\/t.co\/M37sRYHcY0","source":"\u003ca href=\"http:\/\/www.facebook.com\/twitter\" rel=\"nofollow\"\u003eFacebook\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":436306159,"id_str":"436306159","name":"Meagan White","screen_name":"MeaganWhite4","location":"North Carolina","url":null,"description":"I'm a hair stylist and makeup artist ! I absolutely love my life , and I share my life with the love of my life !! :]","protected":false,"followers_count":5,"friends_count":20,"listed_count":0,"created_at":"Wed Dec 14 01:28:21 +0000 2011","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":461,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/1692011517\/App_Photo_-_Copy_normal.jpg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/1692011517\/App_Photo_-_Copy_normal.jpg","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[{"url":"http:\/\/t.co\/M37sRYHcY0","expanded_url":"http:\/\/fb.me\/VmzmQrCN","display_url":"fb.me\/VmzmQrCN","indices":[63,85]}],"user_mentions":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"medium","lang":"en"} 5 | {"created_at":"Thu Feb 07 12:40:34 +0000 2013","id":299497884242554881,"id_str":"299497884242554881","text":"@BenMcGilloway @Antonation @marcjohn71 iam putting my shears up on EBAY Ben as a friend i will give you first shout make me an offer LOL","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":299497131612450816,"in_reply_to_status_id_str":"299497131612450816","in_reply_to_user_id":419709022,"in_reply_to_user_id_str":"419709022","in_reply_to_screen_name":"BenMcGilloway","user":{"id":246019599,"id_str":"246019599","name":"j.k","screen_name":"irishpoint","location":"Dublin","url":null,"description":"If there is no living in life, life is not worth living","protected":false,"followers_count":335,"friends_count":371,"listed_count":2,"created_at":"Tue Feb 01 23:31:43 +0000 2011","favourites_count":322,"utc_offset":0,"time_zone":"London","geo_enabled":true,"verified":false,"statuses_count":4832,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"ACDED6","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme18\/bg.gif","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme18\/bg.gif","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3181160688\/8b3e70c3c2cd866418ef243cc5f962ea_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3181160688\/8b3e70c3c2cd866418ef243cc5f962ea_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/246019599\/1359497734","profile_link_color":"038543","profile_sidebar_border_color":"EEEEEE","profile_sidebar_fill_color":"F6F6F6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[{"screen_name":"BenMcGilloway","name":"Ben McGilloway","id":419709022,"id_str":"419709022","indices":[0,14]},{"screen_name":"Antonation","name":"Antonation","id":28437364,"id_str":"28437364","indices":[15,26]},{"screen_name":"marcjohn71","name":"Shalamar","id":247536766,"id_str":"247536766","indices":[27,38]}]},"favorited":false,"retweeted":false,"lang":"en"} 6 | {"created_at":"Sun Apr 07 23:11:43 +0000 2013","id":321037602394341376,"id_str":"321037602394341376","text":"@DanWaldoHendrix I'm at Cole's house and just realized i left my shears at home...:( gah!","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":321034821478543361,"in_reply_to_status_id_str":"321034821478543361","in_reply_to_user_id":432903080,"in_reply_to_user_id_str":"432903080","in_reply_to_screen_name":"DanWaldoHendrix","user":{"id":228422014,"id_str":"228422014","name":"Rebecca ","screen_name":"RebeccaSPollock","location":"USA","url":null,"description":"21. Cosmetology student. Short. ","protected":false,"followers_count":55,"friends_count":260,"listed_count":0,"created_at":"Sun Dec 19 17:34:53 +0000 2010","favourites_count":83,"utc_offset":-21600,"time_zone":"Central Time (US & Canada)","geo_enabled":false,"verified":false,"statuses_count":347,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"8B542B","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme8\/bg.gif","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme8\/bg.gif","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/2985169056\/fcf6f27906e537a4b8164970abec8955_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/2985169056\/fcf6f27906e537a4b8164970abec8955_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/228422014\/1358741510","profile_link_color":"9D582E","profile_sidebar_border_color":"D9B17E","profile_sidebar_fill_color":"EADEAA","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[{"screen_name":"DanWaldoHendrix","name":"Daniel Hendrix","id":432903080,"id_str":"432903080","indices":[0,16]}]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 7 | {"created_at":"Sun Sep 08 07:53:40 +0000 2013","id":376614298862096385,"id_str":"376614298862096385","text":"45 seconds and my life over... Oh well, I gotta get over it. I put myself in this situation now I gotta get myself out of it...","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":967115324,"id_str":"967115324","name":"Valarie Morales","screen_name":"valarie_morales","location":"","url":null,"description":"I'm all about the cosmo life. My shears and Combs are my Bestfriends.","protected":false,"followers_count":49,"friends_count":75,"listed_count":0,"created_at":"Sat Nov 24 00:54:08 +0000 2012","favourites_count":235,"utc_offset":null,"time_zone":null,"geo_enabled":true,"verified":false,"statuses_count":2594,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"ACDED6","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme18\/bg.gif","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme18\/bg.gif","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000361042980\/deca93f00dae007a46d529bcd1923305_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000361042980\/deca93f00dae007a46d529bcd1923305_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/967115324\/1378247996","profile_link_color":"038543","profile_sidebar_border_color":"000000","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":{"type":"Point","coordinates":[34.5169268,-117.43221963]},"coordinates":{"type":"Point","coordinates":[-117.43221963,34.5169268]},"place":{"id":"93f507760e0f70ff","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/93f507760e0f70ff.json","place_type":"city","name":"Adelanto","full_name":"Adelanto, CA","country_code":"US","country":"United States","bounding_box":{"type":"Polygon","coordinates":[[[-117.507447,34.506485],[-117.507447,34.675216],[-117.334111,34.675216],[-117.334111,34.506485]]]},"attributes":{}},"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 8 | {"created_at":"Sun Sep 08 19:01:32 +0000 2013","id":376782373003608064,"id_str":"376782373003608064","text":"BIGGEST HEADACHE EVER!","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":967115324,"id_str":"967115324","name":"Valarie Morales","screen_name":"valarie_morales","location":"","url":null,"description":"I'm all about the cosmo life. My shears and Combs are my Bestfriends.","protected":false,"followers_count":49,"friends_count":75,"listed_count":0,"created_at":"Sat Nov 24 00:54:08 +0000 2012","favourites_count":235,"utc_offset":null,"time_zone":null,"geo_enabled":true,"verified":false,"statuses_count":2596,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"ACDED6","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme18\/bg.gif","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme18\/bg.gif","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000361042980\/deca93f00dae007a46d529bcd1923305_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000361042980\/deca93f00dae007a46d529bcd1923305_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/967115324\/1378247996","profile_link_color":"038543","profile_sidebar_border_color":"000000","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":{"type":"Point","coordinates":[34.5171133,-117.4322942]},"coordinates":{"type":"Point","coordinates":[-117.4322942,34.5171133]},"place":{"id":"93f507760e0f70ff","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/93f507760e0f70ff.json","place_type":"city","name":"Adelanto","full_name":"Adelanto, CA","country_code":"US","country":"United States","bounding_box":{"type":"Polygon","coordinates":[[[-117.507447,34.506485],[-117.507447,34.675216],[-117.334111,34.675216],[-117.334111,34.506485]]]},"attributes":{}},"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 9 | {"created_at":"Fri Jan 11 20:50:03 +0000 2013","id":289836594381336576,"id_str":"289836594381336576","text":"Everyyyytime I get my hair cut I have to grab my shears n fix my bangs myself smh","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":149134894,"id_str":"149134894","name":"RebeccaValentine","screen_name":"myvalentine_xO","location":"Cumberland, MD","url":null,"description":"they don't make em like me no more... matter fact, they never made em like me before ; ) *jus follow me, Id rather show u than tell u","protected":false,"followers_count":664,"friends_count":359,"listed_count":0,"created_at":"Fri May 28 14:12:34 +0000 2010","favourites_count":520,"utc_offset":-21600,"time_zone":"Central Time (US & Canada)","geo_enabled":true,"verified":false,"statuses_count":22418,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"DBE9ED","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/533772792\/xx2.jpg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/533772792\/xx2.jpg","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3093368902\/50c08f5d88d8243f9e2678b0f61137de_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3093368902\/50c08f5d88d8243f9e2678b0f61137de_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/149134894\/1354643187","profile_link_color":"CC3366","profile_sidebar_border_color":"DBE9ED","profile_sidebar_fill_color":"E6F6F9","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"lang":"en"} 10 | {"created_at":"Thu Apr 11 04:12:16 +0000 2013","id":322200401879367680,"id_str":"322200401879367680","text":"Dropped my shears today #worstdayever","source":"web","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":551600985,"id_str":"551600985","name":"Cassidy McAllen","screen_name":"CassidyMcAllen","location":"Madison","url":null,"description":"F-R-I-E-N-D-S fan for life. 20. Hair stylist.\r\n\r\n Let's hold the night for ransom and kidnap the memories","protected":false,"followers_count":103,"friends_count":153,"listed_count":0,"created_at":"Thu Apr 12 02:58:27 +0000 2012","favourites_count":166,"utc_offset":-18000,"time_zone":"Eastern Time (US & Canada)","geo_enabled":false,"verified":false,"statuses_count":707,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"131516","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme14\/bg.gif","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme14\/bg.gif","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3425991251\/6dc3871725992bb72e008c6c93ca51aa_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3425991251\/6dc3871725992bb72e008c6c93ca51aa_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/551600985\/1359777004","profile_link_color":"009999","profile_sidebar_border_color":"EEEEEE","profile_sidebar_fill_color":"EFEFEF","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"worstdayever","indices":[24,37]}],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 11 | {"created_at":"Mon Nov 11 01:07:21 +0000 2013","id":399704869344272384,"id_str":"399704869344272384","text":"As a hairstylist I find myself judging people's hair..Haha. Sorry dude, ur rat tail is disgusting & I wish I had my shears with me haha","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":453442360,"id_str":"453442360","name":"andrea adams\u270c","screen_name":"anndrieuh","location":"Caldwell Idaho","url":null,"description":"\u2022I want my heart & my passion to be the most beautiful things about me\u2022 http:\/\/anndrieuh.tumblr.com","protected":false,"followers_count":156,"friends_count":208,"listed_count":0,"created_at":"Mon Jan 02 22:56:14 +0000 2012","favourites_count":1352,"utc_offset":-28800,"time_zone":"Pacific Time (US & Canada)","geo_enabled":true,"verified":false,"statuses_count":7343,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"ED4080","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/830362180\/50975d7e7e8d55d4def760df0b1492b7.jpeg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/830362180\/50975d7e7e8d55d4def760df0b1492b7.jpeg","profile_background_tile":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/378800000635824790\/ffdefdb353cbe4aec7302793de948e99_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/378800000635824790\/ffdefdb353cbe4aec7302793de948e99_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/453442360\/1364784918","profile_link_color":"98F0E9","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"272727","profile_text_color":"B3617B","profile_use_background_image":false,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":{"type":"Point","coordinates":[43.67331463,-116.67764883]},"coordinates":{"type":"Point","coordinates":[-116.67764883,43.67331463]},"place":{"id":"7d563a4149988303","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/7d563a4149988303.json","place_type":"city","name":"Caldwell","full_name":"Caldwell, ID","country_code":"US","country":"United States","contained_within":[],"bounding_box":{"type":"Polygon","coordinates":[[[-116.717678,43.611802],[-116.717678,43.697787],[-116.62109,43.697787],[-116.62109,43.611802]]]},"attributes":{}},"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 12 | {"created_at":"Thu Sep 12 16:58:23 +0000 2013","id":378200932774727680,"id_str":"378200932774727680","text":"I hate that I have to round brush my hair, but if I don't then my hair has no volume... #thestruggle","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":967115324,"id_str":"967115324","name":"Valarie Morales","screen_name":"valarie_morales","location":"","url":null,"description":"I'm all about the cosmo life. My shears and Combs are my Bestfriends.","protected":false,"followers_count":50,"friends_count":76,"listed_count":0,"created_at":"Sat Nov 24 00:54:08 +0000 2012","favourites_count":235,"utc_offset":null,"time_zone":null,"geo_enabled":true,"verified":false,"statuses_count":2579,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"ACDED6","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme18\/bg.gif","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme18\/bg.gif","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000361042980\/deca93f00dae007a46d529bcd1923305_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000361042980\/deca93f00dae007a46d529bcd1923305_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/967115324\/1378247996","profile_link_color":"038543","profile_sidebar_border_color":"000000","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":{"type":"Point","coordinates":[34.51708659,-117.43229926]},"coordinates":{"type":"Point","coordinates":[-117.43229926,34.51708659]},"place":{"id":"93f507760e0f70ff","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/93f507760e0f70ff.json","place_type":"city","name":"Adelanto","full_name":"Adelanto, CA","country_code":"US","country":"United States","bounding_box":{"type":"Polygon","coordinates":[[[-117.507447,34.506485],[-117.507447,34.675216],[-117.334111,34.675216],[-117.334111,34.506485]]]},"attributes":{}},"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"thestruggle","indices":[88,100]}],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 13 | {"created_at":"Thu Feb 14 04:26:38 +0000 2013","id":301910296916807681,"id_str":"301910296916807681","text":"I want to get my shears with passion written really fancy underneath them tattooed.","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":275300160,"id_str":"275300160","name":"alisa#swavey . ","screen_name":"alisarosee","location":"worcester","url":null,"description":"19 ` est 93. #TeamVirgo. Licensed Cosmetologist ' Single. Roc Nation. Jermaine Lamarr Cole. ToryLanez. #TeamTatted #TeamiPhone. Follow Me ;-*","protected":false,"followers_count":563,"friends_count":350,"listed_count":1,"created_at":"Fri Apr 01 01:00:50 +0000 2011","favourites_count":784,"utc_offset":null,"time_zone":null,"geo_enabled":true,"verified":false,"statuses_count":57174,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"FFFFFF","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/396571369\/j-cole.jpg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/396571369\/j-cole.jpg","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3221652823\/71c8167d7aefb82b9636acf645fc0eaf_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3221652823\/71c8167d7aefb82b9636acf645fc0eaf_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/275300160\/1360722510","profile_link_color":"999999","profile_sidebar_border_color":"333333","profile_sidebar_fill_color":"000000","profile_text_color":"404040","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false} 14 | {"created_at":"Sat Sep 14 01:13:10 +0000 2013","id":378687836935180288,"id_str":"378687836935180288","text":"RT @demaskingtape: Thought it'd be a good idea to wipe off my shears with my thumb and now there's a fucking bloodbath","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":82925313,"id_str":"82925313","name":"Chelsea Camacho","screen_name":"MachoAvgPerson","location":"Chesapeake, VA","url":null,"description":"iPhone user. Basketball player. PS3 gamer. Motorcycle rider. Mechanic major. So in a nutshell? Not your average.","protected":false,"followers_count":257,"friends_count":266,"listed_count":1,"created_at":"Fri Oct 16 17:10:46 +0000 2009","favourites_count":40,"utc_offset":-14400,"time_zone":"Eastern Time (US & Canada)","geo_enabled":true,"verified":false,"statuses_count":29471,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"131516","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme14\/bg.gif","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme14\/bg.gif","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000398598960\/5613585b725a70ac5de59e114741f4e4_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000398598960\/5613585b725a70ac5de59e114741f4e4_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/82925313\/1378054238","profile_link_color":"B50202","profile_sidebar_border_color":"EEEEEE","profile_sidebar_fill_color":"EFEFEF","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Sat Sep 14 01:10:56 +0000 2013","id":378687273501720576,"id_str":"378687273501720576","text":"Thought it'd be a good idea to wipe off my shears with my thumb and now there's a fucking bloodbath","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":55448888,"id_str":"55448888","name":"Lauren DeMaske","screen_name":"demaskingtape","location":"LACE UP","url":null,"description":"Don't make it weird.","protected":false,"followers_count":517,"friends_count":331,"listed_count":3,"created_at":"Fri Jul 10 02:53:53 +0000 2009","favourites_count":1901,"utc_offset":-18000,"time_zone":"Quito","geo_enabled":true,"verified":false,"statuses_count":34949,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"FAF5FA","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/378800000071601921\/98c4da6ad1f2eebd368aa5446f02d534.jpeg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/378800000071601921\/98c4da6ad1f2eebd368aa5446f02d534.jpeg","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000331890845\/194956ce2c7f734c2a238a191ee9b891_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000331890845\/194956ce2c7f734c2a238a191ee9b891_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/55448888\/1377409122","profile_link_color":"2E5761","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"FCFCF7","profile_text_color":"657070","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":1,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"lang":"en"},"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"demaskingtape","name":"Lauren DeMaske","id":55448888,"id_str":"55448888","indices":[3,17]}]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 15 | {"created_at":"Fri Aug 16 13:32:22 +0000 2013","id":368364614591188992,"id_str":"368364614591188992","text":"Found my shears :D","source":"\u003ca href=\"http:\/\/tweetlogix.com\" rel=\"nofollow\"\u003eTweetlogix\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":172536652,"id_str":"172536652","name":"Jess","screen_name":"jezzynotjessica","location":"kanye's next concert","url":null,"description":"Pokes holes in condoms","protected":false,"followers_count":426,"friends_count":222,"listed_count":1,"created_at":"Thu Jul 29 23:32:48 +0000 2010","favourites_count":985,"utc_offset":-18000,"time_zone":"Quito","geo_enabled":true,"verified":false,"statuses_count":66786,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"8400B8","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/497786901\/f.gif","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/497786901\/f.gif","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000300068687\/1a4df7254469d6e85701206846700db5_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000300068687\/1a4df7254469d6e85701206846700db5_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/172536652\/1364754892","profile_link_color":"8D7191","profile_sidebar_border_color":"5ED4DC","profile_sidebar_fill_color":"95E8EC","profile_text_color":"3C3940","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 16 | {"created_at":"Thu May 16 13:57:49 +0000 2013","id":335031335687188481,"id_str":"335031335687188481","text":"MY SHEARS CAME IN THE MAIL!!!!!!!!!!!","source":"\u003ca href=\"http:\/\/www.apple.com\" rel=\"nofollow\"\u003eiOS\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":878349523,"id_str":"878349523","name":"Kylie Mooney","screen_name":"kkkyyym00n","location":"Brunswick, OH","url":"http:\/\/therealkymoon.tumblr.com","description":"Im cocky and I hate everyone. Team #gokillyourself","protected":false,"followers_count":296,"friends_count":195,"listed_count":1,"created_at":"Sat Oct 13 17:49:00 +0000 2012","favourites_count":415,"utc_offset":-18000,"time_zone":"Eastern Time (US & Canada)","geo_enabled":true,"verified":false,"statuses_count":3845,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"131516","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme14\/bg.gif","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme14\/bg.gif","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3293484137\/2939a6716d2dca12deb69d112c03d57d_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3293484137\/2939a6716d2dca12deb69d112c03d57d_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/878349523\/1363709187","profile_link_color":"009999","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":false,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 17 | {"created_at":"Mon Sep 16 13:22:39 +0000 2013","id":379596193149767680,"id_str":"379596193149767680","text":"OMFG I can't find my shears kit I am way beyond panicking! #awful #waaa #panickattack","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1084598546,"id_str":"1084598546","name":"Olivia Marie Jones","screen_name":"LivsJones20","location":"Boca Raton","url":null,"description":"I love it up here in florida! I love and miss my dogs and my family but, i have great new skewl and friends. I love music and the beach because it's so beauti","protected":false,"followers_count":5,"friends_count":8,"listed_count":0,"created_at":"Sun Jan 13 01:06:42 +0000 2013","favourites_count":0,"utc_offset":-14400,"time_zone":"Eastern Time (US & Canada)","geo_enabled":true,"verified":false,"statuses_count":55,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3121710262\/132255a5bb2a0a2edbdc36119f5abada_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3121710262\/132255a5bb2a0a2edbdc36119f5abada_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1084598546\/1358452513","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"awful","indices":[59,65]},{"text":"waaa","indices":[66,71]},{"text":"panickattack","indices":[72,85]}],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 18 | {"created_at":"Thu Oct 17 21:38:19 +0000 2013","id":390954955575545856,"id_str":"390954955575545856","text":"Cut my knuckle with my shears \ud83d\ude14 #hairdresserprobs","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1568491706,"id_str":"1568491706","name":"Giselle Brooks","screen_name":"_giselleb_","location":"","url":null,"description":"21 Years old \nHair dresser\nMake up artist\nI love making people look and feel pretty :)","protected":false,"followers_count":73,"friends_count":92,"listed_count":0,"created_at":"Thu Jul 04 15:55:12 +0000 2013","favourites_count":198,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":818,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000440218593\/ea07662a0e26903fdbbf640d7198ec11_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000440218593\/ea07662a0e26903fdbbf640d7198ec11_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1568491706\/1372954022","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"hairdresserprobs","indices":[32,49]}],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 19 | {"created_at":"Wed Sep 18 22:57:11 +0000 2013","id":380465554949013505,"id_str":"380465554949013505","text":"@LovePatrisia yes yes , I still need to get my shears and our promise","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":380465046884589569,"in_reply_to_status_id_str":"380465046884589569","in_reply_to_user_id":625171693,"in_reply_to_user_id_str":"625171693","in_reply_to_screen_name":"LovePatrisia","user":{"id":926839818,"id_str":"926839818","name":"lil booty judy.","screen_name":"ayishabiitch","location":"","url":null,"description":"San Antonio, Texas","protected":false,"followers_count":621,"friends_count":619,"listed_count":2,"created_at":"Mon Nov 05 04:31:16 +0000 2012","favourites_count":3004,"utc_offset":null,"time_zone":null,"geo_enabled":true,"verified":false,"statuses_count":20122,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000466503562\/f9491ab96f7098dfe40d6d9c0aab51d2_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000466503562\/f9491ab96f7098dfe40d6d9c0aab51d2_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/926839818\/1379278841","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"LovePatrisia","name":"Pataty","id":625171693,"id_str":"625171693","indices":[0,13]}]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 20 | {"created_at":"Sun Oct 20 21:59:41 +0000 2013","id":392047496274272256,"id_str":"392047496274272256","text":"I hate bangs that don't touch the eyebrow . Ima start walking around with my shears again \ud83d\udc87\ud83d\ude33 .","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":623580496,"id_str":"623580496","name":"Sheila Marie","screen_name":"OHsheeeliaa","location":"","url":null,"description":"she's a lil bit sicker than most \u2728","protected":false,"followers_count":149,"friends_count":116,"listed_count":0,"created_at":"Sun Jul 01 09:15:35 +0000 2012","favourites_count":422,"utc_offset":-25200,"time_zone":"Arizona","geo_enabled":true,"verified":false,"statuses_count":8305,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/653991776\/ob5loewwvvhan3revk9d.jpeg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/653991776\/ob5loewwvvhan3revk9d.jpeg","profile_background_tile":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/378800000581416382\/241ba7ab46b83a8def116c7f2ccb380f_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/378800000581416382\/241ba7ab46b83a8def116c7f2ccb380f_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/623580496\/1382126141","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 21 | {"created_at":"Mon Jan 21 20:32:31 +0000 2013","id":293456060633718786,"id_str":"293456060633718786","text":"@MollyPaxxton Hey r u gonna be in Huntington or charleston this evening? I really need my shears and masters :)","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":29986009,"in_reply_to_user_id_str":"29986009","in_reply_to_screen_name":"MollyPaxxton","user":{"id":972879990,"id_str":"972879990","name":"jaybaybay","screen_name":"jasonclaytonlan","location":"","url":null,"description":null,"protected":false,"followers_count":48,"friends_count":133,"listed_count":0,"created_at":"Mon Nov 26 23:19:19 +0000 2012","favourites_count":53,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":72,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/2996999963\/45e7fd8ae83467213fde458a33744197_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/2996999963\/45e7fd8ae83467213fde458a33744197_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/972879990\/1355971321","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[{"screen_name":"MollyPaxxton","name":"Molly Paxton","id":29986009,"id_str":"29986009","indices":[0,13]}]},"favorited":false,"retweeted":false,"lang":"en"} 22 | {"created_at":"Wed Mar 20 01:24:40 +0000 2013","id":314185690906779649,"id_str":"314185690906779649","text":"The Potter wants to put you back together again:)) Faith moves mountains! Speak to your mountain and tell it to get out of the way..#prayers","source":"\u003ca href=\"http:\/\/www.motorola.com\" rel=\"nofollow\"\u003eMOTOBLUR\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":150745186,"id_str":"150745186","name":"Shannon Rogers","screen_name":"MyShearsRock","location":"Planet Shannon","url":null,"description":"Live, Pray, Enjoy! \r\n\r\nMy Shears Rock because I love what I do. Hair Maven extraordinare! It is my passion.;0)","protected":false,"followers_count":35,"friends_count":193,"listed_count":0,"created_at":"Tue Jun 01 18:22:27 +0000 2010","favourites_count":41,"utc_offset":-18000,"time_zone":"Quito","geo_enabled":false,"verified":false,"statuses_count":211,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/690564038\/e2355664980dd4de83211734558cd4c7.jpeg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/690564038\/e2355664980dd4de83211734558cd4c7.jpeg","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/2748808273\/84fcdb7f6465707279b682a60c11e3c1_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/2748808273\/84fcdb7f6465707279b682a60c11e3c1_normal.jpeg","profile_link_color":"0084B4","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"entities":{"hashtags":[{"text":"prayers","indices":[132,140]}],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium"} 23 | {"created_at":"Fri Aug 23 15:08:42 +0000 2013","id":370925572722917376,"id_str":"370925572722917376","text":"Seriously my Bestfriends #Sebastian #wellalife http:\/\/t.co\/m8mykg4BIs","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":967115324,"id_str":"967115324","name":"Valarie Morales","screen_name":"valarie_morales","location":"","url":null,"description":"I'm all about the cosmo life. My shears and Combs are my Bestfriends.","protected":false,"followers_count":46,"friends_count":71,"listed_count":0,"created_at":"Sat Nov 24 00:54:08 +0000 2012","favourites_count":222,"utc_offset":null,"time_zone":null,"geo_enabled":true,"verified":false,"statuses_count":2514,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"ACDED6","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme18\/bg.gif","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme18\/bg.gif","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000275821643\/f718871ccdcbc85b501e86b77864c0ad_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000275821643\/f718871ccdcbc85b501e86b77864c0ad_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/967115324\/1372899110","profile_link_color":"038543","profile_sidebar_border_color":"000000","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"Sebastian","indices":[25,35]},{"text":"wellalife","indices":[36,46]}],"symbols":[],"urls":[],"user_mentions":[],"media":[{"id":370925572727111680,"id_str":"370925572727111680","indices":[47,69],"media_url":"http:\/\/pbs.twimg.com\/media\/BSXK19iCAAAi2Jg.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/BSXK19iCAAAi2Jg.jpg","url":"http:\/\/t.co\/m8mykg4BIs","display_url":"pic.twitter.com\/m8mykg4BIs","expanded_url":"http:\/\/twitter.com\/valarie_morales\/status\/370925572722917376\/photo\/1","type":"photo","sizes":{"small":{"w":340,"h":453,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":576,"h":768,"resize":"fit"},"medium":{"w":576,"h":768,"resize":"fit"}}}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"medium","lang":"en"} 24 | {"created_at":"Sat Mar 23 06:16:18 +0000 2013","id":315346246464315392,"id_str":"315346246464315392","text":"RT @CommonWhiteGrl: why wear jeans when you can wear leggings?","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":579313230,"id_str":"579313230","name":"Maddie","screen_name":"mjvukelic10","location":"","url":null,"description":"Amy shears is my god ","protected":false,"followers_count":42,"friends_count":27,"listed_count":0,"created_at":"Sun May 13 19:46:38 +0000 2012","favourites_count":19,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":15,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3381130319\/465f80278795066c088a0c03a6e2ab37_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3381130319\/465f80278795066c088a0c03a6e2ab37_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/579313230\/1363312828","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Tue Mar 19 01:41:39 +0000 2013","id":313827574369382401,"id_str":"313827574369382401","text":"why wear jeans when you can wear leggings?","source":"web","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1206476953,"id_str":"1206476953","name":"Common White Girl","screen_name":"CommonWhiteGrl","location":"Starbucks","url":null,"description":"If you fall under any of my tweets, congratulations, you're white. Created by: @thats0jack :)\r\nOFFICIAL COMMON WHITE GIRL VIDEO http:\/\/t.co\/cRK9825VgI","protected":false,"followers_count":169582,"friends_count":7,"listed_count":21,"created_at":"Fri Feb 22 01:26:30 +0000 2013","favourites_count":2,"utc_offset":-18000,"time_zone":"Eastern Time (US & Canada)","geo_enabled":false,"verified":false,"statuses_count":549,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"FFFFFF","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3293607953\/6c8878e90f18bc192c65438225610022_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3293607953\/6c8878e90f18bc192c65438225610022_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/1206476953\/1361658338","profile_link_color":"FF75C3","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":false,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":2203,"entities":{"hashtags":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":true},"retweet_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[{"screen_name":"CommonWhiteGrl","name":"Common White Girl","id":1206476953,"id_str":"1206476953","indices":[3,18]}]},"favorited":false,"retweeted":false,"filter_level":"medium"} 25 | {"created_at":"Thu Oct 24 16:29:54 +0000 2013","id":393414055060185088,"id_str":"393414055060185088","text":"WHAT is going on in homes\/families across America that students are going to school and killing their teachers?!?!??!","source":"\u003ca href=\"http:\/\/www.motorola.com\" rel=\"nofollow\"\u003eMOTOBLUR\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":150745186,"id_str":"150745186","name":"MyShearsRock","screen_name":"MyShearsRock","location":"Atlanta, GA","url":null,"description":"Living life to the MAXXX!\r\n\r\nMy Shears Rock because I am PASSIONATE about what I do. Hair Maven extraordinare! #Lights, Camera--Ready","protected":false,"followers_count":33,"friends_count":210,"listed_count":0,"created_at":"Tue Jun 01 18:22:27 +0000 2010","favourites_count":49,"utc_offset":-18000,"time_zone":"Quito","geo_enabled":false,"verified":false,"statuses_count":282,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/690564038\/e2355664980dd4de83211734558cd4c7.jpeg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/690564038\/e2355664980dd4de83211734558cd4c7.jpeg","profile_background_tile":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/3564679505\/3a5fb1ad5038ae91a0e4917657e48482_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/3564679505\/3a5fb1ad5038ae91a0e4917657e48482_normal.jpeg","profile_link_color":"0084B4","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 26 | {"created_at":"Wed Feb 27 18:03:53 +0000 2013","id":306827006845284352,"id_str":"306827006845284352","text":"The day a man put his hands on me is the day that I will tied him down, take my shears & cut his finger tips off with a smile on my face.","source":"\u003ca href=\"http:\/\/www.tweetcaster.com\" rel=\"nofollow\"\u003eTweetCaster for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":198259667,"id_str":"198259667","name":"HairWeaveKillaTho","screen_name":"TaylonTaylorDuh","location":"Hair Hollywood (Detroit)","url":null,"description":"Hi I live to style hair ... Follow my instagram for more photos:TaylonTaylor.","protected":false,"followers_count":2110,"friends_count":1247,"listed_count":28,"created_at":"Sun Oct 03 20:14:35 +0000 2010","favourites_count":58,"utc_offset":-18000,"time_zone":"Quito","geo_enabled":false,"verified":false,"statuses_count":62630,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"022330","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/352469744\/271139_1853970190178_1268490050_31541093_716448_n.jpg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/352469744\/271139_1853970190178_1268490050_31541093_716448_n.jpg","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3248182236\/1a599e6da7d2b7630c1bb07dd10762f6_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3248182236\/1a599e6da7d2b7630c1bb07dd10762f6_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/198259667\/1351521981","profile_link_color":"0084B4","profile_sidebar_border_color":"A8C7F7","profile_sidebar_fill_color":"C0DFEC","profile_text_color":"C7AE0E","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium"} 27 | {"created_at":"Sun Jan 27 01:31:16 +0000 2013","id":295343182865133569,"id_str":"295343182865133569","text":"Gold all in my chain\nGold all in my ring Gold all in my shears Gold all in my clippers gold all in my iphone http:\/\/t.co\/XqC4oVWa","source":"\u003ca href=\"http:\/\/instagr.am\" rel=\"nofollow\"\u003eInstagram\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":242900069,"id_str":"242900069","name":"\u2133\u026a\u04a1\u0639y","screen_name":"Mikey_Henger","location":"PA,NY, Where I Need To Be","url":"http:\/\/www.facebook.com\/people\/Mikey-Henger\/1021366407","description":"HairStylist. Fashionisto. Soccer Player\nThey say i'm an artist \nbecause I draw attention\nInstagram: @mikeyyyyyyy_","protected":false,"followers_count":1143,"friends_count":545,"listed_count":1,"created_at":"Tue Jan 25 21:18:20 +0000 2011","favourites_count":985,"utc_offset":-21600,"time_zone":"Central Time (US & Canada)","geo_enabled":false,"verified":false,"statuses_count":15916,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"000000","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/405758967\/photo_16_2.jpg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/405758967\/photo_16_2.jpg","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3014343324\/bc26e8999c1f025960f0426b784288e3_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3014343324\/bc26e8999c1f025960f0426b784288e3_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/242900069\/1356068029","profile_link_color":"F00C0C","profile_sidebar_border_color":"000000","profile_sidebar_fill_color":"000000","profile_text_color":"FF0D00","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"entities":{"hashtags":[],"urls":[{"url":"http:\/\/t.co\/XqC4oVWa","expanded_url":"http:\/\/instagr.am\/p\/U9_9VYCGoC\/","display_url":"instagr.am\/p\/U9_9VYCGoC\/","indices":[109,129]}],"user_mentions":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"lang":"en"} 28 | {"created_at":"Tue Aug 27 23:25:55 +0000 2013","id":372500252885925888,"id_str":"372500252885925888","text":"@EmilyThuerauf that's fine I can take my shears to it :) lemme know and I can set u up something.","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":372500075697565696,"in_reply_to_status_id_str":"372500075697565696","in_reply_to_user_id":380126967,"in_reply_to_user_id_str":"380126967","in_reply_to_screen_name":"EmilyThuerauf","user":{"id":299862992,"id_str":"299862992","name":"Emily Pata","screen_name":"Emily_Pata","location":"","url":null,"description":null,"protected":false,"followers_count":112,"friends_count":138,"listed_count":0,"created_at":"Mon May 16 20:09:04 +0000 2011","favourites_count":7521,"utc_offset":-18000,"time_zone":"Central Time (US & Canada)","geo_enabled":true,"verified":false,"statuses_count":2542,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000324917404\/006bb9c7b06806ff9b1f18722c8fe3fe_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000324917404\/006bb9c7b06806ff9b1f18722c8fe3fe_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/299862992\/1362942643","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"EmilyThuerauf","name":"Emily Thuerauf","id":380126967,"id_str":"380126967","indices":[0,14]}]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 29 | {"created_at":"Fri Sep 27 12:44:56 +0000 2013","id":383572967986180097,"id_str":"383572967986180097","text":"I forgot my shears \ud83d\ude31 nooooo","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":110572753,"id_str":"110572753","name":"Meaghan Horner","screen_name":"meaghanhorner","location":"Fort Wayne, Indiana","url":"http:\/\/meaghannichole.tumblr.com","description":"Hi im Meaghan, follow me!","protected":false,"followers_count":242,"friends_count":325,"listed_count":4,"created_at":"Tue Feb 02 01:12:48 +0000 2010","favourites_count":2277,"utc_offset":-14400,"time_zone":"Indiana (East)","geo_enabled":true,"verified":false,"statuses_count":11635,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"EBEBEB","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme7\/bg.gif","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme7\/bg.gif","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000500617478\/3fe2426490081d05a6b5546eedaf3410_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000500617478\/3fe2426490081d05a6b5546eedaf3410_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/110572753\/1379179176","profile_link_color":"990000","profile_sidebar_border_color":"DFDFDF","profile_sidebar_fill_color":"F3F3F3","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 30 | {"created_at":"Tue Jun 25 01:44:10 +0000 2013","id":349342221209841664,"id_str":"349342221209841664","text":"I take naps with my grandma","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":579313230,"id_str":"579313230","name":"Maddie","screen_name":"mjvukelic10","location":"","url":null,"description":"Amy shears is my god","protected":false,"followers_count":64,"friends_count":40,"listed_count":0,"created_at":"Sun May 13 19:46:38 +0000 2012","favourites_count":101,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":106,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3733913060\/206686482099b71bb05eef30f6974f0a_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3733913060\/206686482099b71bb05eef30f6974f0a_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/579313230\/1367088497","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 31 | {"created_at":"Thu Mar 28 04:41:59 +0000 2013","id":317134450196639744,"id_str":"317134450196639744","text":"ily bby you're so good to me \ud83d\ude0d\ud83d\udc95 http:\/\/t.co\/2JDC8Q1vme","source":"\u003ca href=\"http:\/\/www.apple.com\" rel=\"nofollow\"\u003ePhotos on iOS\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":579313230,"id_str":"579313230","name":"Maddie","screen_name":"mjvukelic10","location":"","url":null,"description":"Amy shears is my god ","protected":false,"followers_count":43,"friends_count":28,"listed_count":0,"created_at":"Sun May 13 19:46:38 +0000 2012","favourites_count":22,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":22,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/3381130319\/465f80278795066c088a0c03a6e2ab37_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/3381130319\/465f80278795066c088a0c03a6e2ab37_normal.jpeg","profile_banner_url":"https:\/\/si0.twimg.com\/profile_banners\/579313230\/1363312828","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[],"media":[{"id":317134450205028353,"id_str":"317134450205028353","indices":[32,54],"media_url":"http:\/\/pbs.twimg.com\/media\/BGawGiqCcAEr8Gd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/BGawGiqCcAEr8Gd.jpg","url":"http:\/\/t.co\/2JDC8Q1vme","display_url":"pic.twitter.com\/2JDC8Q1vme","expanded_url":"http:\/\/twitter.com\/mjvukelic10\/status\/317134450196639744\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":600,"h":900,"resize":"fit"},"large":{"w":640,"h":960,"resize":"fit"},"small":{"w":340,"h":510,"resize":"fit"}}}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"medium","lang":"en"} 32 | {"created_at":"Thu Aug 29 14:59:43 +0000 2013","id":373097639182467072,"id_str":"373097639182467072","text":"I think I'm going to die. I'm not getting enough sleep with these make up hours...","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":967115324,"id_str":"967115324","name":"Valarie Morales","screen_name":"valarie_morales","location":"","url":null,"description":"I'm all about the cosmo life. My shears and Combs are my Bestfriends.","protected":false,"followers_count":46,"friends_count":73,"listed_count":0,"created_at":"Sat Nov 24 00:54:08 +0000 2012","favourites_count":225,"utc_offset":null,"time_zone":null,"geo_enabled":true,"verified":false,"statuses_count":2545,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"ACDED6","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme18\/bg.gif","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme18\/bg.gif","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000361042980\/deca93f00dae007a46d529bcd1923305_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000361042980\/deca93f00dae007a46d529bcd1923305_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/967115324\/1377485068","profile_link_color":"038543","profile_sidebar_border_color":"000000","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":{"type":"Point","coordinates":[34.51687584,-117.43226817]},"coordinates":{"type":"Point","coordinates":[-117.43226817,34.51687584]},"place":{"id":"93f507760e0f70ff","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/93f507760e0f70ff.json","place_type":"city","name":"Adelanto","full_name":"Adelanto, CA","country_code":"US","country":"United States","bounding_box":{"type":"Polygon","coordinates":[[[-117.507447,34.506485],[-117.507447,34.675216],[-117.334111,34.675216],[-117.334111,34.506485]]]},"attributes":{}},"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"en"} 33 | -------------------------------------------------------------------------------- /tweet_filter.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | 4 | import codecs 5 | import json 6 | import logging 7 | import re 8 | 9 | # Chromium Compact Language Detector 10 | # https://pypi.python.org/pypi/chromium_compact_language_detector/ 11 | import cld 12 | 13 | 14 | class FilteredTweetReader: 15 | """ 16 | Convenience class for reading only Tweets from a JSON Tweet file 17 | (one JSON object per line) that pass through zero or more filters. 18 | 19 | Usage: 20 | filtered_reader = FilteredTweetReader() 21 | filtered_reader.add_filter(TweetFilterOne()) 22 | filtered_reader.open('tweet_filename') 23 | for json_tweet_string in filtered_reader: 24 | do_something(json_tweet_string) 25 | """ 26 | def __del__(self): 27 | if self._tweet_file: 28 | self._tweet_file.close() 29 | 30 | def __init__(self, filters=[], logger=None): 31 | # First filter is always a TweetFilterValidJSON instance 32 | self._filters = [TweetFilterValidJSON(logger)] + filters 33 | self._tweet_file = None 34 | 35 | def __iter__(self): 36 | return self 37 | 38 | def add_filter(self, filter): 39 | self._filters.append(filter) 40 | 41 | def open(self, tweet_filename): 42 | self._tweet_file = codecs.open(tweet_filename, 'r', 'utf-8') 43 | 44 | def close(self): 45 | self._tweet_file.close() 46 | 47 | def next(self): 48 | while 1: 49 | # _tweet_file.__next__() will throw a StopIteration if EOF reached 50 | json_tweet_string = self._tweet_file.next() 51 | 52 | # Filters will stop being applied after the first filter fails 53 | for filter in self._filters: 54 | if not filter.filter(json_tweet_string): 55 | break 56 | # The else clause runs when no break occurs before the 'for' loop completes 57 | else: 58 | return json_tweet_string 59 | 60 | 61 | class TweetFilter: 62 | """ 63 | Base class for other TweetFilters 64 | """ 65 | def __init__(self, logger=None): 66 | if logger is None: 67 | # Log INFO and above to stderr 68 | self._logger = logging.getLogger() 69 | self._logger.setLevel(logging.INFO) 70 | console_handler = logging.StreamHandler() 71 | console_handler.setLevel(logging.INFO) 72 | self._logger.addHandler(console_handler) 73 | else: 74 | self._logger = logger 75 | 76 | def filter(self, json_tweet_string): 77 | raise NotImplementedError 78 | 79 | 80 | class TweetFilterReliablyEnglish(TweetFilter): 81 | """ 82 | Returns true IFF Chromium Compact Language Detector claims that Tweet is English. 83 | """ 84 | def filter(self, json_tweet_string): 85 | tweet = json.loads(json_tweet_string) 86 | # CLD expects a bytestring encoded as UTF-8, and not a unicode string 87 | tweet_text = codecs.encode(tweet['text'], 'utf-8') 88 | # Per the CLD docs, "isReliable is True if the top language is much better than 2nd best language." 89 | topLanguageName, topLanguageCode, isReliable, textBytesFound, details = cld.detect(tweet_text) 90 | if topLanguageName == "ENGLISH" and isReliable: 91 | return True 92 | else: 93 | return False 94 | 95 | 96 | class TweetFilterNoURLs(TweetFilter): 97 | def filter(self, json_tweet_string): 98 | tweet = json.loads(json_tweet_string) 99 | if re.search(r'https?://', tweet['text']): 100 | return False 101 | else: 102 | return True 103 | 104 | 105 | class TweetFilterOneTweetPerScreenName(TweetFilter): 106 | def __init__(self, logger=None): 107 | self._screen_name_set = set() 108 | TweetFilter.__init__(self, logger=logger) 109 | 110 | def filter(self, json_tweet_string): 111 | tweet = json.loads(json_tweet_string) 112 | screen_name = tweet['user']['screen_name'] 113 | if not screen_name in self._screen_name_set: 114 | self._screen_name_set.add(screen_name) 115 | return True 116 | else: 117 | return False 118 | 119 | 120 | class TweetFilterFieldMatchesRegEx(TweetFilter): 121 | def __init__(self, tweet_field, regex, logger=None): 122 | self._regex = regex 123 | self._tweet_field = tweet_field 124 | TweetFilter.__init__(self, logger=logger) 125 | 126 | def filter(self, json_tweet_string): 127 | """ 128 | Returns True if the Tweet field for the json_tweet_string 129 | matches the regex 130 | """ 131 | tweet = json.loads(json_tweet_string) 132 | if re.search(self._regex, tweet[self._tweet_field]): 133 | return True 134 | else: 135 | return False 136 | 137 | 138 | class TweetFilterIDSet(TweetFilter): 139 | """ 140 | Base class for TweetFilterIDInSet and TweetFilterIDNotInSet 141 | """ 142 | def __init__(self, logger=None): 143 | self._tweet_id_set = set() 144 | TweetFilter.__init__(self, logger=logger) 145 | 146 | def add_tweet(self, json_tweet_string): 147 | tweet = json.loads(json_tweet_string) 148 | self._tweet_id_set.add(tweet['id']) 149 | 150 | def add_tweets(self, json_tweet_string_list): 151 | for json_tweet_string in json_tweet_string_list: 152 | self.add_tweet(json_tweet_string) 153 | 154 | def add_tweet_id(self, tweet_id): 155 | self._tweet_id_set.add(tweet_id) 156 | 157 | def add_tweet_ids(self, tweet_ids): 158 | self._tweet_id_set.update(tweet_ids) 159 | 160 | def filter(self, json_tweet_string): 161 | raise NotImplementedError 162 | 163 | 164 | class TweetFilterTweetIDInSet(TweetFilterIDSet): 165 | def filter(self, json_tweet_string): 166 | """ 167 | Returns True if the Tweet's ID is in the existing set 168 | """ 169 | tweet = json.loads(json_tweet_string) 170 | return (tweet['id'] in self._tweet_id_set) or (tweet['id_str'] in self._tweet_id_set) 171 | 172 | 173 | class TweetFilterTweetIDNotInSet(TweetFilterIDSet): 174 | def filter(self, json_tweet_string): 175 | """ 176 | Returns True if the Tweet's ID is not in the existing set 177 | """ 178 | tweet = json.loads(json_tweet_string) 179 | return (tweet['id'] not in self._tweet_id_set) and (tweet['id_str'] not in self._tweet_id_set) 180 | 181 | 182 | class TweetFilterNotARetweet(TweetFilter): 183 | def filter(self, json_tweet_string): 184 | """ 185 | Returns True if json_tweet_string is not a retweet 186 | """ 187 | tweet = json.loads(json_tweet_string) 188 | 189 | if 'retweeted_status' in tweet: 190 | # Reject Tweets that the Twitter API considers to be retweets 191 | return False 192 | elif re.match(r'\s*RT\b', tweet['text']): 193 | # Reject Tweets that start with 'RT', even if not "officially" a retweet 194 | return False 195 | else: 196 | return True 197 | 198 | 199 | class TweetFilterValidJSON(TweetFilter): 200 | def filter(self, json_tweet_string): 201 | """ 202 | Returns True if json_tweet_string is a parsable JSON Tweet object 203 | """ 204 | try: 205 | tweet = json.loads(json_tweet_string) 206 | except ValueError: 207 | # self._logger.warning("JSON Tweet object could not be parsed") 208 | return False 209 | else: 210 | if type(tweet) is dict: 211 | for tweet_field in ['id', 'id_str', 'text', 'user']: 212 | if tweet_field not in tweet: 213 | # self._logger.warning("JSON Tweet object did not have a '%s' field" % tweet_field) 214 | return False 215 | if 'screen_name' not in tweet['user']: 216 | return False 217 | return True 218 | else: 219 | # self._logger.warning("JSON Tweet object evalauted to a %s instead of a dict" % type(tweet)) 220 | return False 221 | -------------------------------------------------------------------------------- /tweet_filter_timeline_downloadable.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | 4 | import codecs 5 | import json 6 | import logging 7 | import os 8 | import re 9 | import sys 10 | 11 | # Third party modules 12 | from twython import TwythonError 13 | 14 | # Local modules 15 | from tweet_filter import TweetFilter 16 | from twitter_crawler import RateLimitedTwitterEndpoint, save_tweets_to_json_file 17 | 18 | 19 | class TweetFilterTimelineDownloadable(TweetFilter): 20 | def __init__(self, twython, download_path, minimum_tweet_threshold, logger=None): 21 | self._crawler = RateLimitedTwitterEndpoint(twython, "statuses/user_timeline", logger) 22 | self._download_path = download_path 23 | self._minimum_tweet_threshold = minimum_tweet_threshold 24 | self._twython = twython 25 | TweetFilter.__init__(self, logger=logger) 26 | 27 | def filter(self, json_tweet_string): 28 | tweet = json.loads(json_tweet_string) 29 | screen_name = tweet['user']['screen_name'] 30 | 31 | path_to_tweetfile = os.path.join(self._download_path, "%s.tweets" % screen_name) 32 | 33 | # If file already exists for user, don't try to rescrape their timeline 34 | if os.path.exists(path_to_tweetfile): 35 | self._logger.info("Timeline file for '%s' already exists - will not rescrape" % screen_name) 36 | if os.path.getsize(path_to_tweetfile) > 0: 37 | return True 38 | else: 39 | return False 40 | 41 | try: 42 | self._logger.info("Retrieving Tweets for user '%s'" % screen_name) 43 | tweets = self._crawler.get_data(screen_name=screen_name, count=200) 44 | except TwythonError as e: 45 | print "TwythonError: %s" % e 46 | if e.error_code == 404: 47 | self._logger.warn("HTTP 404 error - Most likely, Twitter user '%s' no longer exists" % screen_name) 48 | open(path_to_tweetfile, "w").close() # Create empty file 49 | return False 50 | elif e.error_code == 401: 51 | self._logger.warn("HTTP 401 error - Most likely, Twitter user '%s' no longer publicly accessible" % screen_name) 52 | open(path_to_tweetfile, "w").close() # Create empty file 53 | return False 54 | else: 55 | # Unhandled exception 56 | raise e 57 | else: 58 | if len(tweets) < self._minimum_tweet_threshold: 59 | self._logger.info("User '%s' has only %d Tweets, threshold is %d" % \ 60 | (screen_name, len(tweets), self._minimum_tweet_threshold)) 61 | open(path_to_tweetfile, "w").close() # Create empty file 62 | return False 63 | else: 64 | save_tweets_to_json_file(tweets, path_to_tweetfile) 65 | return True 66 | -------------------------------------------------------------------------------- /twitter_crawler.py: -------------------------------------------------------------------------------- 1 | """ 2 | Shared classes and functions for crawling Twitter 3 | """ 4 | 5 | # Standard Library modules 6 | import codecs 7 | import datetime 8 | import itertools 9 | import json 10 | import logging 11 | import time 12 | 13 | # Third party modules 14 | from twython import Twython, TwythonError 15 | 16 | 17 | 18 | ### Functions ### 19 | 20 | def get_console_info_logger(): 21 | """ 22 | Return a logger that logs INFO and above to stderr 23 | """ 24 | logger = logging.getLogger() 25 | logger.setLevel(logging.INFO) 26 | console_handler = logging.StreamHandler() 27 | console_handler.setLevel(logging.INFO) 28 | logger.addHandler(console_handler) 29 | return logger 30 | 31 | 32 | def get_screen_names_from_file(filename): 33 | """ 34 | Opens a text file containing one Twitter screen name per line, 35 | returns a list of the screen names. 36 | """ 37 | screen_name_file = codecs.open(filename, "r", "utf-8") 38 | screen_names = [] 39 | for line in screen_name_file.readlines(): 40 | if line.strip(): 41 | screen_names.append(line.strip()) 42 | screen_name_file.close() 43 | return screen_names 44 | 45 | 46 | def grouper(iterable, n, fillvalue=None): 47 | """Collect data into fixed-length chunks or blocks""" 48 | # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx 49 | # Taken from: http://docs.python.org/2/library/itertools.html 50 | args = [iter(iterable)] * n 51 | return itertools.izip_longest(fillvalue=fillvalue, *args) 52 | 53 | 54 | def save_screen_names_to_file(screen_names, filename, logger): 55 | """ 56 | Saves a list of Twitter screen names to a text file with one 57 | screen name per line. 58 | """ 59 | logger.info("Saving %d screen names to file '%s'" % (len(screen_names), filename)) 60 | f = codecs.open(filename, 'w', 'utf-8') 61 | for screen_name in screen_names: 62 | f.write("%s\n" % screen_name) 63 | f.close() 64 | 65 | 66 | def save_tweets_to_json_file(tweets, json_filename): 67 | """ 68 | Takes a Python dictionary of Tweets from the Twython API, and 69 | saves the Tweets to a JSON file, storing one JSON object per 70 | line. 71 | """ 72 | json_file = codecs.open(json_filename, "w", "utf-8") 73 | for tweet in tweets: 74 | json_file.write("%s\n" % json.dumps(tweet)) 75 | json_file.close() 76 | 77 | 78 | 79 | ### Classes ### 80 | 81 | class CrawlTwitterTimelines: 82 | def __init__(self, twython, logger=None): 83 | if logger is None: 84 | self._logger = get_console_info_logger() 85 | else: 86 | self._logger = logger 87 | 88 | self._twitter_endpoint = RateLimitedTwitterEndpoint(twython, "statuses/user_timeline", logger=self._logger) 89 | 90 | 91 | def get_all_timeline_tweets_for_screen_name(self, screen_name): 92 | """ 93 | Retrieves all Tweets from a user's timeline based on this procedure: 94 | https://dev.twitter.com/docs/working-with-timelines 95 | """ 96 | # This function stops requesting additional Tweets from the timeline only 97 | # if the most recent number of Tweets retrieved is less than 100. 98 | # 99 | # This threshold may need to be adjusted. 100 | # 101 | # While we request 200 Tweets with each API, the number of Tweets we retrieve 102 | # will often be less than 200 because, for example, "suspended or deleted 103 | # content is removed after the count has been applied." See the API 104 | # documentation for the 'count' parameter for more info: 105 | # https://dev.twitter.com/docs/api/1.1/get/statuses/user_timeline 106 | MINIMUM_TWEETS_REQUIRED_FOR_MORE_API_CALLS = 100 107 | 108 | self._logger.info("Retrieving Tweets for user '%s'" % screen_name) 109 | 110 | # Retrieve first batch of Tweets 111 | tweets = self._twitter_endpoint.get_data(screen_name=screen_name, count=200) 112 | self._logger.info(" Retrieved first %d Tweets for user '%s'" % (len(tweets), screen_name)) 113 | 114 | if len(tweets) < MINIMUM_TWEETS_REQUIRED_FOR_MORE_API_CALLS: 115 | return tweets 116 | 117 | # Retrieve rest of Tweets 118 | while 1: 119 | max_id = int(tweets[-1]['id']) - 1 120 | more_tweets = self._twitter_endpoint.get_data(screen_name=screen_name, count=200, max_id=max_id) 121 | tweets += more_tweets 122 | self._logger.info(" Retrieved %d Tweets for user '%s' with max_id='%d'" % (len(more_tweets), screen_name, max_id)) 123 | 124 | if len(more_tweets) < MINIMUM_TWEETS_REQUIRED_FOR_MORE_API_CALLS: 125 | return tweets 126 | 127 | def get_all_timeline_tweets_for_screen_name_since(self, screen_name, since_id): 128 | """ 129 | Retrieves all Tweets from a user's timeline since the specified Tweet ID 130 | based on this procedure: 131 | https://dev.twitter.com/docs/working-with-timelines 132 | """ 133 | # This function stops requesting additional Tweets from the timeline only 134 | # if the most recent number of Tweets retrieved is less than 100. 135 | # 136 | # This threshold may need to be adjusted. 137 | # 138 | # While we request 200 Tweets with each API, the number of Tweets we retrieve 139 | # will often be less than 200 because, for example, "suspended or deleted 140 | # content is removed after the count has been applied." See the API 141 | # documentation for the 'count' parameter for more info: 142 | # https://dev.twitter.com/docs/api/1.1/get/statuses/user_timeline 143 | MINIMUM_TWEETS_REQUIRED_FOR_MORE_API_CALLS = 100 144 | 145 | self._logger.info("Retrieving Tweets for user '%s'" % screen_name) 146 | 147 | # Retrieve first batch of Tweets 148 | tweets = self._twitter_endpoint.get_data(screen_name=screen_name, count=200, since_id=since_id) 149 | self._logger.info(" Retrieved first %d Tweets for user '%s'" % (len(tweets), screen_name)) 150 | 151 | if len(tweets) < MINIMUM_TWEETS_REQUIRED_FOR_MORE_API_CALLS: 152 | return tweets 153 | 154 | # Retrieve rest of Tweets 155 | while 1: 156 | max_id = int(tweets[-1]['id']) - 1 157 | more_tweets = self._twitter_endpoint.get_data(screen_name=screen_name, count=200, max_id=max_id, since_id=since_id) 158 | tweets += more_tweets 159 | self._logger.info(" Retrieved %d Tweets for user '%s' with max_id='%d'" % (len(more_tweets), screen_name, since_id)) 160 | 161 | if len(more_tweets) < MINIMUM_TWEETS_REQUIRED_FOR_MORE_API_CALLS: 162 | return tweets 163 | 164 | 165 | 166 | class FindFriendFollowers: 167 | def __init__(self, twython, logger=None): 168 | if logger is None: 169 | self._logger = get_console_info_logger() 170 | else: 171 | self._logger = logger 172 | 173 | self._friend_endpoint = RateLimitedTwitterEndpoint(twython, "friends/ids", logger=self._logger) 174 | self._follower_endpoint = RateLimitedTwitterEndpoint(twython, "followers/ids", logger=self._logger) 175 | self._user_lookup_endpoint = RateLimitedTwitterEndpoint(twython, "users/lookup", logger=self._logger) 176 | 177 | 178 | def get_ff_ids_for_screen_name(self, screen_name): 179 | """ 180 | Returns Twitter user IDs for users who are both Friends and Followers 181 | for the specified screen_name. 182 | 183 | The 'friends/ids' and 'followers/ids' endpoints return at most 5000 IDs, 184 | so IF a user has more than 5000 friends or followers, this function WILL 185 | NOT RETURN THE CORRECT ANSWER 186 | """ 187 | try: 188 | friend_ids = self._friend_endpoint.get_data(screen_name=screen_name)[u'ids'] 189 | follower_ids = self._follower_endpoint.get_data(screen_name=screen_name)[u'ids'] 190 | except TwythonError as e: 191 | if e.error_code == 404: 192 | self._logger.warn("HTTP 404 error - Most likely, Twitter user '%s' no longer exists" % screen_name) 193 | elif e.error_code == 401: 194 | self._logger.warn("HTTP 401 error - Most likely, Twitter user '%s' no longer publicly accessible" % screen_name) 195 | else: 196 | # Unhandled exception 197 | raise e 198 | friend_ids = [] 199 | follower_ids = [] 200 | 201 | return list(set(friend_ids).intersection(set(follower_ids))) 202 | 203 | 204 | def get_ff_screen_names_for_screen_name(self, screen_name): 205 | """ 206 | Returns Twitter screen names for users who are both Friends and Followers 207 | for the specified screen_name. 208 | """ 209 | ff_ids = self.get_ff_ids_for_screen_name(screen_name) 210 | 211 | ff_screen_names = [] 212 | # The Twitter API allows us to look up info for 100 users at a time 213 | for ff_id_subset in grouper(ff_ids, 100): 214 | user_ids = ','.join([str(id) for id in ff_id_subset if id is not None]) 215 | users = self._user_lookup_endpoint.get_data(user_id=user_ids, entities=False) 216 | for user in users: 217 | ff_screen_names.append(user[u'screen_name']) 218 | return ff_screen_names 219 | 220 | 221 | 222 | class RateLimitedTwitterEndpoint: 223 | """ 224 | Class used to retrieve data from a Twitter API endpoint without 225 | violating Twitter's API rate limits for that API endpoint. 226 | 227 | Each Twitter API endpoint (e.g. 'statuses/user_timeline') has its 228 | own number of allotted requests per rate limit duration window: 229 | 230 | https://dev.twitter.com/docs/rate-limiting/1.1/limits 231 | 232 | The RateLimitedTwitterEndpoint class has a single public function, 233 | get_data(), that is a thin wrapper around the Twitter API. If the 234 | rate limit for the current window has been reached, the get_data() 235 | function will block for up to 15 minutes until the next rate limit 236 | window starts. 237 | 238 | Only one RateLimitedTwitterEndpoint instance should be running 239 | anywhere in the world per (Twitter API key, Twitter API endpoint) 240 | pair. Each class instance assumes it is the only program using up 241 | the API calls available for the current rate limit window. 242 | """ 243 | def __init__(self, twython, twitter_api_endpoint, logger=None): 244 | """ 245 | twython -- an instance of a twython.Twython object that has 246 | been initialized with a valid set of Twitter API credentials. 247 | 248 | twitter_api_endpoint -- a string that names a Twitter API 249 | endpoint (e.g. 'followers/ids', 'statuses/mentions_timeline'). 250 | The endpoint string should NOT have a leading slash (use 251 | 'followers/ids', NOT '/followers/ids'). For a full list of 252 | endpoints, see: 253 | 254 | https://dev.twitter.com/docs/api/1.1 255 | 256 | logger -- an optional instance of a logging.Logger class. 257 | """ 258 | self._twython = twython 259 | self._twitter_api_endpoint = twitter_api_endpoint 260 | self._twitter_api_endpoint_with_prefix = '/' + twitter_api_endpoint 261 | self._twitter_api_resource = twitter_api_endpoint.split('/')[0] 262 | 263 | if logger is None: 264 | self._logger = get_console_info_logger() 265 | else: 266 | self._logger = logger 267 | 268 | self._update_rate_limit_status() 269 | 270 | 271 | def get_data(self, **twitter_api_parameters): 272 | """ 273 | Retrieve data from the Twitter API endpoint associated with 274 | this class instance. 275 | 276 | This function can block for up to 15 minutes if the rate limit 277 | for this endpoint's window has already been reached. 278 | """ 279 | return self._get_data_with_backoff(60, **twitter_api_parameters) 280 | 281 | 282 | def _get_data_with_backoff(self, backoff, **twitter_api_parameters): 283 | self._sleep_if_rate_limit_reached() 284 | self._api_calls_remaining_for_current_window -= 1 285 | try: 286 | return self._twython.get(self._twitter_api_endpoint, params=twitter_api_parameters) 287 | except TwythonError as e: 288 | self._logger.error("TwythonError: %s" % e) 289 | 290 | # Twitter error codes: 291 | # https://dev.twitter.com/docs/error-codes-responses 292 | 293 | # Update rate limit status if exception is 'Too Many Requests' 294 | if e.error_code == 429: 295 | self._logger.error("Rate limit exceeded for '%s'. Number of expected remaining API calls for current window: %d" % 296 | (self._twitter_api_endpoint, self._api_calls_remaining_for_current_window + 1)) 297 | time.sleep(backoff) 298 | self._update_rate_limit_status() 299 | return self._get_data_with_backoff(backoff*2, **twitter_api_parameters) 300 | # Sleep if Twitter servers are misbehaving 301 | elif e.error_code in [502, 503, 504]: 302 | self._logger.error("Twitter servers are misbehaving - sleeping for %d seconds" % backoff) 303 | time.sleep(backoff) 304 | return self._get_data_with_backoff(backoff*2, **twitter_api_parameters) 305 | # Sleep if Twitter servers returned an empty HTTPS response 306 | elif "Caused by : ''" in str(e): 307 | # Twitter servers can sometimes return an empty HTTP response, e.g.: 308 | # https://dev.twitter.com/discussions/20832 309 | # 310 | # The code currently detects empty HTTPS responses by checking for a particular 311 | # string: 312 | # Caused by : ''" 313 | # in the exception message text, which is fragile and definitely not ideal. Twython 314 | # uses the Requests library, and the "Caused by %s: %s" string comes from the 315 | # version of urllib3 that is bundled with the Requests library. Upgrading to a 316 | # newer version of the Requests library (this code tested with requests 2.0.0) may 317 | # break the detection of empty HTTPS responses. 318 | # 319 | # The httplib library (which is part of the Python Standard Library) throws the 320 | # httplib.BadStatusLine exception, which is caught by urllib3, and then re-thrown 321 | # (with the "Caused by" text) as a urllib3.MaxRetryError. The Requests library 322 | # catches the urllib3.MaxRetryError and throws a requests.ConnectionError, and 323 | # Twython catches the requests.ConnectionError and throws a TwythonError exception - 324 | # which we catch in this function. 325 | self._logger.error("Received an empty HTTPS response from Twitter servers - sleeping for %d seconds" % backoff) 326 | time.sleep(backoff) 327 | return self._get_data_with_backoff(backoff*2, **twitter_api_parameters) 328 | # For all other TwythonErrors, reraise the exception 329 | else: 330 | raise e 331 | 332 | 333 | def _sleep_if_rate_limit_reached(self): 334 | if self._api_calls_remaining_for_current_window < 1: 335 | current_time = time.time() 336 | seconds_to_sleep = self._current_rate_limit_window_ends - current_time 337 | 338 | # Pad the sleep time by 15 seconds to compensate for possible clock skew 339 | seconds_to_sleep += 15 340 | 341 | # If the number of calls available is 0 and the rate limit window has already 342 | # expired, we sleep for 60 seconds before calling self._update_rate_limit_status() 343 | # again. 344 | # 345 | # In testing on 2013-11-06, the rate limit window could be expired for over a 346 | # minute before calls to the Twitter rate_limit_status API would return with 347 | # an updated window expiration timestamp and an updated (non-zero) count for 348 | # the number of API calls available. 349 | if seconds_to_sleep < 0: 350 | seconds_to_sleep = 60 351 | 352 | sleep_until = datetime.datetime.fromtimestamp(current_time + seconds_to_sleep).strftime("%Y-%m-%d %H:%M:%S") 353 | self._logger.info("Rate limit reached for '%s', sleeping for %.2f seconds (until %s)" % \ 354 | (self._twitter_api_endpoint, seconds_to_sleep, sleep_until)) 355 | time.sleep(seconds_to_sleep) 356 | 357 | self._update_rate_limit_status() 358 | 359 | # Recursion! Sleep some more if necessary after updating rate limit status 360 | self._sleep_if_rate_limit_reached() 361 | 362 | 363 | def _update_rate_limit_status(self): 364 | # https://dev.twitter.com/docs/api/1.1/get/application/rate_limit_status 365 | rate_limit_status = self._twython.get_application_rate_limit_status(resources=self._twitter_api_resource) 366 | 367 | self._current_rate_limit_window_ends = rate_limit_status['resources'][self._twitter_api_resource][self._twitter_api_endpoint_with_prefix]['reset'] 368 | 369 | self._api_calls_remaining_for_current_window = rate_limit_status['resources'][self._twitter_api_resource][self._twitter_api_endpoint_with_prefix]['remaining'] 370 | 371 | dt = int(self._current_rate_limit_window_ends - time.time()) 372 | rate_limit_ends = datetime.datetime.fromtimestamp(self._current_rate_limit_window_ends).strftime("%Y-%m-%d %H:%M:%S") 373 | self._logger.info("Rate limit status for '%s': %d calls remaining until %s (for next %d seconds)" % \ 374 | (self._twitter_api_endpoint, self._api_calls_remaining_for_current_window, rate_limit_ends, dt)) 375 | -------------------------------------------------------------------------------- /twitter_oauth_settings.sample.py: -------------------------------------------------------------------------------- 1 | access_token="" 2 | access_token_secret="" 3 | consumer_key="" 4 | consumer_secret="" 5 | --------------------------------------------------------------------------------