├── spider ├── Readme.md ├── settings.py └── main.py ├── beforeact.py ├── dayscats.pickle ├── tweetit.py ├── README.md ├── update.py └── postlog.log /spider/Readme.md: -------------------------------------------------------------------------------- 1 | This is the spider written in Dragline in order to fetch the quotes from a well known website. -------------------------------------------------------------------------------- /spider/settings.py: -------------------------------------------------------------------------------- 1 | SPIDER = { 2 | 3 | } 4 | 5 | REQUEST = { 6 | 7 | } 8 | 9 | CRAWL = { 10 | "RESUME": False 11 | } 12 | -------------------------------------------------------------------------------- /beforeact.py: -------------------------------------------------------------------------------- 1 | #Code for setting all intial values in a proper manner to start bot 2 | import redis 3 | import pickle 4 | from pymongo import MongoClient 5 | import datetime 6 | 7 | #creating mongoclient where quotes are stored in my db 8 | mydb = MongoClient('localhost')['goodread'] 9 | 10 | #Create a redis instance 11 | red = redis.Redis() 12 | red.set('per_day',120) 13 | red.set('today_balance',0) 14 | red.set('today_or_yesterday',datetime.datetime.today().timetuple().tm_mday) 15 | 16 | #Storing category names in redis for future use 17 | for i in mydb.collection_names(): 18 | if i!='system.indexes': 19 | for cur in mydb[i].find(): 20 | red.sadd(i,str(cur['_id'])) 21 | red.sadd('cats',i) 22 | 23 | #Creating a dictionary for mapping category to day in a month 24 | days_cats = {i+1:k for i,k in enumerate(red.smembers('cats'))} 25 | 26 | #Writing that mapping to a pickle file 27 | with open('dayscats.pickle','wb') as f: 28 | pickle.dump(days_cats,f) 29 | 30 | 31 | -------------------------------------------------------------------------------- /dayscats.pickle: -------------------------------------------------------------------------------- 1 | (dp0 2 | I1 3 | S'art' 4 | p1 5 | sI2 6 | S'humour' 7 | p2 8 | sI3 9 | S'quotes' 10 | p3 11 | sI4 12 | S'books' 13 | p4 14 | sI5 15 | S'politics' 16 | p5 17 | sI6 18 | S'love' 19 | p6 20 | sI7 21 | S'happiness' 22 | p7 23 | sI8 24 | S'relationships' 25 | p8 26 | sI9 27 | S'funny' 28 | p9 29 | sI10 30 | S'death' 31 | p10 32 | sI11 33 | S'humor' 34 | p11 35 | sI12 36 | S'inspiration' 37 | p12 38 | sI13 39 | S'god' 40 | p13 41 | sI14 42 | S'poetry' 43 | p14 44 | sI15 45 | S'writing' 46 | p15 47 | sI16 48 | S'romance' 49 | p16 50 | sI17 51 | S'religion' 52 | p17 53 | sI18 54 | S'reading' 55 | p18 56 | sI19 57 | S'war' 58 | p19 59 | sI20 60 | S'hope' 61 | p20 62 | sI21 63 | S'life' 64 | p21 65 | sI22 66 | S'inspirational' 67 | p22 68 | sI23 69 | S'inspirational-quotes' 70 | p23 71 | sI24 72 | S'philosophy' 73 | p24 74 | sI25 75 | S'wisdom' 76 | p25 77 | sI26 78 | S'women' 79 | p26 80 | sI27 81 | S'faith' 82 | p27 83 | sI28 84 | S'success' 85 | p28 86 | sI29 87 | S'science' 88 | p29 89 | sI30 90 | S'truth' 91 | p30 92 | sI31 93 | S'friendship' 94 | p31 95 | s. -------------------------------------------------------------------------------- /tweetit.py: -------------------------------------------------------------------------------- 1 | 2 | import tweepy 3 | 4 | #NPG class deals with all housekeeping for creating instances. 5 | class Tweet: 6 | def __init__(self): 7 | self.auth = tweepy.OAuthHandler(self.consumer_key,self.consumer_secret) 8 | self.auth.set_access_token(self.access_token, self.access_token_secret) 9 | self.handle = tweepy.API(self.auth) 10 | 11 | def get_handle(self): 12 | return self.handle 13 | 14 | def hitme(self,str): 15 | self.handle.update_status(str) 16 | print 'posted succesfully' 17 | 18 | def hit_with_image(self,fil,status=None): 19 | self.handle.update_with_media(fil,status=status) 20 | print 'Done successfully' 21 | 22 | def rehit(self,str): 23 | self.handle.retweet(str) 24 | print 'Done successfully' 25 | 26 | 27 | 28 | class QUOTE(Tweet): 29 | #My Twitter consumer key 30 | 31 | consumer_key='HHHHHHHHHHHHHHHHHHHHHHH' 32 | 33 | #My consumer secret 34 | 35 | consumer_secret='IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII' 36 | 37 | #My access token 38 | access_token='DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD' 39 | 40 | #My access token secret 41 | access_token_secret='EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE' 42 | 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Quotebot 2 | ======== 3 | 4 | A twitter bot that posts Famous quotes every day on Twitter without intervention of a human operator. 5 | The main difference between this bot and others is data posted is purely fetched by Spiders and stored in 6 | MongoDB. 7 | 8 | Twitter allows only 140 characters length of text,but quotes can be much longer.Inorder to overcome that issue ,if quote is converted into image it can be uploaded as a media file and serves the purpose. 9 | 10 | So this project uses an API for converting text to Image,and then uploads it to Twitter. 11 | 12 | Quotes will have the subject according to day in a month,means every 2nd of month Love Quotes will be posted, 13 | 3rd Inspiration quotes,etc.Daily Maximum 120 Quotes will be posted at maximum.For every 5 minutes,bot checks the 14 | connection automatically when system is active and posts the Quote image. 15 | 16 | Two APIs Twitter API,Img4free API are used to achieve the task. 17 | 18 | tweetit.py creates a class for encapsulating user details. 19 | beforeact.py do all housekeeping task and sets every thing ready for Quotebot to run. 20 | dayscats.pickle consists of mapping of month day -> category. 21 | 22 | postlog.log stores all logs of posted quotes and also network statistics 23 | -------------------------------------------------------------------------------- /spider/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from pymongo import MongoClient 3 | from dragline.runner import main 4 | from dragline.htmlparser import HtmlParser 5 | from dragline.http import Request 6 | import settings 7 | 8 | 9 | class Spider: 10 | mydb = MongoClient(host ="localhost")["goodread"] 11 | 12 | def __init__(self, conf): 13 | self.name = "brainy" 14 | self.start = "https://www.goodreads.com/quotes" 15 | self.allowed_domains = ["www.goodreads.com"] 16 | self.conf = conf 17 | 18 | def parse(self,response): 19 | parser = HtmlParser(response) 20 | for url in parser.extract_urls('//a[@class="actionLinkLite serif"]'): 21 | dbname = url.split('/')[-1] 22 | yield Request(url,callback="parseCat",meta={'u':dbname}) 23 | 24 | 25 | def parseCat(self, response): 26 | parser = HtmlParser(response) 27 | dbname= response.meta['u'] 28 | if not parser.xpath('//a[@class="next_page"]'): 29 | for i in parser.xpath('//div[@class="quoteText"]'): 30 | quote = i.text 31 | for j in i.iterfind('a'): 32 | author=j.text 33 | self.mydb[dbname].insert({'quote':quote,'author':author}) 34 | else: 35 | for i in parser.xpath('//div[@class="quoteText"]'): 36 | quote = i.text 37 | for j in i.iterfind('a'): 38 | author=j.text 39 | self.mydb[dbname].insert({'quote':quote,'author':author}) 40 | 41 | for url in parser.extract_urls('//a[@class="next_page"]'): 42 | yield Request(url,callback="parseCat",meta={'u':dbname}) 43 | 44 | if __name__ == '__main__': 45 | main(Spider, settings) 46 | -------------------------------------------------------------------------------- /update.py: -------------------------------------------------------------------------------- 1 | ''' 2 | The heart of the QuoteBot 3 | Here goes the actual thing.The tasks done here in following manner 4 | 1)Check the post limit 5 | 2)Fetch quote from database 6 | 3)Convert it into image(.png) using API 7 | 4)Upload it to twitter 8 | 9 | ''' 10 | 11 | #Imports for tweet and store 12 | 13 | from tweetit import QUOTE 14 | 15 | import datetime 16 | 17 | from beforeact import red,mydb 18 | 19 | import pickle,sys 20 | 21 | from bson import ObjectId 22 | 23 | import logging,requests 24 | 25 | logging.basicConfig(filename='postlog.log',level=logging.DEBUG) 26 | 27 | days_cats = pickle.load(open('dayscats.pickle','r')) 28 | 29 | quote = QUOTE() 30 | 31 | #credentials to be changed 32 | qhandle = quote.get_handle() 33 | 34 | def return_date(): 35 | day_of_month = datetime.datetime.today().timetuple().tm_mday 36 | return day_of_month 37 | 38 | def today_category(): 39 | day = return_date() 40 | return days_cats[day] 41 | 42 | #Creating categories set in redis which is used to Implement day category 43 | def generate_quote_image(qs,qid): 44 | payload = {'text':qs,'font':'arial','color':'000000','size':20,'bcolor':'FFFFFF','type':'png'} 45 | result1 = requests.get('http://api.img4me.com',params=payload) 46 | result2 = requests.get(result1.text.encode('utf-8')) 47 | with open(qid+'.png','wb') as f: 48 | f.write(result2.content) 49 | return qid+'.png' 50 | 51 | 52 | def give_quote(): 53 | quote_id = red.spop(today_category()) 54 | quote = (mydb[today_category()].find_one({'_id':ObjectId(quote_id)})['quote'].lstrip()).rstrip() 55 | author = mydb[today_category()].find_one({'_id':ObjectId(quote_id)})['author'] 56 | message = quote+'\n---\n'+author 57 | try: 58 | filename = generate_quote_image(message,quote_id) 59 | except: 60 | red.sadd(today_category,details[quote_id]) 61 | sys.exit() 62 | return locals() 63 | 64 | def do_status(): 65 | details = give_quote() 66 | try: 67 | qhandle.update_with_media(details['filename']) 68 | red.sadd('O'+today_category(),details['quote_id']) 69 | red.incr('today_balance') 70 | except: 71 | red.sadd(today_category,details[quote_id]) 72 | sys.exit() 73 | 74 | #Main program which runs the 75 | def run_it(): 76 | #Stop posting if count is greater than 120 77 | if int(red.get('today_balance'))