├── .gitignore ├── README ├── example_twitter_settings.py ├── file_system_status.py ├── httplib2 ├── __init__.py └── iri2uri.py ├── main.py ├── markovate.py ├── oauth2 ├── __init__.py └── clients │ ├── __init__.py │ ├── imap.py │ └── smtp.py ├── status.json ├── status_endpoint.py ├── test_status.py ├── test_twitter.py └── twitter.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *~ 3 | twitter_settings.py 4 | 5 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Making nonsense out of your carefully crafted tweets. 2 | 3 | Python app that controls a twitter bot that uses markov chains to produce interesting tweets based on it's followers tweets and the twitterers that mention it. When a user tweets a mention of the bot it will respond with a "markovation" of the last 800 tweets by that user (that don't contain links or mentions of other users). It will also follow that user. Every two hours it will tweet a markovation of the latest 800 tweets from all of the users it follows. 4 | 5 | Live markovator bot is running here: http://twitter.com/markovator 6 | More info here: http://www.heychinaski.com/blog/2010/10/18/markovator-python-google-app-engine-markov-chains-twitter/ 7 | 8 | Easiest way to use the script is to repeatedly call it using something like cron. Make sure to set the screen_name, token and consumer settings in the twitter_settings.py. The screen name should be the username of the twitter account that the bot is hosted on. Consumer should be the key and secret of a twitter app that you have registered and the token should be the key and secret of the twitter account for that app. See https://dev.twitter.com/pages/oauth_single_token for info on getting a single access token and avoiding the oauth flow. 9 | 10 | Example crontab entry: 11 | 12 | * * * * * cd /whatevs/markovator && python main.py >> markovator.log 2>&1 13 | 14 | 15 | -------------------------------------------------------------------------------- /example_twitter_settings.py: -------------------------------------------------------------------------------- 1 | # Fill in the blanks below and rename this file twitter_settings.py 2 | 3 | import oauth2 as oauth 4 | 5 | screen_name='' # Twitter username 6 | token = oauth.Token(key="", secret="") # Twitter users token and secret 7 | consumer = oauth.Consumer(key="", secret="") # Key and secret of the twitter appliction 8 | -------------------------------------------------------------------------------- /file_system_status.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | 4 | LOCATION = "status.json" 5 | 6 | def clear(): 7 | f = open(LOCATION, 'w') 8 | f.write('{}\n') 9 | f.close() 10 | 11 | 12 | def load(): 13 | try: 14 | f = open(LOCATION,'r') 15 | json_data = f.read() 16 | f.close() 17 | except IOError: 18 | json_data = '{}' 19 | return json.loads(json_data) 20 | 21 | def save(app_status): 22 | f = open(LOCATION, 'w') 23 | f.write(json.dumps(app_status)) 24 | f.close() 25 | 26 | 27 | -------------------------------------------------------------------------------- /httplib2/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import generators 2 | """ 3 | httplib2 4 | 5 | A caching http interface that supports ETags and gzip 6 | to conserve bandwidth. 7 | 8 | Requires Python 2.3 or later 9 | 10 | Changelog: 11 | 2007-08-18, Rick: Modified so it's able to use a socks proxy if needed. 12 | 13 | """ 14 | 15 | __author__ = "Joe Gregorio (joe@bitworking.org)" 16 | __copyright__ = "Copyright 2006, Joe Gregorio" 17 | __contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", 18 | "James Antill", 19 | "Xavier Verges Farrero", 20 | "Jonathan Feinberg", 21 | "Blair Zajac", 22 | "Sam Ruby", 23 | "Louis Nyffenegger"] 24 | __license__ = "MIT" 25 | __version__ = "$Rev$" 26 | 27 | import re 28 | import sys 29 | import email 30 | import email.Utils 31 | import email.Message 32 | import email.FeedParser 33 | import StringIO 34 | import gzip 35 | import zlib 36 | import httplib 37 | import urlparse 38 | import base64 39 | import os 40 | import copy 41 | import calendar 42 | import time 43 | import random 44 | # remove depracated warning in python2.6 45 | try: 46 | from hashlib import sha1 as _sha, md5 as _md5 47 | except ImportError: 48 | import sha 49 | import md5 50 | _sha = sha.new 51 | _md5 = md5.new 52 | import hmac 53 | from gettext import gettext as _ 54 | import socket 55 | 56 | try: 57 | import socks 58 | except ImportError: 59 | socks = None 60 | 61 | # Build the appropriate socket wrapper for ssl 62 | try: 63 | import ssl # python 2.6 64 | _ssl_wrap_socket = ssl.wrap_socket 65 | except ImportError: 66 | def _ssl_wrap_socket(sock, key_file, cert_file): 67 | ssl_sock = socket.ssl(sock, key_file, cert_file) 68 | return httplib.FakeSocket(sock, ssl_sock) 69 | 70 | 71 | if sys.version_info >= (2,3): 72 | from iri2uri import iri2uri 73 | else: 74 | def iri2uri(uri): 75 | return uri 76 | 77 | def has_timeout(timeout): # python 2.6 78 | if hasattr(socket, '_GLOBAL_DEFAULT_TIMEOUT'): 79 | return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT) 80 | return (timeout is not None) 81 | 82 | __all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error', 83 | 'RedirectMissingLocation', 'RedirectLimit', 'FailedToDecompressContent', 84 | 'UnimplementedDigestAuthOptionError', 'UnimplementedHmacDigestAuthOptionError', 85 | 'debuglevel'] 86 | 87 | 88 | # The httplib debug level, set to a non-zero value to get debug output 89 | debuglevel = 0 90 | 91 | 92 | # Python 2.3 support 93 | if sys.version_info < (2,4): 94 | def sorted(seq): 95 | seq.sort() 96 | return seq 97 | 98 | # Python 2.3 support 99 | def HTTPResponse__getheaders(self): 100 | """Return list of (header, value) tuples.""" 101 | if self.msg is None: 102 | raise httplib.ResponseNotReady() 103 | return self.msg.items() 104 | 105 | if not hasattr(httplib.HTTPResponse, 'getheaders'): 106 | httplib.HTTPResponse.getheaders = HTTPResponse__getheaders 107 | 108 | # All exceptions raised here derive from HttpLib2Error 109 | class HttpLib2Error(Exception): pass 110 | 111 | # Some exceptions can be caught and optionally 112 | # be turned back into responses. 113 | class HttpLib2ErrorWithResponse(HttpLib2Error): 114 | def __init__(self, desc, response, content): 115 | self.response = response 116 | self.content = content 117 | HttpLib2Error.__init__(self, desc) 118 | 119 | class RedirectMissingLocation(HttpLib2ErrorWithResponse): pass 120 | class RedirectLimit(HttpLib2ErrorWithResponse): pass 121 | class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass 122 | class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass 123 | class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass 124 | 125 | class RelativeURIError(HttpLib2Error): pass 126 | class ServerNotFoundError(HttpLib2Error): pass 127 | 128 | # Open Items: 129 | # ----------- 130 | # Proxy support 131 | 132 | # Are we removing the cached content too soon on PUT (only delete on 200 Maybe?) 133 | 134 | # Pluggable cache storage (supports storing the cache in 135 | # flat files by default. We need a plug-in architecture 136 | # that can support Berkeley DB and Squid) 137 | 138 | # == Known Issues == 139 | # Does not handle a resource that uses conneg and Last-Modified but no ETag as a cache validator. 140 | # Does not handle Cache-Control: max-stale 141 | # Does not use Age: headers when calculating cache freshness. 142 | 143 | 144 | # The number of redirections to follow before giving up. 145 | # Note that only GET redirects are automatically followed. 146 | # Will also honor 301 requests by saving that info and never 147 | # requesting that URI again. 148 | DEFAULT_MAX_REDIRECTS = 5 149 | 150 | # Which headers are hop-by-hop headers by default 151 | HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade'] 152 | 153 | def _get_end2end_headers(response): 154 | hopbyhop = list(HOP_BY_HOP) 155 | hopbyhop.extend([x.strip() for x in response.get('connection', '').split(',')]) 156 | return [header for header in response.keys() if header not in hopbyhop] 157 | 158 | URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") 159 | 160 | def parse_uri(uri): 161 | """Parses a URI using the regex given in Appendix B of RFC 3986. 162 | 163 | (scheme, authority, path, query, fragment) = parse_uri(uri) 164 | """ 165 | groups = URI.match(uri).groups() 166 | return (groups[1], groups[3], groups[4], groups[6], groups[8]) 167 | 168 | def urlnorm(uri): 169 | (scheme, authority, path, query, fragment) = parse_uri(uri) 170 | if not scheme or not authority: 171 | raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri) 172 | authority = authority.lower() 173 | scheme = scheme.lower() 174 | if not path: 175 | path = "/" 176 | # Could do syntax based normalization of the URI before 177 | # computing the digest. See Section 6.2.2 of Std 66. 178 | request_uri = query and "?".join([path, query]) or path 179 | scheme = scheme.lower() 180 | defrag_uri = scheme + "://" + authority + request_uri 181 | return scheme, authority, request_uri, defrag_uri 182 | 183 | 184 | # Cache filename construction (original borrowed from Venus http://intertwingly.net/code/venus/) 185 | re_url_scheme = re.compile(r'^\w+://') 186 | re_slash = re.compile(r'[?/:|]+') 187 | 188 | def safename(filename): 189 | """Return a filename suitable for the cache. 190 | 191 | Strips dangerous and common characters to create a filename we 192 | can use to store the cache in. 193 | """ 194 | 195 | try: 196 | if re_url_scheme.match(filename): 197 | if isinstance(filename,str): 198 | filename = filename.decode('utf-8') 199 | filename = filename.encode('idna') 200 | else: 201 | filename = filename.encode('idna') 202 | except UnicodeError: 203 | pass 204 | if isinstance(filename,unicode): 205 | filename=filename.encode('utf-8') 206 | filemd5 = _md5(filename).hexdigest() 207 | filename = re_url_scheme.sub("", filename) 208 | filename = re_slash.sub(",", filename) 209 | 210 | # limit length of filename 211 | if len(filename)>200: 212 | filename=filename[:200] 213 | return ",".join((filename, filemd5)) 214 | 215 | NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') 216 | def _normalize_headers(headers): 217 | return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) 218 | 219 | def _parse_cache_control(headers): 220 | retval = {} 221 | if headers.has_key('cache-control'): 222 | parts = headers['cache-control'].split(',') 223 | parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] 224 | parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] 225 | retval = dict(parts_with_args + parts_wo_args) 226 | return retval 227 | 228 | # Whether to use a strict mode to parse WWW-Authenticate headers 229 | # Might lead to bad results in case of ill-formed header value, 230 | # so disabled by default, falling back to relaxed parsing. 231 | # Set to true to turn on, usefull for testing servers. 232 | USE_WWW_AUTH_STRICT_PARSING = 0 233 | 234 | # In regex below: 235 | # [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP 236 | # "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space 237 | # Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: 238 | # \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"? 239 | WWW_AUTH_STRICT = re.compile(r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$") 240 | WWW_AUTH_RELAXED = re.compile(r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(? current_age: 340 | retval = "FRESH" 341 | return retval 342 | 343 | def _decompressContent(response, new_content): 344 | content = new_content 345 | try: 346 | encoding = response.get('content-encoding', None) 347 | if encoding in ['gzip', 'deflate']: 348 | if encoding == 'gzip': 349 | content = gzip.GzipFile(fileobj=StringIO.StringIO(new_content)).read() 350 | if encoding == 'deflate': 351 | content = zlib.decompress(content) 352 | response['content-length'] = str(len(content)) 353 | # Record the historical presence of the encoding in a way the won't interfere. 354 | response['-content-encoding'] = response['content-encoding'] 355 | del response['content-encoding'] 356 | except IOError: 357 | content = "" 358 | raise FailedToDecompressContent(_("Content purported to be compressed with %s but failed to decompress.") % response.get('content-encoding'), response, content) 359 | return content 360 | 361 | def _updateCache(request_headers, response_headers, content, cache, cachekey): 362 | if cachekey: 363 | cc = _parse_cache_control(request_headers) 364 | cc_response = _parse_cache_control(response_headers) 365 | if cc.has_key('no-store') or cc_response.has_key('no-store'): 366 | cache.delete(cachekey) 367 | else: 368 | info = email.Message.Message() 369 | for key, value in response_headers.iteritems(): 370 | if key not in ['status','content-encoding','transfer-encoding']: 371 | info[key] = value 372 | 373 | # Add annotations to the cache to indicate what headers 374 | # are variant for this request. 375 | vary = response_headers.get('vary', None) 376 | if vary: 377 | vary_headers = vary.lower().replace(' ', '').split(',') 378 | for header in vary_headers: 379 | key = '-varied-%s' % header 380 | try: 381 | info[key] = request_headers[header] 382 | except KeyError: 383 | pass 384 | 385 | status = response_headers.status 386 | if status == 304: 387 | status = 200 388 | 389 | status_header = 'status: %d\r\n' % response_headers.status 390 | 391 | header_str = info.as_string() 392 | 393 | header_str = re.sub("\r(?!\n)|(? 0: 610 | service = "cl" 611 | # No point in guessing Base or Spreadsheet 612 | #elif request_uri.find("spreadsheets") > 0: 613 | # service = "wise" 614 | 615 | auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers['user-agent']) 616 | resp, content = self.http.request("https://www.google.com/accounts/ClientLogin", method="POST", body=urlencode(auth), headers={'Content-Type': 'application/x-www-form-urlencoded'}) 617 | lines = content.split('\n') 618 | d = dict([tuple(line.split("=", 1)) for line in lines if line]) 619 | if resp.status == 403: 620 | self.Auth = "" 621 | else: 622 | self.Auth = d['Auth'] 623 | 624 | def request(self, method, request_uri, headers, content): 625 | """Modify the request headers to add the appropriate 626 | Authorization header.""" 627 | headers['authorization'] = 'GoogleLogin Auth=' + self.Auth 628 | 629 | 630 | AUTH_SCHEME_CLASSES = { 631 | "basic": BasicAuthentication, 632 | "wsse": WsseAuthentication, 633 | "digest": DigestAuthentication, 634 | "hmacdigest": HmacDigestAuthentication, 635 | "googlelogin": GoogleLoginAuthentication 636 | } 637 | 638 | AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"] 639 | 640 | class FileCache(object): 641 | """Uses a local directory as a store for cached files. 642 | Not really safe to use if multiple threads or processes are going to 643 | be running on the same cache. 644 | """ 645 | def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior 646 | self.cache = cache 647 | self.safe = safe 648 | if not os.path.exists(cache): 649 | os.makedirs(self.cache) 650 | 651 | def get(self, key): 652 | retval = None 653 | cacheFullPath = os.path.join(self.cache, self.safe(key)) 654 | try: 655 | f = file(cacheFullPath, "rb") 656 | retval = f.read() 657 | f.close() 658 | except IOError: 659 | pass 660 | return retval 661 | 662 | def set(self, key, value): 663 | cacheFullPath = os.path.join(self.cache, self.safe(key)) 664 | f = file(cacheFullPath, "wb") 665 | f.write(value) 666 | f.close() 667 | 668 | def delete(self, key): 669 | cacheFullPath = os.path.join(self.cache, self.safe(key)) 670 | if os.path.exists(cacheFullPath): 671 | os.remove(cacheFullPath) 672 | 673 | class Credentials(object): 674 | def __init__(self): 675 | self.credentials = [] 676 | 677 | def add(self, name, password, domain=""): 678 | self.credentials.append((domain.lower(), name, password)) 679 | 680 | def clear(self): 681 | self.credentials = [] 682 | 683 | def iter(self, domain): 684 | for (cdomain, name, password) in self.credentials: 685 | if cdomain == "" or domain == cdomain: 686 | yield (name, password) 687 | 688 | class KeyCerts(Credentials): 689 | """Identical to Credentials except that 690 | name/password are mapped to key/cert.""" 691 | pass 692 | 693 | 694 | class ProxyInfo(object): 695 | """Collect information required to use a proxy.""" 696 | def __init__(self, proxy_type, proxy_host, proxy_port, proxy_rdns=None, proxy_user=None, proxy_pass=None): 697 | """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX 698 | constants. For example: 699 | 700 | p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, proxy_host='localhost', proxy_port=8000) 701 | """ 702 | self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass = proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass 703 | 704 | def astuple(self): 705 | return (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, 706 | self.proxy_user, self.proxy_pass) 707 | 708 | def isgood(self): 709 | return socks and (self.proxy_host != None) and (self.proxy_port != None) 710 | 711 | 712 | class HTTPConnectionWithTimeout(httplib.HTTPConnection): 713 | """HTTPConnection subclass that supports timeouts""" 714 | 715 | def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None): 716 | httplib.HTTPConnection.__init__(self, host, port, strict) 717 | self.timeout = timeout 718 | self.proxy_info = proxy_info 719 | 720 | def connect(self): 721 | """Connect to the host and port specified in __init__.""" 722 | # Mostly verbatim from httplib.py. 723 | msg = "getaddrinfo returns an empty list" 724 | for res in socket.getaddrinfo(self.host, self.port, 0, 725 | socket.SOCK_STREAM): 726 | af, socktype, proto, canonname, sa = res 727 | try: 728 | if self.proxy_info and self.proxy_info.isgood(): 729 | self.sock = socks.socksocket(af, socktype, proto) 730 | self.sock.setproxy(*self.proxy_info.astuple()) 731 | else: 732 | self.sock = socket.socket(af, socktype, proto) 733 | # Different from httplib: support timeouts. 734 | if has_timeout(self.timeout): 735 | self.sock.settimeout(self.timeout) 736 | # End of difference from httplib. 737 | if self.debuglevel > 0: 738 | print "connect: (%s, %s)" % (self.host, self.port) 739 | 740 | self.sock.connect(sa) 741 | except socket.error, msg: 742 | if self.debuglevel > 0: 743 | print 'connect fail:', (self.host, self.port) 744 | if self.sock: 745 | self.sock.close() 746 | self.sock = None 747 | continue 748 | break 749 | if not self.sock: 750 | raise socket.error, msg 751 | 752 | class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): 753 | "This class allows communication via SSL." 754 | 755 | def __init__(self, host, port=None, key_file=None, cert_file=None, 756 | strict=None, timeout=None, proxy_info=None): 757 | httplib.HTTPSConnection.__init__(self, host, port=port, key_file=key_file, 758 | cert_file=cert_file, strict=strict) 759 | self.timeout = timeout 760 | self.proxy_info = proxy_info 761 | 762 | def connect(self): 763 | "Connect to a host on a given (SSL) port." 764 | 765 | if self.proxy_info and self.proxy_info.isgood(): 766 | sock = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) 767 | sock.setproxy(*self.proxy_info.astuple()) 768 | else: 769 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 770 | 771 | if has_timeout(self.timeout): 772 | sock.settimeout(self.timeout) 773 | sock.connect((self.host, self.port)) 774 | self.sock =_ssl_wrap_socket(sock, self.key_file, self.cert_file) 775 | 776 | 777 | 778 | class Http(object): 779 | """An HTTP client that handles: 780 | - all methods 781 | - caching 782 | - ETags 783 | - compression, 784 | - HTTPS 785 | - Basic 786 | - Digest 787 | - WSSE 788 | 789 | and more. 790 | """ 791 | def __init__(self, cache=None, timeout=None, proxy_info=None): 792 | """The value of proxy_info is a ProxyInfo instance. 793 | 794 | If 'cache' is a string then it is used as a directory name 795 | for a disk cache. Otherwise it must be an object that supports 796 | the same interface as FileCache.""" 797 | self.proxy_info = proxy_info 798 | # Map domain name to an httplib connection 799 | self.connections = {} 800 | # The location of the cache, for now a directory 801 | # where cached responses are held. 802 | if cache and isinstance(cache, str): 803 | self.cache = FileCache(cache) 804 | else: 805 | self.cache = cache 806 | 807 | # Name/password 808 | self.credentials = Credentials() 809 | 810 | # Key/cert 811 | self.certificates = KeyCerts() 812 | 813 | # authorization objects 814 | self.authorizations = [] 815 | 816 | # If set to False then no redirects are followed, even safe ones. 817 | self.follow_redirects = True 818 | 819 | # Which HTTP methods do we apply optimistic concurrency to, i.e. 820 | # which methods get an "if-match:" etag header added to them. 821 | self.optimistic_concurrency_methods = ["PUT"] 822 | 823 | # If 'follow_redirects' is True, and this is set to True then 824 | # all redirecs are followed, including unsafe ones. 825 | self.follow_all_redirects = False 826 | 827 | self.ignore_etag = False 828 | 829 | self.force_exception_to_status_code = False 830 | 831 | self.timeout = timeout 832 | 833 | def _auth_from_challenge(self, host, request_uri, headers, response, content): 834 | """A generator that creates Authorization objects 835 | that can be applied to requests. 836 | """ 837 | challenges = _parse_www_authenticate(response, 'www-authenticate') 838 | for cred in self.credentials.iter(host): 839 | for scheme in AUTH_SCHEME_ORDER: 840 | if challenges.has_key(scheme): 841 | yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self) 842 | 843 | def add_credentials(self, name, password, domain=""): 844 | """Add a name and password that will be used 845 | any time a request requires authentication.""" 846 | self.credentials.add(name, password, domain) 847 | 848 | def add_certificate(self, key, cert, domain): 849 | """Add a key and cert that will be used 850 | any time a request requires authentication.""" 851 | self.certificates.add(key, cert, domain) 852 | 853 | def clear_credentials(self): 854 | """Remove all the names and passwords 855 | that are used for authentication""" 856 | self.credentials.clear() 857 | self.authorizations = [] 858 | 859 | def _conn_request(self, conn, request_uri, method, body, headers): 860 | for i in range(2): 861 | try: 862 | conn.request(method, request_uri, body, headers) 863 | except socket.gaierror: 864 | conn.close() 865 | raise ServerNotFoundError("Unable to find the server at %s" % conn.host) 866 | except (socket.error, httplib.HTTPException): 867 | # Just because the server closed the connection doesn't apparently mean 868 | # that the server didn't send a response. 869 | pass 870 | try: 871 | response = conn.getresponse() 872 | except (socket.error, httplib.HTTPException): 873 | if i == 0: 874 | conn.close() 875 | conn.connect() 876 | continue 877 | else: 878 | raise 879 | else: 880 | content = "" 881 | if method == "HEAD": 882 | response.close() 883 | else: 884 | content = response.read() 885 | response = Response(response) 886 | if method != "HEAD": 887 | content = _decompressContent(response, content) 888 | break 889 | return (response, content) 890 | 891 | 892 | def _request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey): 893 | """Do the actual request using the connection object 894 | and also follow one level of redirects if necessary""" 895 | 896 | auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] 897 | auth = auths and sorted(auths)[0][1] or None 898 | if auth: 899 | auth.request(method, request_uri, headers, body) 900 | 901 | (response, content) = self._conn_request(conn, request_uri, method, body, headers) 902 | 903 | if auth: 904 | if auth.response(response, body): 905 | auth.request(method, request_uri, headers, body) 906 | (response, content) = self._conn_request(conn, request_uri, method, body, headers ) 907 | response._stale_digest = 1 908 | 909 | if response.status == 401: 910 | for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): 911 | authorization.request(method, request_uri, headers, body) 912 | (response, content) = self._conn_request(conn, request_uri, method, body, headers, ) 913 | if response.status != 401: 914 | self.authorizations.append(authorization) 915 | authorization.response(response, body) 916 | break 917 | 918 | if (self.follow_all_redirects or (method in ["GET", "HEAD"]) or response.status == 303): 919 | if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: 920 | # Pick out the location header and basically start from the beginning 921 | # remembering first to strip the ETag header and decrement our 'depth' 922 | if redirections: 923 | if not response.has_key('location') and response.status != 300: 924 | raise RedirectMissingLocation( _("Redirected but the response is missing a Location: header."), response, content) 925 | # Fix-up relative redirects (which violate an RFC 2616 MUST) 926 | if response.has_key('location'): 927 | location = response['location'] 928 | (scheme, authority, path, query, fragment) = parse_uri(location) 929 | if authority == None: 930 | response['location'] = urlparse.urljoin(absolute_uri, location) 931 | if response.status == 301 and method in ["GET", "HEAD"]: 932 | response['-x-permanent-redirect-url'] = response['location'] 933 | if not response.has_key('content-location'): 934 | response['content-location'] = absolute_uri 935 | _updateCache(headers, response, content, self.cache, cachekey) 936 | if headers.has_key('if-none-match'): 937 | del headers['if-none-match'] 938 | if headers.has_key('if-modified-since'): 939 | del headers['if-modified-since'] 940 | if response.has_key('location'): 941 | location = response['location'] 942 | old_response = copy.deepcopy(response) 943 | if not old_response.has_key('content-location'): 944 | old_response['content-location'] = absolute_uri 945 | redirect_method = ((response.status == 303) and (method not in ["GET", "HEAD"])) and "GET" or method 946 | (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) 947 | response.previous = old_response 948 | else: 949 | raise RedirectLimit( _("Redirected more times than rediection_limit allows."), response, content) 950 | elif response.status in [200, 203] and method == "GET": 951 | # Don't cache 206's since we aren't going to handle byte range requests 952 | if not response.has_key('content-location'): 953 | response['content-location'] = absolute_uri 954 | _updateCache(headers, response, content, self.cache, cachekey) 955 | 956 | return (response, content) 957 | 958 | def _normalize_headers(self, headers): 959 | return _normalize_headers(headers) 960 | 961 | # Need to catch and rebrand some exceptions 962 | # Then need to optionally turn all exceptions into status codes 963 | # including all socket.* and httplib.* exceptions. 964 | 965 | 966 | def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None): 967 | """ Performs a single HTTP request. 968 | The 'uri' is the URI of the HTTP resource and can begin 969 | with either 'http' or 'https'. The value of 'uri' must be an absolute URI. 970 | 971 | The 'method' is the HTTP method to perform, such as GET, POST, DELETE, etc. 972 | There is no restriction on the methods allowed. 973 | 974 | The 'body' is the entity body to be sent with the request. It is a string 975 | object. 976 | 977 | Any extra headers that are to be sent with the request should be provided in the 978 | 'headers' dictionary. 979 | 980 | The maximum number of redirect to follow before raising an 981 | exception is 'redirections. The default is 5. 982 | 983 | The return value is a tuple of (response, content), the first 984 | being and instance of the 'Response' class, the second being 985 | a string that contains the response entity body. 986 | """ 987 | try: 988 | if headers is None: 989 | headers = {} 990 | else: 991 | headers = self._normalize_headers(headers) 992 | 993 | if not headers.has_key('user-agent'): 994 | headers['user-agent'] = "Python-httplib2/%s" % __version__ 995 | 996 | uri = iri2uri(uri) 997 | 998 | (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) 999 | domain_port = authority.split(":")[0:2] 1000 | if len(domain_port) == 2 and domain_port[1] == '443' and scheme == 'http': 1001 | scheme = 'https' 1002 | authority = domain_port[0] 1003 | 1004 | conn_key = scheme+":"+authority 1005 | if conn_key in self.connections: 1006 | conn = self.connections[conn_key] 1007 | else: 1008 | if not connection_type: 1009 | connection_type = (scheme == 'https') and HTTPSConnectionWithTimeout or HTTPConnectionWithTimeout 1010 | certs = list(self.certificates.iter(authority)) 1011 | if scheme == 'https' and certs: 1012 | conn = self.connections[conn_key] = connection_type(authority, key_file=certs[0][0], 1013 | cert_file=certs[0][1], timeout=self.timeout, proxy_info=self.proxy_info) 1014 | else: 1015 | conn = self.connections[conn_key] = connection_type(authority, timeout=self.timeout, proxy_info=self.proxy_info) 1016 | conn.set_debuglevel(debuglevel) 1017 | 1018 | if method in ["GET", "HEAD"] and 'range' not in headers and 'accept-encoding' not in headers: 1019 | headers['accept-encoding'] = 'gzip, deflate' 1020 | 1021 | info = email.Message.Message() 1022 | cached_value = None 1023 | if self.cache: 1024 | cachekey = defrag_uri 1025 | cached_value = self.cache.get(cachekey) 1026 | if cached_value: 1027 | # info = email.message_from_string(cached_value) 1028 | # 1029 | # Need to replace the line above with the kludge below 1030 | # to fix the non-existent bug not fixed in this 1031 | # bug report: http://mail.python.org/pipermail/python-bugs-list/2005-September/030289.html 1032 | try: 1033 | info, content = cached_value.split('\r\n\r\n', 1) 1034 | feedparser = email.FeedParser.FeedParser() 1035 | feedparser.feed(info) 1036 | info = feedparser.close() 1037 | feedparser._parse = None 1038 | except IndexError: 1039 | self.cache.delete(cachekey) 1040 | cachekey = None 1041 | cached_value = None 1042 | else: 1043 | cachekey = None 1044 | 1045 | if method in self.optimistic_concurrency_methods and self.cache and info.has_key('etag') and not self.ignore_etag and 'if-match' not in headers: 1046 | # http://www.w3.org/1999/04/Editing/ 1047 | headers['if-match'] = info['etag'] 1048 | 1049 | if method not in ["GET", "HEAD"] and self.cache and cachekey: 1050 | # RFC 2616 Section 13.10 1051 | self.cache.delete(cachekey) 1052 | 1053 | # Check the vary header in the cache to see if this request 1054 | # matches what varies in the cache. 1055 | if method in ['GET', 'HEAD'] and 'vary' in info: 1056 | vary = info['vary'] 1057 | vary_headers = vary.lower().replace(' ', '').split(',') 1058 | for header in vary_headers: 1059 | key = '-varied-%s' % header 1060 | value = info[key] 1061 | if headers.get(header, '') != value: 1062 | cached_value = None 1063 | break 1064 | 1065 | if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers: 1066 | if info.has_key('-x-permanent-redirect-url'): 1067 | # Should cached permanent redirects be counted in our redirection count? For now, yes. 1068 | (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) 1069 | response.previous = Response(info) 1070 | response.previous.fromcache = True 1071 | else: 1072 | # Determine our course of action: 1073 | # Is the cached entry fresh or stale? 1074 | # Has the client requested a non-cached response? 1075 | # 1076 | # There seems to be three possible answers: 1077 | # 1. [FRESH] Return the cache entry w/o doing a GET 1078 | # 2. [STALE] Do the GET (but add in cache validators if available) 1079 | # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request 1080 | entry_disposition = _entry_disposition(info, headers) 1081 | 1082 | if entry_disposition == "FRESH": 1083 | if not cached_value: 1084 | info['status'] = '504' 1085 | content = "" 1086 | response = Response(info) 1087 | if cached_value: 1088 | response.fromcache = True 1089 | return (response, content) 1090 | 1091 | if entry_disposition == "STALE": 1092 | if info.has_key('etag') and not self.ignore_etag and not 'if-none-match' in headers: 1093 | headers['if-none-match'] = info['etag'] 1094 | if info.has_key('last-modified') and not 'last-modified' in headers: 1095 | headers['if-modified-since'] = info['last-modified'] 1096 | elif entry_disposition == "TRANSPARENT": 1097 | pass 1098 | 1099 | (response, new_content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) 1100 | 1101 | if response.status == 304 and method == "GET": 1102 | # Rewrite the cache entry with the new end-to-end headers 1103 | # Take all headers that are in response 1104 | # and overwrite their values in info. 1105 | # unless they are hop-by-hop, or are listed in the connection header. 1106 | 1107 | for key in _get_end2end_headers(response): 1108 | info[key] = response[key] 1109 | merged_response = Response(info) 1110 | if hasattr(response, "_stale_digest"): 1111 | merged_response._stale_digest = response._stale_digest 1112 | _updateCache(headers, merged_response, content, self.cache, cachekey) 1113 | response = merged_response 1114 | response.status = 200 1115 | response.fromcache = True 1116 | 1117 | elif response.status == 200: 1118 | content = new_content 1119 | else: 1120 | self.cache.delete(cachekey) 1121 | content = new_content 1122 | else: 1123 | cc = _parse_cache_control(headers) 1124 | if cc.has_key('only-if-cached'): 1125 | info['status'] = '504' 1126 | response = Response(info) 1127 | content = "" 1128 | else: 1129 | (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) 1130 | except Exception, e: 1131 | if self.force_exception_to_status_code: 1132 | if isinstance(e, HttpLib2ErrorWithResponse): 1133 | response = e.response 1134 | content = e.content 1135 | response.status = 500 1136 | response.reason = str(e) 1137 | elif isinstance(e, socket.timeout): 1138 | content = "Request Timeout" 1139 | response = Response( { 1140 | "content-type": "text/plain", 1141 | "status": "408", 1142 | "content-length": len(content) 1143 | }) 1144 | response.reason = "Request Timeout" 1145 | else: 1146 | content = str(e) 1147 | response = Response( { 1148 | "content-type": "text/plain", 1149 | "status": "400", 1150 | "content-length": len(content) 1151 | }) 1152 | response.reason = "Bad Request" 1153 | else: 1154 | raise 1155 | 1156 | 1157 | return (response, content) 1158 | 1159 | 1160 | 1161 | class Response(dict): 1162 | """An object more like email.Message than httplib.HTTPResponse.""" 1163 | 1164 | """Is this response from our local cache""" 1165 | fromcache = False 1166 | 1167 | """HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. """ 1168 | version = 11 1169 | 1170 | "Status code returned by server. " 1171 | status = 200 1172 | 1173 | """Reason phrase returned by server.""" 1174 | reason = "Ok" 1175 | 1176 | previous = None 1177 | 1178 | def __init__(self, info): 1179 | # info is either an email.Message or 1180 | # an httplib.HTTPResponse object. 1181 | if isinstance(info, httplib.HTTPResponse): 1182 | for key, value in info.getheaders(): 1183 | self[key.lower()] = value 1184 | self.status = info.status 1185 | self['status'] = str(self.status) 1186 | self.reason = info.reason 1187 | self.version = info.version 1188 | elif isinstance(info, email.Message.Message): 1189 | for key, value in info.items(): 1190 | self[key] = value 1191 | self.status = int(self['status']) 1192 | else: 1193 | for key, value in info.iteritems(): 1194 | self[key] = value 1195 | self.status = int(self.get('status', self.status)) 1196 | 1197 | 1198 | def __getattr__(self, name): 1199 | if name == 'dict': 1200 | return self 1201 | else: 1202 | raise AttributeError, name 1203 | -------------------------------------------------------------------------------- /httplib2/iri2uri.py: -------------------------------------------------------------------------------- 1 | """ 2 | iri2uri 3 | 4 | Converts an IRI to a URI. 5 | 6 | """ 7 | __author__ = "Joe Gregorio (joe@bitworking.org)" 8 | __copyright__ = "Copyright 2006, Joe Gregorio" 9 | __contributors__ = [] 10 | __version__ = "1.0.0" 11 | __license__ = "MIT" 12 | __history__ = """ 13 | """ 14 | 15 | import urlparse 16 | 17 | 18 | # Convert an IRI to a URI following the rules in RFC 3987 19 | # 20 | # The characters we need to enocde and escape are defined in the spec: 21 | # 22 | # iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD 23 | # ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF 24 | # / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD 25 | # / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD 26 | # / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD 27 | # / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD 28 | # / %xD0000-DFFFD / %xE1000-EFFFD 29 | 30 | escape_range = [ 31 | (0xA0, 0xD7FF ), 32 | (0xE000, 0xF8FF ), 33 | (0xF900, 0xFDCF ), 34 | (0xFDF0, 0xFFEF), 35 | (0x10000, 0x1FFFD ), 36 | (0x20000, 0x2FFFD ), 37 | (0x30000, 0x3FFFD), 38 | (0x40000, 0x4FFFD ), 39 | (0x50000, 0x5FFFD ), 40 | (0x60000, 0x6FFFD), 41 | (0x70000, 0x7FFFD ), 42 | (0x80000, 0x8FFFD ), 43 | (0x90000, 0x9FFFD), 44 | (0xA0000, 0xAFFFD ), 45 | (0xB0000, 0xBFFFD ), 46 | (0xC0000, 0xCFFFD), 47 | (0xD0000, 0xDFFFD ), 48 | (0xE1000, 0xEFFFD), 49 | (0xF0000, 0xFFFFD ), 50 | (0x100000, 0x10FFFD) 51 | ] 52 | 53 | def encode(c): 54 | retval = c 55 | i = ord(c) 56 | for low, high in escape_range: 57 | if i < low: 58 | break 59 | if i >= low and i <= high: 60 | retval = "".join(["%%%2X" % ord(o) for o in c.encode('utf-8')]) 61 | break 62 | return retval 63 | 64 | 65 | def iri2uri(uri): 66 | """Convert an IRI to a URI. Note that IRIs must be 67 | passed in a unicode strings. That is, do not utf-8 encode 68 | the IRI before passing it into the function.""" 69 | if isinstance(uri ,unicode): 70 | (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri) 71 | authority = authority.encode('idna') 72 | # For each character in 'ucschar' or 'iprivate' 73 | # 1. encode as utf-8 74 | # 2. then %-encode each octet of that utf-8 75 | uri = urlparse.urlunsplit((scheme, authority, path, query, fragment)) 76 | uri = "".join([encode(c) for c in uri]) 77 | return uri 78 | 79 | if __name__ == "__main__": 80 | import unittest 81 | 82 | class Test(unittest.TestCase): 83 | 84 | def test_uris(self): 85 | """Test that URIs are invariant under the transformation.""" 86 | invariant = [ 87 | u"ftp://ftp.is.co.za/rfc/rfc1808.txt", 88 | u"http://www.ietf.org/rfc/rfc2396.txt", 89 | u"ldap://[2001:db8::7]/c=GB?objectClass?one", 90 | u"mailto:John.Doe@example.com", 91 | u"news:comp.infosystems.www.servers.unix", 92 | u"tel:+1-816-555-1212", 93 | u"telnet://192.0.2.16:80/", 94 | u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2" ] 95 | for uri in invariant: 96 | self.assertEqual(uri, iri2uri(uri)) 97 | 98 | def test_iri(self): 99 | """ Test that the right type of escaping is done for each part of the URI.""" 100 | self.assertEqual("http://xn--o3h.com/%E2%98%84", iri2uri(u"http://\N{COMET}.com/\N{COMET}")) 101 | self.assertEqual("http://bitworking.org/?fred=%E2%98%84", iri2uri(u"http://bitworking.org/?fred=\N{COMET}")) 102 | self.assertEqual("http://bitworking.org/#%E2%98%84", iri2uri(u"http://bitworking.org/#\N{COMET}")) 103 | self.assertEqual("#%E2%98%84", iri2uri(u"#\N{COMET}")) 104 | self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}")) 105 | self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}"))) 106 | self.assertNotEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}".encode('utf-8'))) 107 | 108 | unittest.main() 109 | 110 | 111 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import twitter 4 | import file_system_status as status 5 | from markovate import Markovator 6 | 7 | import twitter_settings 8 | import sys 9 | import random 10 | 11 | def create_markovated_tweet(tweets, max_length, unwanted_markovations=[]): 12 | tweets_texts = map(lambda t: t['text'].strip(), tweets) 13 | markovator = Markovator() 14 | markovator.parse_sentences(tweets_texts) 15 | markovation = markovator.markovate() 16 | 17 | unwanted_markovations.extend(tweets_texts) 18 | 19 | count = 0 20 | while len(markovation) > max_length or markovation in unwanted_markovations: 21 | markovation = markovator.markovate() 22 | count += 1 23 | if count > 20: 24 | return None 25 | 26 | return markovation 27 | 28 | def filter_tweets(tweets): 29 | return filter_out_mentions(filter_out_links(filter_out_bad_words(tweets))) 30 | 31 | def filter_out_mentions(tweets): 32 | # TODO This is to be polite, we could keep tweets that mention people that follow us 33 | return filter(lambda t:not '@' in t['text'], tweets) 34 | 35 | def filter_out_links(tweets): 36 | # Links are almost guaranteed to ruin the context of the markovation 37 | return filter(lambda t:not 'http://' in t['text'].lower(), tweets) 38 | 39 | def filter_out_bad_words(tweets): 40 | # Might be offensive/inappropriate for humour 41 | return filter(lambda t:not ('cancer' in t['text'].lower() or 42 | 'r.i.p' in t['text'].lower() or 43 | 'RIP' in t['text']), tweets) 44 | 45 | def reply_to_user(user, app_status): 46 | if user['protected']: 47 | print("@" + user['screen_name'] + " sorry, I can't process protected users :(") 48 | return 49 | 50 | screen_name = user['screen_name'] 51 | 52 | print(screen_name) 53 | 54 | tweets = filter_tweets(twitter.get_tweets(screen_name, True)) 55 | 56 | if len(tweets) <= 1: 57 | print("Not enough tweets") 58 | fail_reply = "@" + screen_name + " sorry, you need to tweet more (or tweet less @ mentions and links) :(" 59 | twitter.post_tweet(fail_reply) 60 | app_status['latest_reply'] = fail_reply 61 | return 62 | 63 | tweet_prefix = '@' + screen_name + ' markovated: ' 64 | ideal_tweet_length = 140 - len(tweet_prefix) 65 | 66 | best_tweet = create_markovated_tweet(tweets, ideal_tweet_length) 67 | 68 | if best_tweet != None: 69 | tweet = tweet_prefix + best_tweet 70 | twitter.post_tweet(tweet) 71 | encoded = unicode(tweet).encode('utf-8') 72 | print(encoded + '(' + str(len(encoded)) + ')') 73 | app_status['latest_reply'] = encoded 74 | else: 75 | print('

Could not generate reply

') 76 | app_status['latest_reply'] = 'Could not generate' 77 | 78 | def process_replies(): 79 | app_status = status.load() 80 | since_id = app_status.get('reply_since_id', -1) 81 | 82 | if since_id: 83 | mentions = twitter.get_mentions(since_id) 84 | else: 85 | mentions = twitter.get_mentions() 86 | 87 | print(str(len(mentions))+" mentions since "+str(since_id)) 88 | 89 | mentions.reverse() 90 | for mention in mentions: 91 | twitter.follow_user(mentions[-1]['user']['screen_name']) 92 | reply_to_user(mention['user'], app_status) 93 | 94 | app_status['reply_since_id'] = mention['id'] 95 | app_status['latest_user_replied_to'] = mention['user']['screen_name'] 96 | 97 | # Save after each one so if we crash we don't resend replies 98 | status.save(app_status) 99 | 100 | def produce_next_tweet(app_status): 101 | app_status = status.load() 102 | # Just get the latest tweets 103 | tweets = twitter.get_timeline_tweets(800) 104 | tweets = filter_tweets(tweets) 105 | tweets = filter(lambda t:not t['user']['screen_name'] == twitter_settings.screen_name, tweets) 106 | 107 | if len(tweets) <= 1: 108 | print('Could not generate tweet (not enough eligible tweets)') 109 | app_status['latest_tweet'] = 'Could not generate tweet (not enough eligible tweets)' 110 | return 111 | 112 | recent_tweets = twitter.get_tweets(twitter_settings.screen_name, True) 113 | 114 | best_tweet = create_markovated_tweet(tweets, 140, map(lambda t: t['text'].strip(), recent_tweets)) 115 | 116 | if best_tweet != None: 117 | twitter.post_tweet(best_tweet) 118 | encoded = unicode(best_tweet).encode('utf-8') 119 | print(encoded + '(' + str(len(encoded)) + ')') 120 | app_status['latest_tweet'] = encoded; 121 | else: 122 | print('Could not generate tweet') 123 | app_status['latest_tweet'] = 'Could not generate tweet' 124 | 125 | status.save(app_status) 126 | 127 | print("Started") 128 | process_replies() 129 | if random.randrange(240) == 0: 130 | produce_next_tweet(status) 131 | print("Finished") 132 | 133 | -------------------------------------------------------------------------------- /markovate.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | class Markovator: 4 | 5 | def __init__(self): 6 | self.words = {} 7 | self.starting_entry = {'word': None, 'following_words':[]} 8 | 9 | def add_to_entry(self, word, entry): 10 | existing_following_words = filter(lambda following_word: following_word['word'] == word, entry['following_words']) 11 | 12 | if len(existing_following_words) == 0: 13 | entry['following_words'].append({'word': word, 'count': 1}) 14 | else: 15 | existing_following_words[0]['count'] += 1 16 | 17 | def flatten_entry(self, entry): 18 | flattened_following_words = map(lambda following_word : [following_word['word'] for x in range(0, following_word['count'])] , entry['following_words']) 19 | return [item for sublist in flattened_following_words for item in sublist] 20 | 21 | def markovate(self): 22 | current_reply_word = random.choice(self.flatten_entry(self.starting_entry)) 23 | markovation = "" 24 | while not current_reply_word == None: 25 | markovation += current_reply_word + " " 26 | current_reply_word = random.choice(self.words[current_reply_word]['following_words'])['word'] 27 | 28 | return markovation.strip() 29 | 30 | def parse_sentence(self, sentence): 31 | new_words = sentence.lstrip().rstrip().split(' ') 32 | if len(new_words) > 0: 33 | previous_word = new_words.pop(0) 34 | self.add_to_entry(previous_word, self.starting_entry) 35 | 36 | for word in new_words: 37 | if not previous_word in self.words: 38 | self.words[previous_word] = {'word': previous_word, 'following_words':[]} 39 | 40 | if len(word) > 0: 41 | self.add_to_entry(word, self.words[previous_word]) 42 | previous_word = word 43 | 44 | if not previous_word in self.words: 45 | self.words[previous_word] = {'word': previous_word, 'following_words':[]} 46 | 47 | self.add_to_entry(None, self.words[previous_word]) 48 | 49 | def parse_sentences(self, sentences): 50 | for sentence in sentences: 51 | self.parse_sentence(sentence) 52 | -------------------------------------------------------------------------------- /oauth2/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | The MIT License 3 | 4 | Copyright (c) 2007-2010 Leah Culver, Joe Stump, Mark Paschal, Vic Fryzel 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | """ 24 | 25 | import urllib 26 | import time 27 | import random 28 | import urlparse 29 | import hmac 30 | import binascii 31 | import httplib2 32 | 33 | try: 34 | from urlparse import parse_qs, parse_qsl 35 | except ImportError: 36 | from cgi import parse_qs, parse_qsl 37 | 38 | 39 | VERSION = '1.0' # Hi Blaine! 40 | HTTP_METHOD = 'GET' 41 | SIGNATURE_METHOD = 'PLAINTEXT' 42 | 43 | 44 | class Error(RuntimeError): 45 | """Generic exception class.""" 46 | 47 | def __init__(self, message='OAuth error occurred.'): 48 | self._message = message 49 | 50 | @property 51 | def message(self): 52 | """A hack to get around the deprecation errors in 2.6.""" 53 | return self._message 54 | 55 | def __str__(self): 56 | return self._message 57 | 58 | 59 | class MissingSignature(Error): 60 | pass 61 | 62 | 63 | def build_authenticate_header(realm=''): 64 | """Optional WWW-Authenticate header (401 error)""" 65 | return {'WWW-Authenticate': 'OAuth realm="%s"' % realm} 66 | 67 | 68 | def build_xoauth_string(url, consumer, token=None): 69 | """Build an XOAUTH string for use in SMTP/IMPA authentication.""" 70 | request = Request.from_consumer_and_token(consumer, token, 71 | "GET", url) 72 | 73 | signing_method = SignatureMethod_HMAC_SHA1() 74 | request.sign_request(signing_method, consumer, token) 75 | 76 | params = [] 77 | for k, v in sorted(request.iteritems()): 78 | if v is not None: 79 | params.append('%s="%s"' % (k, escape(v))) 80 | 81 | return "%s %s %s" % ("GET", url, ','.join(params)) 82 | 83 | 84 | def escape(s): 85 | """Escape a URL including any /.""" 86 | return urllib.quote(s, safe='~') 87 | 88 | 89 | def _utf8_str(s): 90 | if isinstance(s, unicode): 91 | return s.encode('utf-8') 92 | else: 93 | return str(s) 94 | 95 | 96 | def generate_timestamp(): 97 | """Get seconds since epoch (UTC).""" 98 | return int(time.time()) 99 | 100 | 101 | def generate_nonce(length=8): 102 | """Generate pseudorandom number.""" 103 | return ''.join([str(random.randint(0, 9)) for i in range(length)]) 104 | 105 | 106 | def generate_verifier(length=8): 107 | """Generate pseudorandom number.""" 108 | return ''.join([str(random.randint(0, 9)) for i in range(length)]) 109 | 110 | 111 | class Consumer(object): 112 | """A consumer of OAuth-protected services. 113 | 114 | The OAuth consumer is a "third-party" service that wants to access 115 | protected resources from an OAuth service provider on behalf of an end 116 | user. It's kind of the OAuth client. 117 | 118 | Usually a consumer must be registered with the service provider by the 119 | developer of the consumer software. As part of that process, the service 120 | provider gives the consumer a *key* and a *secret* with which the consumer 121 | software can identify itself to the service. The consumer will include its 122 | key in each request to identify itself, but will use its secret only when 123 | signing requests, to prove that the request is from that particular 124 | registered consumer. 125 | 126 | Once registered, the consumer can then use its consumer credentials to ask 127 | the service provider for a request token, kicking off the OAuth 128 | authorization process. 129 | """ 130 | 131 | key = None 132 | secret = None 133 | 134 | def __init__(self, key, secret): 135 | self.key = key 136 | self.secret = secret 137 | 138 | if self.key is None or self.secret is None: 139 | raise ValueError("Key and secret must be set.") 140 | 141 | def __str__(self): 142 | data = {'oauth_consumer_key': self.key, 143 | 'oauth_consumer_secret': self.secret} 144 | 145 | return urllib.urlencode(data) 146 | 147 | 148 | class Token(object): 149 | """An OAuth credential used to request authorization or a protected 150 | resource. 151 | 152 | Tokens in OAuth comprise a *key* and a *secret*. The key is included in 153 | requests to identify the token being used, but the secret is used only in 154 | the signature, to prove that the requester is who the server gave the 155 | token to. 156 | 157 | When first negotiating the authorization, the consumer asks for a *request 158 | token* that the live user authorizes with the service provider. The 159 | consumer then exchanges the request token for an *access token* that can 160 | be used to access protected resources. 161 | """ 162 | 163 | key = None 164 | secret = None 165 | callback = None 166 | callback_confirmed = None 167 | verifier = None 168 | 169 | def __init__(self, key, secret): 170 | self.key = key 171 | self.secret = secret 172 | 173 | if self.key is None or self.secret is None: 174 | raise ValueError("Key and secret must be set.") 175 | 176 | def set_callback(self, callback): 177 | self.callback = callback 178 | self.callback_confirmed = 'true' 179 | 180 | def set_verifier(self, verifier=None): 181 | if verifier is not None: 182 | self.verifier = verifier 183 | else: 184 | self.verifier = generate_verifier() 185 | 186 | def get_callback_url(self): 187 | if self.callback and self.verifier: 188 | # Append the oauth_verifier. 189 | parts = urlparse.urlparse(self.callback) 190 | scheme, netloc, path, params, query, fragment = parts[:6] 191 | if query: 192 | query = '%s&oauth_verifier=%s' % (query, self.verifier) 193 | else: 194 | query = 'oauth_verifier=%s' % self.verifier 195 | return urlparse.urlunparse((scheme, netloc, path, params, 196 | query, fragment)) 197 | return self.callback 198 | 199 | def to_string(self): 200 | """Returns this token as a plain string, suitable for storage. 201 | 202 | The resulting string includes the token's secret, so you should never 203 | send or store this string where a third party can read it. 204 | """ 205 | 206 | data = { 207 | 'oauth_token': self.key, 208 | 'oauth_token_secret': self.secret, 209 | } 210 | 211 | if self.callback_confirmed is not None: 212 | data['oauth_callback_confirmed'] = self.callback_confirmed 213 | return urllib.urlencode(data) 214 | 215 | @staticmethod 216 | def from_string(s): 217 | """Deserializes a token from a string like one returned by 218 | `to_string()`.""" 219 | 220 | if not len(s): 221 | raise ValueError("Invalid parameter string.") 222 | 223 | params = parse_qs(s, keep_blank_values=False) 224 | if not len(params): 225 | raise ValueError("Invalid parameter string.") 226 | 227 | try: 228 | key = params['oauth_token'][0] 229 | except Exception: 230 | raise ValueError("'oauth_token' not found in OAuth request.") 231 | 232 | try: 233 | secret = params['oauth_token_secret'][0] 234 | except Exception: 235 | raise ValueError("'oauth_token_secret' not found in " 236 | "OAuth request.") 237 | 238 | token = Token(key, secret) 239 | try: 240 | token.callback_confirmed = params['oauth_callback_confirmed'][0] 241 | except KeyError: 242 | pass # 1.0, no callback confirmed. 243 | return token 244 | 245 | def __str__(self): 246 | return self.to_string() 247 | 248 | 249 | def setter(attr): 250 | name = attr.__name__ 251 | 252 | def getter(self): 253 | try: 254 | return self.__dict__[name] 255 | except KeyError: 256 | raise AttributeError(name) 257 | 258 | def deleter(self): 259 | del self.__dict__[name] 260 | 261 | return property(getter, attr, deleter) 262 | 263 | 264 | class Request(dict): 265 | 266 | """The parameters and information for an HTTP request, suitable for 267 | authorizing with OAuth credentials. 268 | 269 | When a consumer wants to access a service's protected resources, it does 270 | so using a signed HTTP request identifying itself (the consumer) with its 271 | key, and providing an access token authorized by the end user to access 272 | those resources. 273 | 274 | """ 275 | 276 | version = VERSION 277 | 278 | def __init__(self, method=HTTP_METHOD, url=None, parameters=None): 279 | self.method = method 280 | self.url = url 281 | if parameters is not None: 282 | self.update(parameters) 283 | 284 | @setter 285 | def url(self, value): 286 | self.__dict__['url'] = value 287 | if value is not None: 288 | scheme, netloc, path, params, query, fragment = urlparse.urlparse(value) 289 | 290 | # Exclude default port numbers. 291 | if scheme == 'http' and netloc[-3:] == ':80': 292 | netloc = netloc[:-3] 293 | elif scheme == 'https' and netloc[-4:] == ':443': 294 | netloc = netloc[:-4] 295 | if scheme not in ('http', 'https'): 296 | raise ValueError("Unsupported URL %s (%s)." % (value, scheme)) 297 | 298 | # Normalized URL excludes params, query, and fragment. 299 | self.normalized_url = urlparse.urlunparse((scheme, netloc, path, None, None, None)) 300 | else: 301 | self.normalized_url = None 302 | self.__dict__['url'] = None 303 | 304 | @setter 305 | def method(self, value): 306 | self.__dict__['method'] = value.upper() 307 | 308 | def _get_timestamp_nonce(self): 309 | return self['oauth_timestamp'], self['oauth_nonce'] 310 | 311 | def get_nonoauth_parameters(self): 312 | """Get any non-OAuth parameters.""" 313 | return dict([(k, v) for k, v in self.iteritems() 314 | if not k.startswith('oauth_')]) 315 | 316 | def to_header(self, realm=''): 317 | """Serialize as a header for an HTTPAuth request.""" 318 | oauth_params = ((k, v) for k, v in self.items() 319 | if k.startswith('oauth_')) 320 | stringy_params = ((k, escape(_utf8_str(v))) for k, v in oauth_params) 321 | header_params = ('%s="%s"' % (k, v) for k, v in stringy_params) 322 | params_header = ', '.join(header_params) 323 | 324 | auth_header = 'OAuth realm="%s"' % realm 325 | if params_header: 326 | auth_header = "%s, %s" % (auth_header, params_header) 327 | 328 | return {'Authorization': auth_header} 329 | 330 | def to_postdata(self): 331 | """Serialize as post data for a POST request.""" 332 | # tell urlencode to deal with sequence values and map them correctly 333 | # to resulting querystring. for example self["k"] = ["v1", "v2"] will 334 | # result in 'k=v1&k=v2' and not k=%5B%27v1%27%2C+%27v2%27%5D 335 | return urllib.urlencode(self, True).replace('+', '%20') 336 | 337 | def to_url(self): 338 | """Serialize as a URL for a GET request.""" 339 | base_url = urlparse.urlparse(self.url) 340 | try: 341 | query = base_url.query 342 | except AttributeError: 343 | # must be python <2.5 344 | query = base_url[4] 345 | query = parse_qs(query) 346 | for k, v in self.items(): 347 | query.setdefault(k, []).append(v) 348 | 349 | try: 350 | scheme = base_url.scheme 351 | netloc = base_url.netloc 352 | path = base_url.path 353 | params = base_url.params 354 | fragment = base_url.fragment 355 | except AttributeError: 356 | # must be python <2.5 357 | scheme = base_url[0] 358 | netloc = base_url[1] 359 | path = base_url[2] 360 | params = base_url[3] 361 | fragment = base_url[5] 362 | 363 | url = (scheme, netloc, path, params, 364 | urllib.urlencode(query, True), fragment) 365 | return urlparse.urlunparse(url) 366 | 367 | def get_parameter(self, parameter): 368 | ret = self.get(parameter) 369 | if ret is None: 370 | raise Error('Parameter not found: %s' % parameter) 371 | 372 | return ret 373 | 374 | def get_normalized_parameters(self): 375 | """Return a string that contains the parameters that must be signed.""" 376 | items = [] 377 | for key, value in self.iteritems(): 378 | if key == 'oauth_signature': 379 | continue 380 | # 1.0a/9.1.1 states that kvp must be sorted by key, then by value, 381 | # so we unpack sequence values into multiple items for sorting. 382 | if hasattr(value, '__iter__'): 383 | items.extend((key, item) for item in value) 384 | else: 385 | items.append((key, value)) 386 | 387 | # Include any query string parameters from the provided URL 388 | query = urlparse.urlparse(self.url)[4] 389 | 390 | url_items = self._split_url_string(query).items() 391 | non_oauth_url_items = list([(k, v) for k, v in url_items if not k.startswith('oauth_')]) 392 | items.extend(non_oauth_url_items) 393 | 394 | items = [(_utf8_str(k), _utf8_str(v)) for k, v in items] 395 | encoded_str = urllib.urlencode(sorted(items)) 396 | # Encode signature parameters per Oauth Core 1.0 protocol 397 | # spec draft 7, section 3.6 398 | # (http://tools.ietf.org/html/draft-hammer-oauth-07#section-3.6) 399 | # Spaces must be encoded with "%20" instead of "+" 400 | return encoded_str.replace('+', '%20').replace('%7E', '~') 401 | 402 | def sign_request(self, signature_method, consumer, token): 403 | """Set the signature parameter to the result of sign.""" 404 | 405 | if 'oauth_consumer_key' not in self: 406 | self['oauth_consumer_key'] = consumer.key 407 | 408 | if token and 'oauth_token' not in self: 409 | self['oauth_token'] = token.key 410 | 411 | self['oauth_signature_method'] = signature_method.name 412 | self['oauth_signature'] = signature_method.sign(self, consumer, token) 413 | 414 | @classmethod 415 | def make_timestamp(cls): 416 | """Get seconds since epoch (UTC).""" 417 | return str(int(time.time())) 418 | 419 | @classmethod 420 | def make_nonce(cls): 421 | """Generate pseudorandom number.""" 422 | return str(random.randint(0, 100000000)) 423 | 424 | @classmethod 425 | def from_request(cls, http_method, http_url, headers=None, parameters=None, 426 | query_string=None): 427 | """Combines multiple parameter sources.""" 428 | if parameters is None: 429 | parameters = {} 430 | 431 | # Headers 432 | if headers and 'Authorization' in headers: 433 | auth_header = headers['Authorization'] 434 | # Check that the authorization header is OAuth. 435 | if auth_header[:6] == 'OAuth ': 436 | auth_header = auth_header[6:] 437 | try: 438 | # Get the parameters from the header. 439 | header_params = cls._split_header(auth_header) 440 | parameters.update(header_params) 441 | except: 442 | raise Error('Unable to parse OAuth parameters from ' 443 | 'Authorization header.') 444 | 445 | # GET or POST query string. 446 | if query_string: 447 | query_params = cls._split_url_string(query_string) 448 | parameters.update(query_params) 449 | 450 | # URL parameters. 451 | param_str = urlparse.urlparse(http_url)[4] # query 452 | url_params = cls._split_url_string(param_str) 453 | parameters.update(url_params) 454 | 455 | if parameters: 456 | return cls(http_method, http_url, parameters) 457 | 458 | return None 459 | 460 | @classmethod 461 | def from_consumer_and_token(cls, consumer, token=None, 462 | http_method=HTTP_METHOD, http_url=None, parameters=None): 463 | if not parameters: 464 | parameters = {} 465 | 466 | defaults = { 467 | 'oauth_consumer_key': consumer.key, 468 | 'oauth_timestamp': cls.make_timestamp(), 469 | 'oauth_nonce': cls.make_nonce(), 470 | 'oauth_version': cls.version, 471 | } 472 | 473 | defaults.update(parameters) 474 | parameters = defaults 475 | 476 | if token: 477 | parameters['oauth_token'] = token.key 478 | if token.verifier: 479 | parameters['oauth_verifier'] = token.verifier 480 | 481 | return Request(http_method, http_url, parameters) 482 | 483 | @classmethod 484 | def from_token_and_callback(cls, token, callback=None, 485 | http_method=HTTP_METHOD, http_url=None, parameters=None): 486 | 487 | if not parameters: 488 | parameters = {} 489 | 490 | parameters['oauth_token'] = token.key 491 | 492 | if callback: 493 | parameters['oauth_callback'] = callback 494 | 495 | return cls(http_method, http_url, parameters) 496 | 497 | @staticmethod 498 | def _split_header(header): 499 | """Turn Authorization: header into parameters.""" 500 | params = {} 501 | parts = header.split(',') 502 | for param in parts: 503 | # Ignore realm parameter. 504 | if param.find('realm') > -1: 505 | continue 506 | # Remove whitespace. 507 | param = param.strip() 508 | # Split key-value. 509 | param_parts = param.split('=', 1) 510 | # Remove quotes and unescape the value. 511 | params[param_parts[0]] = urllib.unquote(param_parts[1].strip('\"')) 512 | return params 513 | 514 | @staticmethod 515 | def _split_url_string(param_str): 516 | """Turn URL string into parameters.""" 517 | parameters = parse_qs(param_str, keep_blank_values=False) 518 | for k, v in parameters.iteritems(): 519 | parameters[k] = urllib.unquote(v[0]) 520 | return parameters 521 | 522 | 523 | class Client(httplib2.Http): 524 | """OAuthClient is a worker to attempt to execute a request.""" 525 | 526 | def __init__(self, consumer, token=None, cache=None, timeout=None, 527 | proxy_info=None): 528 | 529 | if consumer is not None and not isinstance(consumer, Consumer): 530 | raise ValueError("Invalid consumer.") 531 | 532 | if token is not None and not isinstance(token, Token): 533 | raise ValueError("Invalid token.") 534 | 535 | self.consumer = consumer 536 | self.token = token 537 | self.method = SignatureMethod_HMAC_SHA1() 538 | 539 | httplib2.Http.__init__(self, cache=cache, timeout=timeout, 540 | proxy_info=proxy_info) 541 | 542 | def set_signature_method(self, method): 543 | if not isinstance(method, SignatureMethod): 544 | raise ValueError("Invalid signature method.") 545 | 546 | self.method = method 547 | 548 | def request(self, uri, method="GET", body=None, headers=None, 549 | redirections=httplib2.DEFAULT_MAX_REDIRECTS, connection_type=None): 550 | DEFAULT_CONTENT_TYPE = 'application/x-www-form-urlencoded' 551 | 552 | if not isinstance(headers, dict): 553 | headers = {} 554 | 555 | is_multipart = method == 'POST' and headers.get('Content-Type', 556 | DEFAULT_CONTENT_TYPE) != DEFAULT_CONTENT_TYPE 557 | 558 | if body and method == "POST" and not is_multipart: 559 | parameters = dict(parse_qsl(body)) 560 | else: 561 | parameters = None 562 | 563 | req = Request.from_consumer_and_token(self.consumer, 564 | token=self.token, http_method=method, http_url=uri, 565 | parameters=parameters) 566 | 567 | req.sign_request(self.method, self.consumer, self.token) 568 | 569 | if method == "POST": 570 | headers['Content-Type'] = headers.get('Content-Type', 571 | DEFAULT_CONTENT_TYPE) 572 | if is_multipart: 573 | headers.update(req.to_header()) 574 | else: 575 | body = req.to_postdata() 576 | elif method == "GET": 577 | uri = req.to_url() 578 | else: 579 | headers.update(req.to_header()) 580 | 581 | return httplib2.Http.request(self, uri, method=method, body=body, 582 | headers=headers, redirections=redirections, 583 | connection_type=connection_type) 584 | 585 | 586 | class Server(object): 587 | """A skeletal implementation of a service provider, providing protected 588 | resources to requests from authorized consumers. 589 | 590 | This class implements the logic to check requests for authorization. You 591 | can use it with your web server or web framework to protect certain 592 | resources with OAuth. 593 | """ 594 | 595 | timestamp_threshold = 300 # In seconds, five minutes. 596 | version = VERSION 597 | signature_methods = None 598 | 599 | def __init__(self, signature_methods=None): 600 | self.signature_methods = signature_methods or {} 601 | 602 | def add_signature_method(self, signature_method): 603 | self.signature_methods[signature_method.name] = signature_method 604 | return self.signature_methods 605 | 606 | def verify_request(self, request, consumer, token): 607 | """Verifies an api call and checks all the parameters.""" 608 | 609 | version = self._get_version(request) 610 | self._check_signature(request, consumer, token) 611 | parameters = request.get_nonoauth_parameters() 612 | return parameters 613 | 614 | def build_authenticate_header(self, realm=''): 615 | """Optional support for the authenticate header.""" 616 | return {'WWW-Authenticate': 'OAuth realm="%s"' % realm} 617 | 618 | def _get_version(self, request): 619 | """Verify the correct version request for this server.""" 620 | try: 621 | version = request.get_parameter('oauth_version') 622 | except: 623 | version = VERSION 624 | 625 | if version and version != self.version: 626 | raise Error('OAuth version %s not supported.' % str(version)) 627 | 628 | return version 629 | 630 | def _get_signature_method(self, request): 631 | """Figure out the signature with some defaults.""" 632 | try: 633 | signature_method = request.get_parameter('oauth_signature_method') 634 | except: 635 | signature_method = SIGNATURE_METHOD 636 | 637 | try: 638 | # Get the signature method object. 639 | signature_method = self.signature_methods[signature_method] 640 | except: 641 | signature_method_names = ', '.join(self.signature_methods.keys()) 642 | raise Error('Signature method %s not supported try one of the following: %s' % (signature_method, signature_method_names)) 643 | 644 | return signature_method 645 | 646 | def _get_verifier(self, request): 647 | return request.get_parameter('oauth_verifier') 648 | 649 | def _check_signature(self, request, consumer, token): 650 | timestamp, nonce = request._get_timestamp_nonce() 651 | self._check_timestamp(timestamp) 652 | signature_method = self._get_signature_method(request) 653 | 654 | try: 655 | signature = request.get_parameter('oauth_signature') 656 | except: 657 | raise MissingSignature('Missing oauth_signature.') 658 | 659 | # Validate the signature. 660 | valid = signature_method.check(request, consumer, token, signature) 661 | 662 | if not valid: 663 | key, base = signature_method.signing_base(request, consumer, token) 664 | 665 | raise Error('Invalid signature. Expected signature base ' 666 | 'string: %s' % base) 667 | 668 | built = signature_method.sign(request, consumer, token) 669 | 670 | def _check_timestamp(self, timestamp): 671 | """Verify that timestamp is recentish.""" 672 | timestamp = int(timestamp) 673 | now = int(time.time()) 674 | lapsed = now - timestamp 675 | if lapsed > self.timestamp_threshold: 676 | raise Error('Expired timestamp: given %d and now %s has a ' 677 | 'greater difference than threshold %d' % (timestamp, now, 678 | self.timestamp_threshold)) 679 | 680 | 681 | class SignatureMethod(object): 682 | """A way of signing requests. 683 | 684 | The OAuth protocol lets consumers and service providers pick a way to sign 685 | requests. This interface shows the methods expected by the other `oauth` 686 | modules for signing requests. Subclass it and implement its methods to 687 | provide a new way to sign requests. 688 | """ 689 | 690 | def signing_base(self, request, consumer, token): 691 | """Calculates the string that needs to be signed. 692 | 693 | This method returns a 2-tuple containing the starting key for the 694 | signing and the message to be signed. The latter may be used in error 695 | messages to help clients debug their software. 696 | 697 | """ 698 | raise NotImplementedError 699 | 700 | def sign(self, request, consumer, token): 701 | """Returns the signature for the given request, based on the consumer 702 | and token also provided. 703 | 704 | You should use your implementation of `signing_base()` to build the 705 | message to sign. Otherwise it may be less useful for debugging. 706 | 707 | """ 708 | raise NotImplementedError 709 | 710 | def check(self, request, consumer, token, signature): 711 | """Returns whether the given signature is the correct signature for 712 | the given consumer and token signing the given request.""" 713 | built = self.sign(request, consumer, token) 714 | return built == signature 715 | 716 | 717 | class SignatureMethod_HMAC_SHA1(SignatureMethod): 718 | name = 'HMAC-SHA1' 719 | 720 | def signing_base(self, request, consumer, token): 721 | if request.normalized_url is None: 722 | raise ValueError("Base URL for request is not set.") 723 | 724 | sig = ( 725 | escape(request.method), 726 | escape(request.normalized_url), 727 | escape(request.get_normalized_parameters()), 728 | ) 729 | 730 | key = '%s&' % escape(consumer.secret) 731 | if token: 732 | key += escape(token.secret) 733 | raw = '&'.join(sig) 734 | return key, raw 735 | 736 | def sign(self, request, consumer, token): 737 | """Builds the base signature string.""" 738 | key, raw = self.signing_base(request, consumer, token) 739 | 740 | # HMAC object. 741 | try: 742 | from hashlib import sha1 as sha 743 | except ImportError: 744 | import sha # Deprecated 745 | 746 | hashed = hmac.new(key, raw, sha) 747 | 748 | # Calculate the digest base 64. 749 | return binascii.b2a_base64(hashed.digest())[:-1] 750 | 751 | 752 | class SignatureMethod_PLAINTEXT(SignatureMethod): 753 | 754 | name = 'PLAINTEXT' 755 | 756 | def signing_base(self, request, consumer, token): 757 | """Concatenates the consumer key and secret with the token's 758 | secret.""" 759 | sig = '%s&' % escape(consumer.secret) 760 | if token: 761 | sig = sig + escape(token.secret) 762 | return sig, sig 763 | 764 | def sign(self, request, consumer, token): 765 | key, raw = self.signing_base(request, consumer, token) 766 | return raw 767 | -------------------------------------------------------------------------------- /oauth2/clients/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tom-martin/markovator/7c8477787c4ed3afb40adf31f446ea34a8848a81/oauth2/clients/__init__.py -------------------------------------------------------------------------------- /oauth2/clients/imap.py: -------------------------------------------------------------------------------- 1 | """ 2 | The MIT License 3 | 4 | Copyright (c) 2007-2010 Leah Culver, Joe Stump, Mark Paschal, Vic Fryzel 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | """ 24 | 25 | import oauth2 26 | import imaplib 27 | 28 | 29 | class IMAP4_SSL(imaplib.IMAP4_SSL): 30 | """IMAP wrapper for imaplib.IMAP4_SSL that implements XOAUTH.""" 31 | 32 | def authenticate(self, url, consumer, token): 33 | if consumer is not None and not isinstance(consumer, oauth2.Consumer): 34 | raise ValueError("Invalid consumer.") 35 | 36 | if token is not None and not isinstance(token, oauth2.Token): 37 | raise ValueError("Invalid token.") 38 | 39 | imaplib.IMAP4_SSL.authenticate(self, 'XOAUTH', 40 | lambda x: oauth2.build_xoauth_string(url, consumer, token)) 41 | -------------------------------------------------------------------------------- /oauth2/clients/smtp.py: -------------------------------------------------------------------------------- 1 | """ 2 | The MIT License 3 | 4 | Copyright (c) 2007-2010 Leah Culver, Joe Stump, Mark Paschal, Vic Fryzel 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | """ 24 | 25 | import oauth2 26 | import smtplib 27 | import base64 28 | 29 | 30 | class SMTP(smtplib.SMTP): 31 | """SMTP wrapper for smtplib.SMTP that implements XOAUTH.""" 32 | 33 | def authenticate(self, url, consumer, token): 34 | if consumer is not None and not isinstance(consumer, oauth2.Consumer): 35 | raise ValueError("Invalid consumer.") 36 | 37 | if token is not None and not isinstance(token, oauth2.Token): 38 | raise ValueError("Invalid token.") 39 | 40 | self.docmd('AUTH', 'XOAUTH %s' % \ 41 | base64.b64encode(oauth2.build_xoauth_string(url, consumer, token))) 42 | -------------------------------------------------------------------------------- /status.json: -------------------------------------------------------------------------------- 1 | {"latest_user_replied_to": "markovator_test", "reply_since_id": 407213638743638016, "latest_reply": "@markovator_test markovated: \u043e!\u043e\u0447\u0435\u043d\u044c \u043e\u0447\u0435\u043d\u044c of it was a Scottish heavyweight wrestling champion. He was not the family Arctiidae.", "latest_tweet": "Hello December. I go to be thankful that I hate those little styrofoam popcorns with a prescription, exactly how long?"} -------------------------------------------------------------------------------- /status_endpoint.py: -------------------------------------------------------------------------------- 1 | import cgi 2 | from django.utils import simplejson as json 3 | import random 4 | 5 | import twitter 6 | 7 | from google.appengine.ext import webapp 8 | from google.appengine.ext import db 9 | 10 | from google.appengine.api import urlfetch 11 | 12 | 13 | import twitter 14 | import status 15 | 16 | class StatusHandler(webapp.RequestHandler): 17 | def get(self): 18 | entity = status.load_entity() 19 | self.response.out.write("
" + entity.json_string +  "
") 20 | self.response.out.write("
") 21 | 22 | self.response.out.write("
" + json.dumps(twitter.get_rate_limit_status(True)) + "
"); 23 | 24 | 25 | def post(self): 26 | status.save(json.loads(self.request.get("json_status"))) 27 | self.get() 28 | 29 | class ClearStatusHandler(webapp.RequestHandler): 30 | def get(self): 31 | status.clear() 32 | self.response.out.write("

Cleared

") 33 | 34 | 35 | -------------------------------------------------------------------------------- /test_status.py: -------------------------------------------------------------------------------- 1 | import file_system_status as status 2 | 3 | s = {} 4 | s['reply_since_id'] = '123' 5 | status.save(s) 6 | s = status.load() 7 | assert s['reply_since_id'] == '123' 8 | status.clear() 9 | s = status.load() 10 | assert 'reply_since_id' not in s 11 | print("Tests passed") -------------------------------------------------------------------------------- /test_twitter.py: -------------------------------------------------------------------------------- 1 | import twitter 2 | 3 | print twitter.get_rate_limit_status() 4 | 5 | mentions = twitter.get_mentions() 6 | assert len(mentions) > 1 7 | 8 | tweets = twitter.get_tweets('markovator_dev') 9 | assert len(tweets) > 1 10 | 11 | print("Tests passed") 12 | -------------------------------------------------------------------------------- /twitter.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | import json 4 | 5 | import oauth2 as oauth 6 | 7 | import httplib2 8 | 9 | import urllib 10 | 11 | import twitter_settings 12 | 13 | class TwitterError(Exception): 14 | def __init__(self, status_code, content): 15 | self.status_code = status_code 16 | self.content = content 17 | def __str__(self): 18 | return "Twitter returned " + str(self.status_code) + " : " + self.content 19 | 20 | 21 | def get_mentions(since=-1): 22 | client = oauth.Client(twitter_settings.consumer, twitter_settings.token) 23 | 24 | if since > -1: 25 | resp, content = client.request("https://api.twitter.com/1.1/statuses/mentions_timeline.json?count=800&since_id=" + str(since), "GET") 26 | else: 27 | resp, content = client.request("https://api.twitter.com/1.1/statuses/mentions_timeline.json?count=200", "GET") 28 | 29 | if resp.status != 200: 30 | raise TwitterError(resp.status, content) 31 | 32 | return json.loads(content) 33 | 34 | def get_tweets(screen_name, auth=True): 35 | if auth: 36 | client = oauth.Client(twitter_settings.consumer, twitter_settings.token) 37 | else: 38 | client = httplib2.Http(timeout=30) 39 | 40 | 41 | resp, content = client.request('https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=' + screen_name + '&count=800&trim_user=true', "GET") 42 | 43 | if resp.status != 200: 44 | raise TwitterError(resp.status, content) 45 | 46 | return json.loads(content) 47 | 48 | def get_timeline_tweets(count): 49 | client = oauth.Client(twitter_settings.consumer, twitter_settings.token) 50 | 51 | resp, content = client.request('https://api.twitter.com/1.1/statuses/home_timeline.json?count=' + str(count), "GET") 52 | 53 | if resp.status != 200: 54 | raise TwitterError(resp.status, content) 55 | 56 | return json.loads(content) 57 | 58 | def get_timeline_tweets_since(since_id=-1): 59 | client = oauth.Client(twitter_settings.consumer, twitter_settings.token) 60 | tweets = [] 61 | 62 | if since_id < 0: 63 | resp, content = client.request('https://api.twitter.com/1.1/statuses/home_timeline.json', "GET") 64 | 65 | if resp.status != 200: 66 | raise TwitterError(resp.status, content) 67 | 68 | tweets.extend(json.loads(content)) 69 | else: 70 | # TODO 1 or 0? 71 | current_page = 0 72 | while len(tweets) == 0 or not since_id >= max(map(lambda t:int(t['id']), tweets)): 73 | resp, content = client.request('https://api.twitter.com/1.1/statuses/home_timeline.json?count=800&page=' + str(current_page), "GET") 74 | new_tweets = json.loads(content) 75 | if len(new_tweets) == 0: 76 | break 77 | tweets.extend(new_tweets) 78 | current_page += 1 79 | 80 | return tweets 81 | 82 | def post_tweet(text): 83 | client = oauth.Client(twitter_settings.consumer, twitter_settings.token) 84 | resp, content = client.request("https://api.twitter.com/1.1/statuses/update.json", "POST", urllib.urlencode([("status", unicode(text).encode('utf-8'))])) 85 | 86 | # TODO Check status code 87 | 88 | return content 89 | 90 | def follow_user(screen_name): 91 | client = oauth.Client(twitter_settings.consumer, twitter_settings.token) 92 | resp, content = client.request("https://api.twitter.com/1.1/friendships/create.json", "POST", urllib.urlencode([("screen_name", screen_name)])) 93 | 94 | # TODO Check status code 95 | 96 | return content 97 | 98 | def get_rate_limit_status(auth=True): 99 | if auth: 100 | client = oauth.Client(twitter_settings.consumer, twitter_settings.token) 101 | else: 102 | client = httplib2.Http(timeout=30) 103 | resp, content = client.request('https://api.twitter.com/1.1/application/rate_limit_status.json', "GET") 104 | 105 | return json.loads(content) 106 | --------------------------------------------------------------------------------