├── .gitignore ├── .gitmodules ├── README.markdown ├── etc ├── add_timestamps.py ├── convert_db.py ├── create_couch.py ├── schema.sql ├── scratch.py └── verify_couch.py ├── lib ├── paisley.py └── twitterspy │ ├── __init__.py │ ├── adhoc_commands.py │ ├── cache.py │ ├── config.py │ ├── db.py │ ├── db_base.py │ ├── db_couch.py │ ├── db_sql.py │ ├── moodiness.py │ ├── protocol.py │ ├── scheduling.py │ ├── search_collector.py │ ├── url_expansion.py │ ├── xmpp_commands.py │ └── xmpp_ping.py ├── test ├── expansion_test.py ├── mood_test.py ├── scheduling_test.py └── search_collector_test.py ├── twitterspy.conf.sample ├── twitterspy.start └── twitterspy.tac /.gitignore: -------------------------------------------------------------------------------- 1 | twitterspy.log 2 | twitterspy.conf 3 | *.sqlite3 4 | *.sqlite3-journal 5 | *.pyc 6 | log 7 | twistd.pid 8 | *~ 9 | _trial_temp 10 | 11 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "lib/twitty-twister"] 2 | path = lib/twitty-twister 3 | url = git://github.com/dustin/twitty-twister.git 4 | [submodule "lib/wokkel"] 5 | path = lib/wokkel 6 | url = git://github.com/dustin/wokkel.git 7 | [submodule "lib/twisted-longurl"] 8 | path = lib/twisted-longurl 9 | url = git://github.com/dustin/twisted-longurl 10 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # Twitter Spy 2 | 3 | TwitterSpy is a supplemental bot for twitter that does the stuff the twitter 4 | one used to do, and a few things it doesn't. 5 | 6 | # Usage 7 | 8 | IM `help` to [im@twitterspy.org](xmpp:im@twitterspy.org) to see what 9 | you can do. 10 | 11 | # More Info 12 | 13 | Go to the [project page](http://dustin.github.com/twitterspy/) for more info. 14 | -------------------------------------------------------------------------------- /etc/add_timestamps.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | sys.path.append('lib') 5 | sys.path.append('../lib') 6 | 7 | import models 8 | 9 | models._engine.execute("alter table users add column created_at timestamp") 10 | -------------------------------------------------------------------------------- /etc/convert_db.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | sys.path.extend(["lib", "../lib"]) 5 | 6 | from sqlite3 import dbapi2 as sqlite 7 | 8 | from twisted.internet import reactor, defer 9 | 10 | from twitterspy import db 11 | 12 | GET_USERS=""" 13 | select jid, username, password, active, status, min_id, language, 14 | auto_post, friend_timeline_id, direct_message_id, created_at, id 15 | from users 16 | """ 17 | 18 | GET_TRACKS=""" 19 | select query 20 | from tracks join user_tracks on (tracks.id = user_tracks.track_id) 21 | where user_tracks.user_id = ? 22 | """ 23 | 24 | DB=sqlite.connect(sys.argv[1]) 25 | 26 | CUR=DB.cursor() 27 | 28 | def parse_timestamp(ts): 29 | return None 30 | 31 | def create(e, r): 32 | print "Creating record for", r[0] 33 | user = db.User() 34 | user.jid = r[0] 35 | user.username = r[1] 36 | user.password = r[2] 37 | user.active = bool(r[3]) 38 | user.status = r[4] 39 | user.min_id = r[5] 40 | user.language = r[6] 41 | user.auto_post = bool(r[7]) 42 | user.friend_timeline_id = r[8] 43 | user.direct_message_id = r[9] 44 | user.created_at = parse_timestamp(r[10]) 45 | 46 | for tr in CUR.execute(GET_TRACKS, [r[11]]).fetchall(): 47 | user.track(tr[0]) 48 | 49 | return user.save() 50 | 51 | @defer.deferredGenerator 52 | def load_records(): 53 | couch = db.get_couch() 54 | 55 | for r in CUR.execute(GET_USERS).fetchall(): 56 | d = couch.openDoc(db.DB_NAME, str(r[0])) 57 | d.addErrback(create, r) 58 | wfd = defer.waitForDeferred(d) 59 | yield wfd 60 | 61 | reactor.stop() 62 | 63 | reactor.callWhenRunning(load_records) 64 | reactor.run() 65 | -------------------------------------------------------------------------------- /etc/create_couch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | sys.path.extend(["lib", "../lib"]) 5 | 6 | from twisted.internet import reactor, defer 7 | 8 | from twitterspy import db, cache 9 | 10 | def parse_timestamp(ts): 11 | return None 12 | 13 | @defer.deferredGenerator 14 | def create_database(): 15 | couch = db.get_couch() 16 | d = couch.createDB(db.DB_NAME) 17 | wfd = defer.waitForDeferred(d) 18 | yield wfd 19 | print wfd.getResult() 20 | 21 | doc=""" 22 | {"language": "javascript", 23 | "views": { 24 | "counts": { 25 | "map": "function(doc) { 26 | if(doc.doctype == 'User') { 27 | var cnt = 0; 28 | if(doc.tracks) { 29 | cnt = doc.tracks.length; 30 | } 31 | emit(null, {users: 1, tracks: cnt}); 32 | } 33 | }", 34 | "reduce": "function(key, values) { 35 | var result = {users: 0, tracks: 0}; 36 | values.forEach(function(p) { 37 | result.users += p.users; 38 | result.tracks += p.tracks; 39 | }); 40 | return result; 41 | }" 42 | }, 43 | "status": { 44 | "map": "function(doc) { 45 | if(doc.doctype == 'User') { 46 | emit(doc.status, 1); 47 | } 48 | }", 49 | "reduce": "function(k, v) { 50 | return sum(v); 51 | }" 52 | }, 53 | "service": { 54 | "map": "function(doc) { 55 | emit(doc.service_jid, 1); 56 | }", 57 | "reduce": "function(k, v) { 58 | return sum(v); 59 | }" 60 | } 61 | }} 62 | """ 63 | d = couch.saveDoc(db.DB_NAME, doc, '_design/counts') 64 | wfd = defer.waitForDeferred(d) 65 | yield wfd 66 | print wfd.getResult() 67 | 68 | doc=""" 69 | {"language":"javascript","views":{"query_counts":{"map":"function(doc) {\n if(doc.doctype == 'User') {\n doc.tracks.forEach(function(query) {\n emit(query, 1);\n });\n }\n}","reduce":"function(key, values) {\n return sum(values);\n}"}}} 70 | """ 71 | 72 | d = couch.saveDoc(db.DB_NAME, doc, '_design/query_counts') 73 | wfd = defer.waitForDeferred(d) 74 | yield wfd 75 | print wfd.getResult() 76 | 77 | doc=""" 78 | {"language": "javascript", 79 | "views": { 80 | "active": { 81 | "map": "function(doc) { 82 | if(doc.doctype == 'User' && doc.active) { 83 | emit(null, doc._id); 84 | } 85 | }" 86 | }, 87 | "to_be_migrated": { 88 | "map": "function(doc) { 89 | if(doc.service_jid === 'twitterspy@jabber.org/bot') { 90 | emit(doc.service_jid, null); 91 | } 92 | }" 93 | } 94 | }} 95 | """ 96 | 97 | d = couch.saveDoc(db.DB_NAME, doc, '_design/users') 98 | wfd = defer.waitForDeferred(d) 99 | yield wfd 100 | print wfd.getResult() 101 | 102 | reactor.stop() 103 | 104 | reactor.callWhenRunning(cache.connect) 105 | reactor.callWhenRunning(create_database) 106 | reactor.run() 107 | -------------------------------------------------------------------------------- /etc/schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE tracks ( 2 | id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, 3 | query VARCHAR(50) NOT NULL, 4 | max_seen INTEGER); 5 | 6 | CREATE TABLE "user_tracks" ( 7 | id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, 8 | user_id INTEGER NOT NULL, 9 | track_id INTEGER NOT NULL, 10 | created_at DATETIME); 11 | 12 | CREATE TABLE "users" ( 13 | id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, 14 | jid VARCHAR(128) NOT NULL, 15 | service_jid VARCHAR(128) NULL, 16 | username VARCHAR(50), 17 | password VARCHAR(50), 18 | active BOOLEAN NOT NULL DEFAULT 't', 19 | status VARCHAR(50), 20 | min_id INTEGER NOT NULL DEFAULT 0, 21 | language VARCHAR(2), 22 | auto_post BOOLEAN NOT NULL DEFAULT 'f', 23 | friend_timeline_id integer, 24 | direct_message_id integer, 25 | created_at timestamp); 26 | 27 | CREATE UNIQUE INDEX unique_index_user_tracks_id ON user_tracks (id); 28 | CREATE UNIQUE INDEX unique_index_user_tracks_idx_ut_ut ON user_tracks (user_id, track_id); 29 | CREATE UNIQUE INDEX unique_index_users_id ON users (id); 30 | CREATE UNIQUE INDEX unique_index_users_jid ON users (jid); 31 | -------------------------------------------------------------------------------- /etc/scratch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | sys.path.extend(['lib', '../lib']) 5 | 6 | from twisted.internet import defer, reactor 7 | 8 | from twitterspy import db 9 | 10 | @defer.deferredGenerator 11 | def f(): 12 | d = db.get_active_users() 13 | wfd = defer.waitForDeferred(d) 14 | yield wfd 15 | u = wfd.getResult() 16 | print u 17 | 18 | reactor.stop() 19 | 20 | reactor.callWhenRunning(f) 21 | reactor.run() 22 | -------------------------------------------------------------------------------- /etc/verify_couch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | sys.path.extend(["lib", "../lib"]) 5 | 6 | from sqlite3 import dbapi2 as sqlite 7 | 8 | from twisted.internet import reactor, defer 9 | 10 | from twitterspy import db 11 | 12 | GET_USERS=""" 13 | select jid, username, password, active, status, min_id, language, 14 | auto_post, friend_timeline_id, direct_message_id, created_at, id 15 | from users 16 | """ 17 | 18 | DB=sqlite.connect(sys.argv[1]) 19 | 20 | CUR=DB.cursor() 21 | 22 | def parse_timestamp(ts): 23 | return None 24 | 25 | @defer.deferredGenerator 26 | def verify_users(): 27 | couch = db.get_couch() 28 | for r in CUR.execute(GET_USERS).fetchall(): 29 | d = couch.openDoc(db.DB_NAME, str(r[0])) 30 | d.addErrback(lambda x: sys.stdout.write("Can't find %s\n" % r[0])) 31 | wfd = defer.waitForDeferred(d) 32 | yield wfd 33 | 34 | reactor.stop() 35 | 36 | reactor.callWhenRunning(verify_users) 37 | reactor.run() 38 | -------------------------------------------------------------------------------- /lib/paisley.py: -------------------------------------------------------------------------------- 1 | # -*- test-case-name: test_paisley -*- 2 | # Copyright (c) 2007-2008 3 | # See LICENSE for details. 4 | 5 | """ 6 | CouchDB client. 7 | """ 8 | 9 | import simplejson 10 | from urllib import urlencode 11 | 12 | from twisted.web.client import HTTPClientFactory 13 | from twisted.internet import defer 14 | 15 | from twitterspy import cache 16 | 17 | try: 18 | from base64 import b64encode 19 | except ImportError: 20 | import base64 21 | 22 | def b64encode(s): 23 | return "".join(base64.encodestring(s).split("\n")) 24 | 25 | try: 26 | from functools import partial 27 | except ImportError: 28 | class partial(object): 29 | def __init__(self, fn, *args, **kw): 30 | self.fn = fn 31 | self.args = args 32 | self.kw = kw 33 | 34 | def __call__(self, *args, **kw): 35 | if kw and self.kw: 36 | d = self.kw.copy() 37 | d.update(kw) 38 | else: 39 | d = kw or self.kw 40 | return self.fn(*(self.args + args), **d) 41 | 42 | 43 | 44 | class CouchDB(object): 45 | """ 46 | CouchDB client: hold methods for accessing a couchDB. 47 | """ 48 | 49 | def __init__(self, host, port=5984, dbName=None): 50 | """ 51 | Initialize the client for given host. 52 | 53 | @param host: address of the server. 54 | @type host: C{str} 55 | 56 | @param port: if specified, the port of the server. 57 | @type port: C{int} 58 | 59 | @param dbName: if specified, all calls needing a database name will use 60 | this one by default. 61 | @type dbName: C{str} 62 | """ 63 | self.host = host 64 | self.port = int(port) 65 | self.url_template = "http://%s:%s%%s" % (self.host, self.port) 66 | if dbName is not None: 67 | self.bindToDB(dbName) 68 | 69 | 70 | def parseResult(self, result): 71 | """ 72 | Parse JSON result from the DB. 73 | """ 74 | return simplejson.loads(result) 75 | 76 | 77 | def bindToDB(self, dbName): 78 | """ 79 | Bind all operations asking for a DB name to the given DB. 80 | """ 81 | for methname in ["createDB", "deleteDB", "infoDB", "listDoc", 82 | "openDoc", "saveDoc", "deleteDoc", "openView", 83 | "tempView"]: 84 | method = getattr(self, methname) 85 | newMethod = partial(method, dbName) 86 | setattr(self, methname, newMethod) 87 | 88 | 89 | # Database operations 90 | 91 | def createDB(self, dbName): 92 | """ 93 | Creates a new database on the server. 94 | """ 95 | # Responses: {u'ok': True}, 409 Conflict, 500 Internal Server Error 96 | return self.put("/%s/" % (dbName,), "" 97 | ).addCallback(self.parseResult) 98 | 99 | 100 | def deleteDB(self, dbName): 101 | """ 102 | Deletes the database on the server. 103 | """ 104 | # Responses: {u'ok': True}, 404 Object Not Found 105 | return self.delete("/%s/" % (dbName,) 106 | ).addCallback(self.parseResult) 107 | 108 | 109 | def listDB(self): 110 | """ 111 | List the databases on the server. 112 | """ 113 | # Responses: list of db names 114 | return self.get("/_all_dbs").addCallback(self.parseResult) 115 | 116 | 117 | def infoDB(self, dbName): 118 | """ 119 | Returns info about the couchDB. 120 | """ 121 | # Responses: {u'update_seq': 0, u'db_name': u'mydb', u'doc_count': 0} 122 | # 404 Object Not Found 123 | return self.get("/%s/" % (dbName,) 124 | ).addCallback(self.parseResult) 125 | 126 | 127 | # Document operations 128 | 129 | def listDoc(self, dbName, reverse=False, startKey=0, count=-1): 130 | """ 131 | List all documents in a given database. 132 | """ 133 | # Responses: {u'rows': [{u'_rev': -1825937535, u'_id': u'mydoc'}], 134 | # u'view': u'_all_docs'}, 404 Object Not Found 135 | uri = "/%s/_all_docs" % (dbName,) 136 | args = {} 137 | if reverse: 138 | args["reverse"] = "true" 139 | if startKey > 0: 140 | args["startkey"] = int(startKey) 141 | if count >= 0: 142 | args["count"] = int(count) 143 | if args: 144 | uri += "?%s" % (urlencode(args),) 145 | return self.get(uri 146 | ).addCallback(self.parseResult) 147 | 148 | 149 | def openDoc(self, dbName, docId, revision=None, full=False, attachment=""): 150 | """ 151 | Open a document in a given database. 152 | 153 | @param revision: if specified, the revision of the document desired. 154 | @type revision: C{str} 155 | 156 | @param full: if specified, return the list of all the revisions of the 157 | document, along with the document itself. 158 | @type full: C{bool} 159 | 160 | @param attachment: if specified, return the named attachment from the 161 | document. 162 | @type attachment: C{str} 163 | """ 164 | # Responses: {u'_rev': -1825937535, u'_id': u'mydoc', ...} 165 | # 404 Object Not Found 166 | uri = "/%s/%s" % (dbName, docId) 167 | if revision is not None: 168 | uri += "?%s" % (urlencode({"rev": revision}),) 169 | elif full: 170 | uri += "?%s" % (urlencode({"full": "true"}),) 171 | elif attachment: 172 | uri += "?%s" % (urlencode({"attachment": attachment}),) 173 | # No parsing 174 | return self.get(uri) 175 | 176 | rv = defer.Deferred() 177 | 178 | def mc_res(res): 179 | if res[1]: 180 | rv.callback(self.parseResult(res[1])) 181 | else: 182 | d = self.get(uri) 183 | def cf(s): 184 | cache.mc.set(uri, s) 185 | return s 186 | d.addCallback(cf) 187 | d.addCallback(lambda s: rv.callback(self.parseResult(s))) 188 | d.addErrback(lambda e: rv.errback(e)) 189 | 190 | cache.mc.get(uri).addCallback(mc_res) 191 | 192 | return rv 193 | 194 | def addAttachments(self, document, attachments): 195 | """ 196 | Add attachments to a document, before sending it to the DB. 197 | 198 | @param document: the document to modify. 199 | @type document: C{dict} 200 | 201 | @param attachments: the attachments to add. 202 | @type attachments: C{dict} 203 | """ 204 | document.setdefault("_attachments", {}) 205 | for name, data in attachments.iteritems(): 206 | data = b64encode(data) 207 | document["_attachments"][name] = {"type": "base64", "data": data} 208 | 209 | 210 | def saveDoc(self, dbName, body, docId=None): 211 | """ 212 | Save/create a document to/in a given database. 213 | 214 | @param dbName: identifier of the database. 215 | @type dbName: C{str} 216 | 217 | @param body: content of the document. 218 | @type body: C{str} or any structured object 219 | 220 | @param docId: if specified, the identifier to be used in the database. 221 | @type docId: C{str} 222 | """ 223 | # Responses: {u'_rev': 1175338395, u'_id': u'mydoc', u'ok': True} 224 | # 409 Conflict, 500 Internal Server Error 225 | if not isinstance(body, (str, unicode)): 226 | body = simplejson.dumps(body) 227 | if docId is not None: 228 | uri = "/%s/%s" % (dbName, docId) 229 | cache.mc.delete(uri) 230 | d = self.put(uri, body) 231 | else: 232 | d = self.post("/%s/" % (dbName,), body) 233 | 234 | return d.addCallback(self.parseResult) 235 | 236 | 237 | def deleteDoc(self, dbName, docId, revision): 238 | """ 239 | Delete a document on given database. 240 | """ 241 | # Responses: {u'_rev': 1469561101, u'ok': True} 242 | # 500 Internal Server Error 243 | return self.delete("/%s/%s?%s" % ( 244 | dbName, 245 | docId, 246 | urlencode({'rev': revision})) 247 | ).addCallback(self.parseResult) 248 | 249 | 250 | # View operations 251 | 252 | def openView(self, dbName, docId, viewId, **kwargs): 253 | """ 254 | Open a view of a document in a given database. 255 | """ 256 | uri = "/%s/_design/%s/_view/%s" % (dbName, docId, viewId) 257 | 258 | if kwargs: 259 | uri += "?%s" % (urlencode(kwargs),) 260 | 261 | return self.get(uri 262 | ).addCallback(self.parseResult) 263 | 264 | 265 | def addViews(self, document, views): 266 | """ 267 | Add views to a document. 268 | 269 | @param document: the document to modify. 270 | @type document: C{dict} 271 | 272 | @param views: the views to add. 273 | @type views: C{dict} 274 | """ 275 | document.setdefault("views", {}) 276 | for name, data in views.iteritems(): 277 | document["views"][name] = data 278 | 279 | 280 | def tempView(self, dbName, view): 281 | """ 282 | Make a temporary view on the server. 283 | """ 284 | d = self.post("/%s/_temp_view" % (dbName,), view) 285 | return d.addCallback(self.parseResult) 286 | 287 | 288 | # Basic http methods 289 | 290 | def _getPage(self, uri, **kwargs): 291 | """ 292 | C{getPage}-like. 293 | """ 294 | url = self.url_template % (uri,) 295 | if not 'headers' in kwargs: 296 | kwargs['headers'] = {} 297 | kwargs["headers"]["Accept"] = "application/json" 298 | if 'timeout' not in kwargs: 299 | kwargs['timeout'] = 10 300 | factory = HTTPClientFactory(url, **kwargs) 301 | from twisted.internet import reactor 302 | reactor.connectTCP(self.host, self.port, factory) 303 | return factory.deferred 304 | 305 | 306 | def get(self, uri): 307 | """ 308 | Execute a C{GET} at C{uri}. 309 | """ 310 | return self._getPage(uri, method="GET") 311 | 312 | 313 | def post(self, uri, body, **kwargs): 314 | """ 315 | Execute a C{POST} of C{body} at C{uri}. 316 | """ 317 | kwargs['postdata'] = body 318 | kwargs['method'] = 'POST' 319 | return self._getPage(uri, **kwargs) 320 | 321 | 322 | def put(self, uri, body): 323 | """ 324 | Execute a C{PUT} of C{body} at C{uri}. 325 | """ 326 | return self._getPage(uri, method="PUT", postdata=body) 327 | 328 | 329 | def delete(self, uri): 330 | """ 331 | Execute a C{DELETE} at C{uri}. 332 | """ 333 | return self._getPage(uri, method="DELETE") 334 | -------------------------------------------------------------------------------- /lib/twitterspy/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Twitterspy. 3 | """ 4 | -------------------------------------------------------------------------------- /lib/twitterspy/adhoc_commands.py: -------------------------------------------------------------------------------- 1 | from zope.interface import implements 2 | 3 | from twisted.python import log 4 | from twisted.internet import defer 5 | from twisted.words.xish import domish 6 | from wokkel.subprotocols import XMPPHandler, IQHandlerMixin 7 | from twisted.words.protocols.jabber import jid, error 8 | from twisted.words.protocols.jabber.xmlstream import toResponse 9 | from wokkel import disco 10 | from wokkel import generic 11 | from wokkel import data_form 12 | from wokkel.iwokkel import IDisco 13 | 14 | import db 15 | import protocol 16 | import scheduling 17 | 18 | NS_CMD = 'http://jabber.org/protocol/commands' 19 | CMD = generic.IQ_SET + '/command[@xmlns="' + NS_CMD + '"]' 20 | 21 | all_commands = {} 22 | 23 | def form_required(orig): 24 | def every(self, user, iq, cmd): 25 | if cmd.firstChildElement(): 26 | form = data_form.Form.fromElement(cmd.firstChildElement()) 27 | return orig(self, user, iq, cmd, form) 28 | else: 29 | form = data_form.Form(formType="form", title=self.name) 30 | self.fillForm(user, iq, cmd, form) 31 | return self.genFormCmdResponse(iq, cmd, form) 32 | return every 33 | 34 | class BaseCommand(object): 35 | """Base class for xep 0050 command processors.""" 36 | 37 | def __init__(self, node, name): 38 | self.node = node 39 | self.name = name 40 | 41 | def _genCmdResponse(self, iq, cmd, status=None): 42 | 43 | command = domish.Element(('http://jabber.org/protocol/commands', 44 | "command")) 45 | command['node'] = cmd['node'] 46 | if status: 47 | command['status'] = status 48 | try: 49 | command['action'] = cmd['action'] 50 | except KeyError: 51 | pass 52 | 53 | return command 54 | 55 | def genFormCmdResponse(self, iq, cmd, form): 56 | command = self._genCmdResponse(iq, cmd, 'executing') 57 | 58 | actions = command.addElement('actions') 59 | actions['execute'] = 'next' 60 | actions.addElement('next') 61 | 62 | command.addChild(form.toElement()) 63 | 64 | return command 65 | 66 | def __call__(self, user, iq, cmd): 67 | # Will return success 68 | pass 69 | 70 | class TrackManagementCommand(BaseCommand): 71 | 72 | def __init__(self): 73 | super(TrackManagementCommand, self).__init__('tracks', 74 | 'List and manage tracks') 75 | 76 | def fillForm(self, user, iq, cmd, form): 77 | form.instructions = ["Select the items you no longer wish to track."] 78 | form.addField(data_form.Field(var='tracks', fieldType='list-multi', 79 | options=(data_form.Option(v, v) 80 | for v in sorted(user.tracks)))) 81 | 82 | @form_required 83 | def __call__(self, user, iq, cmd, form): 84 | vals = set(form.fields['tracks'].values) 85 | log.msg("Removing: %s" % vals) 86 | user.tracks = list(set(user.tracks).difference(vals)) 87 | 88 | def worked(stuff): 89 | for v in vals: 90 | scheduling.queries.untracked(user.jid, v) 91 | 92 | user.save().addCallback(worked) 93 | 94 | class TrackManagementCommand(BaseCommand): 95 | 96 | def __init__(self): 97 | super(TrackManagementCommand, self).__init__('tracks', 98 | 'List and manage tracks') 99 | 100 | def fillForm(self, user, iq, cmd, form): 101 | form.instructions = ["Select the items you no longer wish to track."] 102 | form.addField(data_form.Field(var='tracks', fieldType='list-multi', 103 | options=(data_form.Option(v, v) 104 | for v in sorted(user.tracks)))) 105 | 106 | @form_required 107 | def __call__(self, user, iq, cmd, form): 108 | vals = set(form.fields['tracks'].values) 109 | log.msg("Removing: %s" % vals) 110 | user.tracks = list(set(user.tracks).difference(vals)) 111 | 112 | def worked(stuff): 113 | for v in vals: 114 | scheduling.queries.untracked(user.jid, v) 115 | 116 | user.save().addCallback(worked) 117 | 118 | for __t in (t for t in globals().values() if isinstance(type, type(t))): 119 | if BaseCommand in __t.__mro__: 120 | try: 121 | i = __t() 122 | all_commands[i.node] = i 123 | except TypeError, e: 124 | # Ignore abstract bases 125 | log.msg("Error loading %s: %s" % (__t.__name__, str(e))) 126 | pass 127 | 128 | class AdHocHandler(XMPPHandler, IQHandlerMixin): 129 | 130 | implements(IDisco) 131 | 132 | iqHandlers = { CMD: 'onCommand' } 133 | 134 | def connectionInitialized(self): 135 | super(AdHocHandler, self).connectionInitialized() 136 | self.xmlstream.addObserver(CMD, self.handleRequest) 137 | 138 | def _onUserCmd(self, user, iq, cmd): 139 | return all_commands[cmd['node']](user, iq, cmd) 140 | 141 | def onCommand(self, iq): 142 | log.msg("Got an adhoc command request") 143 | cmd = iq.firstChildElement() 144 | assert cmd.name == 'command' 145 | 146 | return db.User.by_jid(jid.JID(iq['from']).userhost() 147 | ).addCallback(self._onUserCmd, iq, cmd) 148 | 149 | def getDiscoInfo(self, requestor, target, node): 150 | info = set() 151 | 152 | if node: 153 | info.add(disco.DiscoIdentity('automation', 'command-node')) 154 | info.add(disco.DiscoFeature('http://jabber.org/protocol/commands')) 155 | else: 156 | info.add(disco.DiscoFeature(NS_CMD)) 157 | 158 | return defer.succeed(info) 159 | 160 | def getDiscoItems(self, requestor, target, node): 161 | myjid = jid.internJID(protocol.default_conn.jid) 162 | return defer.succeed([disco.DiscoItem(myjid, c.node, c.name) 163 | for c in all_commands.values()]) 164 | 165 | -------------------------------------------------------------------------------- /lib/twitterspy/cache.py: -------------------------------------------------------------------------------- 1 | from twisted.python import log 2 | from twisted.internet import protocol, reactor 3 | from twisted.protocols import memcache 4 | 5 | mc = None 6 | 7 | class MemcacheFactory(protocol.ReconnectingClientFactory): 8 | 9 | def buildProtocol(self, addr): 10 | global mc 11 | self.resetDelay() 12 | log.msg("Connected to memcached.") 13 | mc = memcache.MemCacheProtocol() 14 | return mc 15 | 16 | def connect(): 17 | reactor.connectTCP('localhost', memcache.DEFAULT_PORT, 18 | MemcacheFactory()) 19 | -------------------------------------------------------------------------------- /lib/twitterspy/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Configuration for twitterspy. 4 | 5 | Copyright (c) 2008 Dustin Sallings 6 | """ 7 | 8 | import ConfigParser 9 | import commands 10 | 11 | CONF=ConfigParser.ConfigParser() 12 | CONF.read('twitterspy.conf') 13 | VERSION=commands.getoutput("git describe").strip() 14 | ADMINS=CONF.get("general", "admins").split(' ') 15 | -------------------------------------------------------------------------------- /lib/twitterspy/db.py: -------------------------------------------------------------------------------- 1 | import ConfigParser 2 | 3 | import config 4 | 5 | db_type = config.CONF.get("db", "type") 6 | 7 | if db_type == 'couch': 8 | from db_couch import * 9 | elif db_type == 'sql': 10 | from db_sql import * 11 | else: 12 | raise ConfigParser.Error("Unknown database type: " + db_type) 13 | -------------------------------------------------------------------------------- /lib/twitterspy/db_base.py: -------------------------------------------------------------------------------- 1 | import time 2 | import base64 3 | 4 | import config 5 | 6 | class BaseUser(object): 7 | 8 | def __init__(self, jid=None): 9 | self.jid = jid 10 | self.active = True 11 | self.min_id = 0 12 | self.auto_post = False 13 | self.username = None 14 | self.password = None 15 | self.status = None 16 | self.friend_timeline_id = None 17 | self.direct_message_id = None 18 | self.created_at = time.time() 19 | self._rev = None 20 | self.service_jid = None 21 | self.tracks = [] 22 | 23 | def __repr__(self): 24 | return "" % (self.jid, len(self.tracks)) 25 | 26 | def track(self, query): 27 | self.tracks.append(query) 28 | 29 | def untrack(self, query): 30 | try: 31 | self.tracks.remove(query) 32 | return True 33 | except ValueError: 34 | return False 35 | 36 | @property 37 | def has_credentials(self): 38 | return self.username and self.password 39 | 40 | @property 41 | def decoded_password(self): 42 | return base64.decodestring(self.password) if self.password else None 43 | 44 | @property 45 | def is_admin(self): 46 | return self.jid in config.ADMINS 47 | 48 | -------------------------------------------------------------------------------- /lib/twitterspy/db_couch.py: -------------------------------------------------------------------------------- 1 | """All kinds of stuff for talking to databases.""" 2 | 3 | import time 4 | 5 | from twisted.python import log 6 | from twisted.internet import defer, task 7 | 8 | import paisley 9 | 10 | import config 11 | import db_base 12 | 13 | DB_NAME='twitterspy' 14 | 15 | def get_couch(): 16 | try: 17 | port = config.CONF.getint('db', 'port') 18 | except: 19 | port = 5984 20 | return paisley.CouchDB(config.CONF.get('db', 'host'), port=port) 21 | 22 | class User(db_base.BaseUser): 23 | 24 | @staticmethod 25 | def from_doc(doc): 26 | user = User() 27 | user.jid = doc['_id'] 28 | v = doc 29 | user.active = v.get('active') 30 | user.auto_post = v.get('auto_post') 31 | user.username = v.get('username') 32 | user.password = v.get('password') 33 | user.status = v.get('status') 34 | user.friend_timeline_id = v.get('friend_timeline_id') 35 | user.direct_message_id = v.get('direct_message_id') 36 | user.service_jid = v.get('service_jid') 37 | user.created_at = v.get('created_at', time.time()) 38 | user._rev = v.get('_rev') 39 | user.tracks = v.get('tracks', []) 40 | return user 41 | 42 | def to_doc(self): 43 | rv = dict(self.__dict__) 44 | rv['doctype'] = 'User' 45 | for k in [k for k,v in rv.items() if not v]: 46 | del rv[k] 47 | # Don't need two copies of the jids. 48 | del rv['jid'] 49 | return rv 50 | 51 | @staticmethod 52 | def by_jid(jid): 53 | couch = get_couch() 54 | d = couch.openDoc(DB_NAME, str(jid)) 55 | rv = defer.Deferred() 56 | d.addCallback(lambda doc: rv.callback(User.from_doc(doc))) 57 | d.addErrback(lambda e: rv.callback(User(jid))) 58 | return rv 59 | 60 | def save(self): 61 | return get_couch().saveDoc(DB_NAME, self.to_doc(), str(self.jid)) 62 | 63 | def initialize(): 64 | def periodic(name, path): 65 | log.msg("Performing %s." % name) 66 | def cb(x): 67 | log.msg("%s result: %s" % (name, repr(x))) 68 | headers = {'Content-Type': 'application/json'} 69 | get_couch().post("/" + DB_NAME + path, 70 | '', headers=headers).addCallback(cb) 71 | 72 | compactLoop = task.LoopingCall(periodic, 'compaction', '/_compact') 73 | compactLoop.start(3600, now=False) 74 | 75 | viewCleanLoop = task.LoopingCall(periodic, 'view cleanup', 76 | '/_view_cleanup') 77 | viewCleanLoop.start(3*3600, now=False) 78 | 79 | def model_counts(): 80 | """Returns a deferred whose callback will receive a dict of object 81 | counts, e.g. 82 | 83 | {'users': n, 'tracks': m} 84 | """ 85 | d = defer.Deferred() 86 | docd = get_couch().openView(DB_NAME, "counts", "counts") 87 | docd.addCallback(lambda r: d.callback(r['rows'][0]['value'])) 88 | docd.addErrback(lambda e: d.errback(e)) 89 | 90 | return d 91 | 92 | def get_top10(n=10): 93 | """Returns a deferred whose callback will receive a list of at 94 | most `n` (number, 'tag') pairs sorted in reverse""" 95 | d = defer.Deferred() 96 | docd = get_couch().openView(DB_NAME, "query_counts", "query_counts", 97 | group="true") 98 | def processResults(resp): 99 | rows = sorted([(r['value'], r['key']) for r in resp['rows']], 100 | reverse=True) 101 | d.callback(rows[:n]) 102 | docd.addCallback(processResults) 103 | docd.addErrback(lambda e: d.errback(e)) 104 | return d 105 | 106 | def get_active_users(): 107 | """Returns a deferred whose callback will receive a list of active JIDs.""" 108 | d = defer.Deferred() 109 | docd = get_couch().openView(DB_NAME, "users", "active") 110 | docd.addCallback(lambda res: d.callback([r['value'] for r in res['rows']])) 111 | docd.addErrback(lambda e: d.errback(e)) 112 | return d 113 | 114 | def get_service_distribution(): 115 | """Returns a deferred whose callback will receive a list of jid -> count pairs""" 116 | d = defer.Deferred() 117 | docd = get_couch().openView(DB_NAME, "counts", "service", group='true') 118 | docd.addCallback(lambda rv: d.callback([(r['key'], r['value']) for r in rv['rows']])) 119 | docd.addErrback(lambda e: d.errback(e)) 120 | return d 121 | -------------------------------------------------------------------------------- /lib/twitterspy/db_sql.py: -------------------------------------------------------------------------------- 1 | """All kinds of stuff for talking to databases.""" 2 | 3 | import base64 4 | import time 5 | 6 | from twisted.python import log 7 | from twisted.internet import defer, task 8 | from twisted.enterprise import adbapi 9 | 10 | import config 11 | import db_base 12 | 13 | DB_POOL = adbapi.ConnectionPool(config.CONF.get("db", "driver"), 14 | *eval(config.CONF.get("db", "args", raw=True))) 15 | 16 | def parse_time(t): 17 | return None 18 | 19 | def maybe_int(t): 20 | if t: 21 | return int(t) 22 | 23 | class User(db_base.BaseUser): 24 | 25 | def __init__(self, jid=None): 26 | super(User, self).__init__(jid) 27 | self._id = -1 28 | 29 | @staticmethod 30 | def by_jid(jid): 31 | def load_user(txn): 32 | txn.execute("select active, auto_post, username, password, " 33 | "friend_timeline_id, direct_message_id, created_at, " 34 | "status, service_jid, id " 35 | "from users where jid = ?", [jid]) 36 | u = txn.fetchall() 37 | if u: 38 | r = u[0] 39 | log.msg("Loading from %s" % str(r)) 40 | user = User() 41 | user.jid = jid 42 | user.active = maybe_int(r[0]) == 1 43 | user.auto_post = maybe_int(r[1]) == 1 44 | user.username = r[2] 45 | user.password = r[3] 46 | user.friend_timeline_id = maybe_int(r[4]) 47 | user.direct_message_id = maybe_int(r[5]) 48 | user.created_at = parse_time(r[6]) 49 | user.status = r[7] 50 | user.service_jid = r[8] 51 | user._id = r[9] 52 | 53 | txn.execute("""select query 54 | from tracks join user_tracks on (tracks.id = user_tracks.track_id) 55 | where user_tracks.user_id = ?""", [user._id]) 56 | user.tracks = [t[0] for t in txn.fetchall()] 57 | 58 | log.msg("Loaded %s (%s)" % (user, user.active)) 59 | return user 60 | else: 61 | return User(jid) 62 | return DB_POOL.runInteraction(load_user) 63 | 64 | def _save_in_txn(self, txn): 65 | 66 | active = 1 if self.active else 0 67 | 68 | if self._id == -1: 69 | txn.execute("insert into users(" 70 | " jid, active, auto_post, username, password, status, " 71 | " friend_timeline_id, direct_message_id, " 72 | " service_jid, created_at )" 73 | " values(?, ?, ?, ?, ?, ?, ?, ?, ?, current_timestamp)", 74 | [self.jid, active, self.auto_post, self.status, 75 | self.username, self.password, 76 | self.friend_timeline_id, 77 | self.direct_message_id, self.service_jid]) 78 | 79 | # sqlite specific... 80 | txn.execute("select last_insert_rowid()") 81 | self._id = txn.fetchall()[0][0] 82 | else: 83 | txn.execute("update users set active=?, auto_post=?, " 84 | " username=?, password=?, status=?, " 85 | " friend_timeline_id=?, direct_message_id=?, " 86 | " service_jid = ? " 87 | " where id = ?", 88 | [active, self.auto_post, 89 | self.username, self.password, self.status, 90 | self.friend_timeline_id, 91 | self.direct_message_id, self.service_jid, 92 | self._id]) 93 | 94 | # TODO: Begin difficult process of synchronizing track lists 95 | txn.execute("""select user_tracks.id, query 96 | from tracks join user_tracks on (tracks.id = user_tracks.track_id) 97 | where user_tracks.user_id = ?""", [self._id]) 98 | db_tracks = {} 99 | for i, q in txn.fetchall(): 100 | db_tracks[q] = str(i) 101 | 102 | rm_ids = [db_tracks[q] for q in db_tracks.keys() if not q in self.tracks] 103 | 104 | # Remove track records that no longer exist. 105 | txn.execute("delete from user_tracks where id in (?)", 106 | [', '.join(rm_ids)]) 107 | 108 | # Add the missing tracks. 109 | for q in [q for q in self.tracks if not q in db_tracks]: 110 | txn.execute("insert into user_tracks(user_id, track_id, created_at) " 111 | " values(?, ?, current_timestamp)", 112 | [self._id, self._qid(txn, q)]) 113 | 114 | return True 115 | 116 | def _qid(self, txn, q): 117 | txn.execute("select id from tracks where query = ?", [q]) 118 | r = txn.fetchall() 119 | if r: 120 | return r[0][0] 121 | else: 122 | txn.execute("insert into tracks (query) values(?)", [q]) 123 | txn.execute("select last_insert_rowid()") 124 | res = txn.fetchall() 125 | return res[0][0] 126 | 127 | def save(self): 128 | return DB_POOL.runInteraction(self._save_in_txn) 129 | 130 | def initialize(): 131 | pass 132 | 133 | def model_counts(): 134 | """Returns a deferred whose callback will receive a dict of object 135 | counts, e.g. 136 | 137 | {'users': n, 'tracks': m} 138 | """ 139 | d = defer.Deferred() 140 | 141 | dbd = DB_POOL.runQuery("""select 'users', count(*) from users 142 | union all 143 | select 'tracks', count(*) from user_tracks""") 144 | 145 | def cb(rows): 146 | rv = {} 147 | for r in rows: 148 | rv[r[0]] = int(r[1]) 149 | d.callback(rv) 150 | 151 | dbd.addCallback(cb) 152 | dbd.addErrback(lambda e: d.errback(e)) 153 | 154 | return d 155 | 156 | def get_top10(n=10): 157 | """Returns a deferred whose callback will receive a list of at 158 | most `n` (number, 'tag') pairs sorted in reverse""" 159 | 160 | return DB_POOL.runQuery("""select count(*), t.query as watchers 161 | from tracks t join user_tracks ut on (t.id = ut.track_id) 162 | group by t.query 163 | order by watchers desc, query 164 | limit 10""") 165 | 166 | 167 | def get_active_users(): 168 | """Returns a deferred whose callback will receive a list of active JIDs.""" 169 | 170 | d = defer.Deferred() 171 | dbd = DB_POOL.runQuery("select jid from users where active = 1") 172 | dbd.addCallback(lambda res: d.callback([r[0] for r in res])) 173 | dbd.addErrback(lambda e: d.errback(e)) 174 | 175 | return d 176 | -------------------------------------------------------------------------------- /lib/twitterspy/moodiness.py: -------------------------------------------------------------------------------- 1 | from collections import deque, defaultdict 2 | import random 3 | 4 | from twisted.python import log 5 | 6 | import protocol 7 | 8 | MAX_RESULTS = 1000 9 | 10 | class Moodiness(object): 11 | 12 | MOOD_CHOICES=[ 13 | (0.9, ('happy', 'humbled')), 14 | (0.5, ('frustrated', 'annoyed', 'anxious', 'grumpy')), 15 | (0.1, ('annoyed', 'dismayed', 'depressed', 'worried')), 16 | (float('-inf'), ('angry', 'cranky', 'disappointed')) 17 | ] 18 | 19 | def __init__(self): 20 | self.recent_results = deque() 21 | self.previous_good = (0, 0) 22 | 23 | def current_mood(self): 24 | """Get the current mood (good, total, percentage)""" 25 | if not self.recent_results: 26 | log.msg("Short-circuiting tally results since there aren't any.") 27 | return None, None, None, None 28 | try: 29 | good = reduce(lambda x, y: x + 1 if (y is True) else x, self.recent_results, 0) 30 | except TypeError: 31 | log.msg("Error reducing: %s" % str(self.recent_results)) 32 | raise 33 | total = len(self.recent_results) 34 | percentage = float(good) / float(total) 35 | choices=[v for a,v in self.MOOD_CHOICES if percentage >= a][0] 36 | mood=random.choice(choices) 37 | 38 | return mood, good, total, percentage 39 | 40 | def result_counts(self): 41 | rv = defaultdict(lambda: 0) 42 | for v in self.recent_results: 43 | rv[v] += 1 44 | return rv 45 | 46 | def __call__(self): 47 | mood, good, total, percentage = self.current_mood() 48 | if mood is None: 49 | return 50 | self.previous_good = (good, total) 51 | 52 | msg = ("Processed %d out of %d recent searches (previously %d/%d)." 53 | % (good, total, self.previous_good[0], self.previous_good[1])) 54 | 55 | log.msg(msg + " my mood is " + mood) 56 | for conn in protocol.current_conns.values(): 57 | if conn.pubsub: 58 | conn.publish_mood(mood, msg) 59 | 60 | def add(self, result): 61 | if len(self.recent_results) >= MAX_RESULTS: 62 | self.recent_results.popleft() 63 | self.recent_results.append(result) 64 | 65 | def markSuccess(self, *args): 66 | """Record that a search was successfully performed.""" 67 | self.add(True) 68 | 69 | def markFailure(self, error): 70 | """Record that a search failed to complete successfully.""" 71 | try: 72 | erval = error.value.status 73 | except AttributeError: 74 | erval = False 75 | self.add(erval) 76 | return error 77 | 78 | moodiness = Moodiness() 79 | -------------------------------------------------------------------------------- /lib/twitterspy/protocol.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import with_statement 4 | 5 | import time 6 | 7 | from twisted.python import log 8 | from twisted.internet import protocol, reactor, threads 9 | from twisted.words.xish import domish 10 | from twisted.words.protocols.jabber.jid import JID 11 | from twisted.words.protocols.jabber.xmlstream import IQ 12 | 13 | from wokkel.xmppim import MessageProtocol, PresenceClientProtocol 14 | from wokkel.xmppim import AvailablePresence 15 | 16 | import db 17 | import xmpp_commands 18 | import config 19 | import cache 20 | import scheduling 21 | import string 22 | 23 | CHATSTATE_NS = 'http://jabber.org/protocol/chatstates' 24 | 25 | current_conns = {} 26 | presence_conns = {} 27 | 28 | # This... could get big 29 | # user_jid -> service_jid 30 | service_mapping = {} 31 | 32 | default_conn = None 33 | default_presence = None 34 | 35 | class TwitterspyMessageProtocol(MessageProtocol): 36 | 37 | pubsub = True 38 | 39 | def __init__(self, jid): 40 | super(TwitterspyMessageProtocol, self).__init__() 41 | self._pubid = 1 42 | self.jid = jid.full() 43 | 44 | self.preferred = self.jid == config.CONF.get("xmpp", "jid") 45 | 46 | goodChars=string.letters + string.digits + "/=,_+.-~@" 47 | self.jidtrans = self._buildGoodSet(goodChars) 48 | 49 | def _buildGoodSet(self, goodChars, badChar='_'): 50 | allChars=string.maketrans("", "") 51 | badchars=string.translate(allChars, allChars, goodChars) 52 | rv=string.maketrans(badchars, badChar * len(badchars)) 53 | return rv 54 | 55 | def connectionInitialized(self): 56 | super(TwitterspyMessageProtocol, self).connectionInitialized() 57 | log.msg("Connected!") 58 | 59 | commands=xmpp_commands.all_commands 60 | self.commands={} 61 | for c in commands.values(): 62 | self.commands[c.name] = c 63 | for a in c.aliases: 64 | self.commands[a] = c 65 | log.msg("Loaded commands: %s" % `sorted(commands.keys())`) 66 | 67 | self.pubsub = True 68 | 69 | # Let the scheduler know we connected. 70 | scheduling.connected() 71 | 72 | self._pubid = 1 73 | 74 | global current_conns, default_conn 75 | current_conns[self.jid] = self 76 | if self.preferred: 77 | default_conn = self 78 | 79 | def connectionLost(self, reason): 80 | log.msg("Disconnected!") 81 | 82 | global current_conns, default_conn 83 | del current_conns[self.jid] 84 | 85 | if default_conn == self: 86 | default_conn = None 87 | 88 | scheduling.disconnected() 89 | 90 | def _gen_id(self, prefix): 91 | self._pubid += 1 92 | return prefix + str(self._pubid) 93 | 94 | def publish_mood(self, mood_str, text): 95 | iq = IQ(self.xmlstream, 'set') 96 | iq['from'] = self.jid 97 | pubsub = iq.addElement(('http://jabber.org/protocol/pubsub', 'pubsub')) 98 | moodpub = pubsub.addElement('publish') 99 | moodpub['node'] = 'http://jabber.org/protocol/mood' 100 | item = moodpub.addElement('item') 101 | mood = item.addElement(('http://jabber.org/protocol/mood', 'mood')) 102 | mood.addElement(mood_str) 103 | mood.addElement('text').addContent(text) 104 | def _doLog(x): 105 | log.msg("Delivered mood: %s (%s)" % (mood_str, text)) 106 | def _hasError(x): 107 | log.err(x) 108 | log.msg("Error delivering mood, disabling for %s." % self.jid) 109 | self.pubsub = False 110 | log.msg("Delivering mood: %s" % iq.toXml()) 111 | d = iq.send() 112 | d.addCallback(_doLog) 113 | d.addErrback(_hasError) 114 | 115 | def typing_notification(self, jid): 116 | """Send a typing notification to the given jid.""" 117 | 118 | msg = domish.Element((None, "message")) 119 | msg["to"] = jid 120 | msg["from"] = self.jid 121 | msg.addElement((CHATSTATE_NS, 'composing')) 122 | self.send(msg) 123 | 124 | def create_message(self): 125 | msg = domish.Element((None, "message")) 126 | msg.addElement((CHATSTATE_NS, 'active')) 127 | return msg 128 | 129 | def send_plain(self, jid, content): 130 | msg = self.create_message() 131 | msg["to"] = jid 132 | msg["from"] = self.jid 133 | msg["type"] = 'chat' 134 | msg.addElement("body", content=content) 135 | 136 | self.send(msg) 137 | 138 | def send_html(self, jid, body, html): 139 | msg = self.create_message() 140 | msg["to"] = jid 141 | msg["from"] = self.jid 142 | msg["type"] = 'chat' 143 | html = u""+unicode(html)+u"" 144 | msg.addRawXml(u"" + unicode(body) + u"") 145 | msg.addRawXml(unicode(html)) 146 | 147 | self.send(msg) 148 | 149 | def send_html_deduped(self, jid, body, html, key): 150 | key = string.translate(str(key), self.jidtrans)[0:128] 151 | def checkedSend(is_new, jid, body, html): 152 | if is_new: 153 | self.send_html(jid, body, html) 154 | cache.mc.add(key, "x").addCallback(checkedSend, jid, body, html) 155 | 156 | def onError(self, msg): 157 | log.msg("Error received for %s: %s" % (msg['from'], msg.toXml())) 158 | scheduling.unavailable_user(JID(msg['from'])) 159 | 160 | def onMessage(self, msg): 161 | try: 162 | self.__onMessage(msg) 163 | except KeyError: 164 | log.err() 165 | 166 | def __onUserMessage(self, user, a, args, msg): 167 | cmd = self.commands.get(a[0].lower()) 168 | if cmd: 169 | cmd(user, self, args) 170 | else: 171 | d = None 172 | if user.auto_post: 173 | d=self.commands['post'] 174 | elif a[0][0] == '@': 175 | d=self.commands['post'] 176 | if d: 177 | d(user, self, unicode(msg.body).strip()) 178 | else: 179 | self.send_plain(msg['from'], 180 | "No such command: %s\n" 181 | "Send 'help' for known commands\n" 182 | "If you intended to post your message, " 183 | "please start your message with 'post', or see " 184 | "'help autopost'" % a[0]) 185 | 186 | def __onMessage(self, msg): 187 | if msg.getAttribute("type") == 'chat' and hasattr(msg, "body") and msg.body: 188 | self.typing_notification(msg['from']) 189 | a=unicode(msg.body).strip().split(None, 1) 190 | args = a[1] if len(a) > 1 else None 191 | db.User.by_jid(JID(msg['from']).userhost() 192 | ).addCallback(self.__onUserMessage, a, args, msg) 193 | else: 194 | log.msg("Non-chat/body message: %s" % msg.toXml()) 195 | 196 | class TwitterspyPresenceProtocol(PresenceClientProtocol): 197 | 198 | _tracking=-1 199 | _users=-1 200 | started = time.time() 201 | connected = None 202 | lost = None 203 | num_connections = 0 204 | 205 | def __init__(self, jid): 206 | super(TwitterspyPresenceProtocol, self).__init__() 207 | self.jid = jid.full() 208 | 209 | self.preferred = self.jid == config.CONF.get("xmpp", "jid") 210 | 211 | def connectionInitialized(self): 212 | super(TwitterspyPresenceProtocol, self).connectionInitialized() 213 | self._tracking=-1 214 | self._users=-1 215 | self.connected = time.time() 216 | self.lost = None 217 | self.num_connections += 1 218 | self.update_presence() 219 | 220 | global presence_conns, default_presence 221 | presence_conns[self.jid] = self 222 | if self.preferred: 223 | default_presence = self 224 | 225 | def connectionLost(self, reason): 226 | self.connected = None 227 | self.lost = time.time() 228 | 229 | def presence_fallback(self, *stuff): 230 | log.msg("Running presence fallback.") 231 | self.available(None, None, {None: "Hi, everybody!"}) 232 | 233 | def update_presence(self): 234 | try: 235 | if scheduling.available_requests > 0: 236 | self._update_presence_ready() 237 | else: 238 | self._update_presence_not_ready() 239 | except: 240 | log.err() 241 | 242 | def _update_presence_ready(self): 243 | def gotResult(counts): 244 | users = counts['users'] 245 | tracking = counts['tracks'] 246 | if tracking != self._tracking or users != self._users: 247 | status="Tracking %s topics for %s users" % (tracking, users) 248 | self.available(None, None, {None: status}) 249 | self._tracking = tracking 250 | self._users = users 251 | db.model_counts().addCallback(gotResult).addErrback(self.presence_fallback) 252 | 253 | def _update_presence_not_ready(self): 254 | status="Ran out of Twitter API requests." 255 | self.available(None, 'away', {None: status}) 256 | self._tracking = -1 257 | self._users = -1 258 | 259 | def availableReceived(self, entity, show=None, statuses=None, priority=0): 260 | log.msg("Available from %s (%s, %s, pri=%s)" % ( 261 | entity.full(), show, statuses, priority)) 262 | 263 | if priority >= 0 and show not in ['xa', 'dnd']: 264 | scheduling.available_user(entity) 265 | else: 266 | log.msg("Marking jid unavailable due to negative priority or " 267 | "being somewhat unavailable.") 268 | scheduling.unavailable_user(entity) 269 | self._find_and_set_status(entity.userhost(), show) 270 | 271 | def unavailableReceived(self, entity, statuses=None): 272 | log.msg("Unavailable from %s" % entity.full()) 273 | 274 | def cb(): 275 | scheduling.unavailable_user(entity) 276 | 277 | self._find_and_set_status(entity.userhost(), 'offline', cb) 278 | 279 | def subscribedReceived(self, entity): 280 | log.msg("Subscribe received from %s" % (entity.userhost())) 281 | welcome_message="""Welcome to twitterspy. 282 | 283 | Here you can use your normal IM client to post to twitter, track topics, watch 284 | your friends, make new ones, and more. 285 | 286 | Type "help" to get started. 287 | """ 288 | global current_conns 289 | conn = current_conns[self.jid] 290 | conn.send_plain(entity.full(), welcome_message) 291 | def send_notices(counts): 292 | cnt = counts['users'] 293 | msg = "New subscriber: %s ( %d )" % (entity.userhost(), cnt) 294 | for a in config.ADMINS: 295 | conn.send_plain(a, msg) 296 | db.model_counts().addCallback(send_notices) 297 | 298 | def _set_status(self, u, status, cb): 299 | 300 | # If we've got them on the preferred service, unsubscribe them 301 | # from this one. 302 | if not self.preferred and (u.service_jid and u.service_jid != self.jid): 303 | log.msg("Unsubscribing %s from non-preferred service %s" % ( 304 | u.jid, self.jid)) 305 | self.unsubscribe(JID(u.jid)) 306 | self.unsubscribed(JID(u.jid)) 307 | return 308 | 309 | modified = False 310 | 311 | j = self.jid 312 | if (not u.service_jid) or (self.preferred and u.service_jid != j): 313 | u.service_jid = j 314 | modified = True 315 | 316 | if u.status != status: 317 | u.status=status 318 | modified = True 319 | 320 | global service_mapping 321 | service_mapping[u.jid] = u.service_jid 322 | log.msg("Service mapping for %s is %s" % (u.jid, u.service_jid)) 323 | 324 | if modified: 325 | if cb: 326 | cb() 327 | return u.save() 328 | 329 | def _find_and_set_status(self, jid, status, cb=None): 330 | if status is None: 331 | status = 'available' 332 | def f(): 333 | db.User.by_jid(jid).addCallback(self._set_status, status, cb) 334 | scheduling.available_sem.run(f) 335 | 336 | def unsubscribedReceived(self, entity): 337 | log.msg("Unsubscribed received from %s" % (entity.userhost())) 338 | self._find_and_set_status(entity.userhost(), 'unsubscribed') 339 | self.unsubscribe(entity) 340 | self.unsubscribed(entity) 341 | 342 | def subscribeReceived(self, entity): 343 | log.msg("Subscribe received from %s" % (entity.userhost())) 344 | self.subscribe(entity) 345 | self.subscribed(entity) 346 | self.update_presence() 347 | 348 | def unsubscribeReceived(self, entity): 349 | log.msg("Unsubscribe received from %s" % (entity.userhost())) 350 | self._find_and_set_status(entity.userhost(), 'unsubscribed') 351 | self.unsubscribe(entity) 352 | self.unsubscribed(entity) 353 | self.update_presence() 354 | 355 | def conn_for(jid): 356 | return current_conns[service_mapping[jid]] 357 | 358 | def presence_for(jid): 359 | return presence_conns[service_mapping[jid]] 360 | 361 | def send_html_deduped(jid, plain, html, key): 362 | conn_for(jid).send_html_deduped(jid, plain, html, key) 363 | 364 | def send_html(jid, plain, html): 365 | conn_for(jid).send_html(jid, plain, html) 366 | 367 | def send_plain(jid, plain): 368 | conn_for(jid).send_plain(jid, plain) 369 | -------------------------------------------------------------------------------- /lib/twitterspy/scheduling.py: -------------------------------------------------------------------------------- 1 | import time 2 | import bisect 3 | import random 4 | import hashlib 5 | 6 | from twisted.python import log 7 | from twisted.internet import task, defer, reactor, threads 8 | from twisted.words.protocols.jabber.jid import JID 9 | from twisted.web import error 10 | 11 | import twitter 12 | import protocol 13 | 14 | import db 15 | import moodiness 16 | import cache 17 | import config 18 | import search_collector 19 | 20 | search_semaphore = defer.DeferredSemaphore(tokens=5) 21 | private_semaphore = defer.DeferredSemaphore(tokens=20) 22 | available_sem = defer.DeferredSemaphore(tokens=1) 23 | 24 | MAX_REQUESTS = 20000 25 | REQUEST_PERIOD = 3600 26 | 27 | QUERY_FREQUENCY = 15 * 60 28 | USER_FREQUENCY = 3 * 60 29 | 30 | TIMEOUT=5 31 | 32 | available_requests = MAX_REQUESTS 33 | reported_empty = False 34 | empty_resets = 0 35 | 36 | suspended_until = 0 37 | 38 | locks_requested = 0 39 | locks_acquired = 0 40 | 41 | def getTwitterAPI(*args): 42 | global available_requests, reported_empty 43 | now = time.time() 44 | if suspended_until > now: 45 | return ErrorGenerator() 46 | elif available_requests > 0: 47 | available_requests -= 1 48 | return twitter.Twitter(*args) 49 | else: 50 | if not reported_empty: 51 | reported_empty = True 52 | for conn in protocol.presence_conns.values(): 53 | conn.update_presence() 54 | log.msg("Out of requests. :(") 55 | # Return something that just generates deferreds that error. 56 | class ErrorGenerator(object): 57 | def __getattr__(self, attr): 58 | def error_generator(*args): 59 | return defer.fail( 60 | "There are no more available twitter requests.") 61 | return error_generator 62 | return ErrorGenerator() 63 | 64 | def resetRequests(): 65 | global available_requests, empty_resets, reported_empty 66 | if available_requests == 0: 67 | empty_resets += 1 68 | reported_empty = False 69 | available_requests = MAX_REQUESTS 70 | for conn in protocol.presence_conns.values(): 71 | conn.update_presence() 72 | log.msg("Available requests are reset to %d" % available_requests) 73 | 74 | class JidSet(set): 75 | 76 | def bare_jids(self): 77 | return set([JID(j).userhost() for j in self]) 78 | 79 | class Query(JidSet): 80 | 81 | loop_time = QUERY_FREQUENCY 82 | 83 | def __init__(self, query, last_id=0, getAPI=getTwitterAPI): 84 | super(Query, self).__init__() 85 | self.getAPI = getAPI 86 | self.query = query 87 | self.cache_key = self._compute_cache_key(query) 88 | self.loop = None 89 | 90 | cache.mc.get(self.cache_key).addCallback(self._doStart) 91 | 92 | def _compute_cache_key(self, query): 93 | return hashlib.md5(query.encode('utf-8')).hexdigest() 94 | 95 | def _doStart(self, res): 96 | if res[1]: 97 | self.last_id = res[1] 98 | # log.msg("Loaded last ID for %s from memcache: %s" 99 | # % (self.query, self.last_id)) 100 | else: 101 | log.msg("No last ID for %s" % (self.query,)) 102 | self.last_id = 0 103 | r=random.Random() 104 | then = r.randint(1, min(60, self.loop_time / 2)) 105 | # log.msg("Starting %s in %ds" % (self.query, then)) 106 | reactor.callLater(then, self.start) 107 | 108 | def _sendMessages(self, something, results): 109 | first_shot = self.last_id == 0 110 | self.last_id = results.last_id 111 | if not first_shot: 112 | def send(r): 113 | for eid, plain, html in results.results: 114 | for jid in self.bare_jids(): 115 | key = str(eid) + "@" + jid 116 | protocol.send_html_deduped(jid, plain, html, key) 117 | dl = defer.DeferredList(results.deferreds) 118 | dl.addCallback(send) 119 | 120 | def __call__(self): 121 | # Don't bother if we're not connected... 122 | if protocol.current_conns: 123 | global search_semaphore, locks_requested 124 | locks_requested += 1 125 | # log.msg("Acquiring lock for %s" % self.query) 126 | d = search_semaphore.run(self._do_search) 127 | def _complete(x): 128 | global locks_requested, locks_acquired 129 | locks_requested -= 1 130 | locks_acquired -= 1 131 | # log.msg("Released lock for %s" % self.query) 132 | d.addBoth(_complete) 133 | else: 134 | log.msg("No xmpp connection, so skipping search of %s" % self.query) 135 | 136 | def _reportError(self, e): 137 | if int(e.value.status) == 420: 138 | global available_requests 139 | suspended_until = time.time() + 5 140 | log.msg("Twitter is reporting that we're out of stuff: " % str(e)) 141 | else: 142 | log.msg("Error in search %s: %s" % (self.query, str(e))) 143 | 144 | def _save_track_id(self, x, old_id): 145 | if old_id != self.last_id: 146 | cache.mc.set(self.cache_key, str(self.last_id)) 147 | 148 | def _do_search(self): 149 | global locks_acquired 150 | locks_acquired += 1 151 | # log.msg("Searching %s" % self.query) 152 | params = {} 153 | if self.last_id > 0: 154 | params['since_id'] = str(self.last_id) 155 | results=search_collector.SearchCollector(self.last_id) 156 | return self.getAPI().search(self.query, results.gotResult, 157 | params 158 | ).addCallback(moodiness.moodiness.markSuccess 159 | ).addErrback(moodiness.moodiness.markFailure 160 | ).addCallback(self._sendMessages, results 161 | ).addCallback(self._save_track_id, self.last_id 162 | ).addErrback(self._reportError) 163 | 164 | def start(self): 165 | self.loop = task.LoopingCall(self) 166 | d = self.loop.start(self.loop_time) 167 | d.addCallback(lambda x: log.msg("Search query for %s has been stopped: %s" 168 | % (self.query, x))) 169 | d.addErrback(lambda e: log.err("Search query for %s has errored: %s" 170 | % (self.query, e))) 171 | 172 | def stop(self): 173 | log.msg("Stopping query %s" % self.query) 174 | if self.loop: 175 | self.loop.stop() 176 | self.loop = None 177 | 178 | class QueryRegistry(object): 179 | 180 | def __init__(self, getAPI=getTwitterAPI): 181 | self.queries = {} 182 | self.getAPI = getAPI 183 | 184 | def add(self, user, query_str, last_id=0): 185 | # log.msg("Adding %s: %s" % (user, query_str)) 186 | if not self.queries.has_key(query_str): 187 | self.queries[query_str] = Query(query_str, last_id, self.getAPI) 188 | self.queries[query_str].add(user) 189 | 190 | def untracked(self, user, query): 191 | q = self.queries.get(query) 192 | if q: 193 | if user in q: 194 | log.msg("Untracking %s from %s" % (query, user)) 195 | q.discard(user) 196 | if not q: 197 | q.stop() 198 | del self.queries[query] 199 | else: 200 | log.msg("Query %s not found when untracking." % query) 201 | 202 | def remove(self, user): 203 | log.msg("Removing %s" % user) 204 | for k in list(self.queries.keys()): 205 | self.untracked(user, k) 206 | 207 | def remove_user(self, user, jids): 208 | for k in list(self.queries.keys()): 209 | for j in jids: 210 | self.untracked(j, k) 211 | 212 | def __len__(self): 213 | return len(self.queries) 214 | 215 | class UserStuff(JidSet): 216 | 217 | loop_time = USER_FREQUENCY 218 | 219 | def __init__(self, short_jid, friends_id, dm_id): 220 | super(UserStuff, self).__init__() 221 | self.short_jid = short_jid 222 | self.last_friend_id = friends_id 223 | self.last_dm_id = dm_id 224 | 225 | self.username = None 226 | self.password = None 227 | self.loop = None 228 | 229 | def _format_message(self, type, entry, results): 230 | s = getattr(entry, 'sender', None) 231 | if not s: 232 | s=entry.user 233 | u = s.screen_name 234 | plain="[%s] %s: %s" % (type, u, entry.text) 235 | aurl = "https://twitter.com/" + u 236 | htype = '' + type + '' 237 | html="[%s] %s: %s" % (htype, aurl, u, entry.text) 238 | bisect.insort(results, (entry.id, plain, html)) 239 | 240 | def _deliver_messages(self, whatever, messages): 241 | for eid, plain, html in messages: 242 | for jid in self.bare_jids(): 243 | key = str(eid) + "@" + jid 244 | protocol.send_html_deduped(jid, plain, html, key) 245 | 246 | def _gotDMResult(self, results): 247 | def f(entry): 248 | self.last_dm_id = max(self.last_dm_id, int(entry.id)) 249 | self._format_message('direct', entry, results) 250 | return f 251 | 252 | def _gotFriendsResult(self, results): 253 | def f(entry): 254 | self.last_friend_id = max(self.last_friend_id, int(entry.id)) 255 | self._format_message('friend', entry, results) 256 | return f 257 | 258 | def _deferred_write(self, u, mprop, new_val): 259 | setattr(u, mprop, new_val) 260 | u.save() 261 | 262 | def _maybe_update_prop(self, prop, mprop): 263 | old_val = getattr(self, prop) 264 | def f(x): 265 | new_val = getattr(self, prop) 266 | if old_val != new_val: 267 | db.User.by_jid(self.short_jid).addCallback(self._deferred_write, 268 | mprop, new_val) 269 | return f 270 | 271 | def __call__(self): 272 | if self.username and self.password and protocol.current_conns: 273 | global private_semaphore 274 | private_semaphore.run(self._get_user_stuff) 275 | 276 | def _cleanup401s(self, e): 277 | e.trap(error.Error) 278 | if int(e.value.status) == 401: 279 | log.msg("Error 401 getting user data for %s, disabling" 280 | % self.short_jid) 281 | self.stop() 282 | else: 283 | log.msg("Unknown http error: %s: %s" % (e.value.status, str(e))) 284 | 285 | def _reportError(self, e): 286 | log.msg("Error getting user data for %s: %s" % (self.short_jid, str(e))) 287 | 288 | def _get_user_stuff(self): 289 | log.msg("Getting privates for %s" % self.short_jid) 290 | params = {} 291 | if self.last_dm_id > 0: 292 | params['since_id'] = str(self.last_dm_id) 293 | tw = getTwitterAPI(self.username, self.password) 294 | dm_list=[] 295 | tw.direct_messages(self._gotDMResult(dm_list), params).addCallback( 296 | self._maybe_update_prop('last_dm_id', 'direct_message_id') 297 | ).addCallback(self._deliver_messages, dm_list 298 | ).addErrback(self._cleanup401s).addErrback(self._reportError) 299 | 300 | if self.last_friend_id is not None: 301 | friend_list=[] 302 | tw.friends(self._gotFriendsResult(friend_list), 303 | {'since_id': str(self.last_friend_id)}).addCallback( 304 | self._maybe_update_prop( 305 | 'last_friend_id', 'friend_timeline_id') 306 | ).addCallback(self._deliver_messages, friend_list 307 | ).addErrback(self._cleanup401s).addErrback(self._reportError) 308 | 309 | def start(self): 310 | log.msg("Starting %s" % self.short_jid) 311 | self.loop = task.LoopingCall(self) 312 | self.loop.start(self.loop_time, now=False) 313 | 314 | def stop(self): 315 | if self.loop: 316 | log.msg("Stopping user %s" % self.short_jid) 317 | self.loop.stop() 318 | self.loop = None 319 | 320 | class UserRegistry(object): 321 | 322 | def __init__(self): 323 | self.users = {} 324 | 325 | def add(self, short_jid, full_jid, friends_id, dm_id): 326 | log.msg("Adding %s as %s" % (short_jid, full_jid)) 327 | if not self.users.has_key(short_jid): 328 | self.users[short_jid] = UserStuff(short_jid, friends_id, dm_id) 329 | self.users[short_jid].add(full_jid) 330 | 331 | def set_creds(self, short_jid, un, pw): 332 | u=self.users.get(short_jid) 333 | if u: 334 | u.username = un 335 | u.password = pw 336 | available = un and pw 337 | if available and not u.loop: 338 | u.start() 339 | elif u.loop and not available: 340 | u.stop() 341 | else: 342 | log.msg("Couldn't find %s to set creds" % short_jid) 343 | 344 | def __len__(self): 345 | return len(self.users) 346 | 347 | def remove(self, short_jid, full_jid=None): 348 | q = self.users.get(short_jid) 349 | if not q: 350 | return 351 | q.discard(full_jid) 352 | if not q: 353 | q.stop() 354 | del self.users[short_jid] 355 | 356 | queries = QueryRegistry() 357 | users = UserRegistry() 358 | 359 | def _entity_to_jid(entity): 360 | return entity if isinstance(entity, basestring) else entity.userhost() 361 | 362 | def __init_user(entity, jids=[]): 363 | jid = _entity_to_jid(entity) 364 | def f(u): 365 | if u.active: 366 | full_jids = users.users.get(jid, jids) 367 | for j in full_jids: 368 | users.add(jid, j, u.friend_timeline_id, u.direct_message_id) 369 | for q in u.tracks: 370 | queries.add(j, q) 371 | users.set_creds(jid, u.username, u.decoded_password) 372 | db.User.by_jid(jid).addCallback(f) 373 | 374 | def enable_user(jid): 375 | global available_sem 376 | available_sem.run(__init_user, jid) 377 | 378 | def disable_user(jid): 379 | queries.remove_user(jid, users.users.get(jid, [])) 380 | users.set_creds(jid, None, None) 381 | 382 | def available_user(entity): 383 | global available_sem 384 | available_sem.run(__init_user, entity, [entity.full()]) 385 | 386 | def unavailable_user(entity): 387 | queries.remove(entity.full()) 388 | users.remove(entity.userhost(), entity.full()) 389 | 390 | def resources(jid): 391 | """Find all watched resources for the given JID.""" 392 | jids=users.users.get(jid, []) 393 | return [JID(j).resource for j in jids] 394 | 395 | def _reset_all(): 396 | global queries 397 | global users 398 | for q in queries.queries.values(): 399 | q.clear() 400 | q.stop() 401 | for u in users.users.values(): 402 | u.clear() 403 | u.stop() 404 | queries = QueryRegistry() 405 | users = UserRegistry() 406 | 407 | def connected(): 408 | # _reset_all() 409 | pass 410 | 411 | def disconnected(): 412 | # _reset_all() 413 | pass 414 | -------------------------------------------------------------------------------- /lib/twitterspy/search_collector.py: -------------------------------------------------------------------------------- 1 | import bisect 2 | 3 | from twisted.python import log 4 | 5 | import url_expansion 6 | 7 | class SearchCollector(object): 8 | 9 | def __init__(self, last_id=0): 10 | self.results=[] 11 | self.last_id = last_id 12 | self.deferreds = [] 13 | 14 | def gotResult(self, entry): 15 | eid = int(entry.id.split(':')[-1]) 16 | self.last_id = max(self.last_id, eid) 17 | u = entry.author.name.split(' ')[0] 18 | plain=u + ": " + entry.title 19 | hcontent=entry.content.replace("<", "<" 20 | ).replace(">", ">" 21 | ).replace('&', '&') 22 | html="%s: %s" % (entry.author.uri, u, hcontent) 23 | def errHandler(e): 24 | log.err(e) 25 | return plain, html 26 | def saveResults(t): 27 | p, h = t 28 | bisect.insort(self.results, (eid, p, h)) 29 | d = url_expansion.expander.expand(plain, html).addErrback( 30 | errHandler).addCallback(saveResults) 31 | self.deferreds.append(d) 32 | -------------------------------------------------------------------------------- /lib/twitterspy/url_expansion.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from twisted.internet import task, reactor, defer 4 | from twisted.python import log 5 | 6 | import cache 7 | import longurl 8 | 9 | class BasicUrl(object): 10 | 11 | def __init__(self, title, url): 12 | self.title = title 13 | self.url = url 14 | 15 | class Expander(object): 16 | 17 | cache = True 18 | 19 | def __init__(self): 20 | self.lu = longurl.LongUrl('twitterspy') 21 | self.regex = None 22 | 23 | def loadServices(self): 24 | def _e(e): 25 | log.msg("Error loading expansion rules. Trying again in 5s") 26 | reactor.callLater(5, self.loadServices) 27 | self.lu.getServices().addCallback(self._registerServices).addErrback(_e) 28 | 29 | def _registerServices(self, svcs): 30 | domains = set() 31 | for s in svcs.values(): 32 | domains.update(s.domains) 33 | 34 | self.regex_str = "(http://(" + '|'.join(self.__fixup(d) for d in domains) + r")/\S+)" 35 | self.regex = re.compile(self.regex_str) 36 | 37 | def __fixup(self, d): 38 | return d.replace('.', r'\.') 39 | 40 | def _e(self, u): 41 | return u.replace("&", "&") 42 | 43 | def expand(self, plain, html=None): 44 | rv = defer.Deferred() 45 | 46 | m = self.regex and self.regex.search(plain) 47 | if m: 48 | u, k = m.groups() 49 | def gotErr(e): 50 | log.err(e) 51 | reactor.callWhenRunning(rv.callback, (plain, html)) 52 | def gotRes(res): 53 | # Sometimes, the expander returns its input. That sucks. 54 | if res.url == u: 55 | plainSub = plain 56 | htmlSub = html 57 | else: 58 | plainSub = plain.encode('utf-8').replace( 59 | u, "%s (from %s)" % (self._e(res.url), u)) 60 | if html: 61 | htmlSub = html.encode('utf-8').replace( 62 | u, "%s" % (self._e(res.url),)) 63 | else: 64 | htmlSub = None 65 | log.msg("rewrote %s to %s" % (plain, plainSub)) 66 | reactor.callWhenRunning(rv.callback, (plainSub, htmlSub)) 67 | self._expand(u).addCallback(gotRes).addErrback(gotErr) 68 | else: 69 | # No match, immediately hand the message back. 70 | reactor.callWhenRunning(rv.callback, (plain, html)) 71 | 72 | return rv 73 | 74 | def _cached_lookup(self, u, mc): 75 | rv = defer.Deferred() 76 | 77 | def identity(ignored_param): 78 | rv.callback(BasicUrl(None, u)) 79 | 80 | def mc_res(res): 81 | if res[1]: 82 | rv.callback(BasicUrl(None, res[1])) 83 | else: 84 | def save_res(lu_res): 85 | if lu_res: 86 | mc.set(u, lu_res.url.encode('utf-8')) 87 | rv.callback(BasicUrl(None, lu_res.url)) 88 | else: 89 | log.msg("No response found for %s" % u) 90 | rv.callback(BasicUrl(None, u)) 91 | self.lu.expand(u).addCallback(save_res).addErrback(identity) 92 | 93 | mc.get(u).addCallback(mc_res).addErrback(identity) 94 | 95 | return rv 96 | 97 | def _expand(self, u): 98 | if self.cache: 99 | import protocol 100 | if cache.mc: 101 | return self._cached_lookup(u.encode('utf-8'), cache.mc) 102 | else: 103 | return self.lu.expand(u) 104 | else: 105 | return self.lu.expand(u) 106 | 107 | expander = Expander() 108 | -------------------------------------------------------------------------------- /lib/twitterspy/xmpp_commands.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import re 3 | import sys 4 | import time 5 | import types 6 | import base64 7 | import datetime 8 | import urlparse 9 | import sre_constants 10 | 11 | from twisted.python import log 12 | from twisted.words.xish import domish 13 | from twisted.words.protocols.jabber.jid import JID 14 | from twisted.web import client 15 | from twisted.internet import reactor, threads, defer 16 | from wokkel import ping 17 | 18 | import db 19 | 20 | import config 21 | import twitter 22 | import scheduling 23 | import search_collector 24 | import protocol 25 | import moodiness 26 | 27 | all_commands={} 28 | 29 | def arg_required(validator=lambda n: n): 30 | def f(orig): 31 | def every(self, user, prot, args): 32 | if validator(args): 33 | orig(self, user, prot, args) 34 | else: 35 | prot.send_plain(user.jid, "Arguments required for %s:\n%s" 36 | % (self.name, self.extended_help)) 37 | return every 38 | return f 39 | 40 | def login_required(orig): 41 | def every(self, user, prot, args): 42 | if user.has_credentials: 43 | orig(self, user, prot, args) 44 | else: 45 | prot.send_plain(user.jid, "You must twlogin before calling %s" 46 | % self.name) 47 | return every 48 | 49 | def admin_required(orig): 50 | def every(self, user, prot, args): 51 | if user.is_admin: 52 | orig(self, user, prot, args) 53 | else: 54 | prot.send_plain(user.jid, "You're not an admin.") 55 | return every 56 | 57 | class BaseCommand(object): 58 | """Base class for command processors.""" 59 | 60 | def __get_extended_help(self): 61 | if self.__extended_help: 62 | return self.__extended_help 63 | else: 64 | return self.help 65 | 66 | def __set_extended_help(self, v): 67 | self.__extended_help=v 68 | 69 | extended_help=property(__get_extended_help, __set_extended_help) 70 | 71 | def __init__(self, name, help=None, extended_help=None, aliases=[]): 72 | self.name=name 73 | self.help=help 74 | self.aliases=aliases 75 | self.extended_help=extended_help 76 | 77 | def __call__(self, user, prot, args, session): 78 | raise NotImplementedError() 79 | 80 | def is_a_url(self, u): 81 | try: 82 | parsed = urlparse.urlparse(str(u)) 83 | return parsed.scheme in ['http', 'https'] and parsed.netloc 84 | except: 85 | return False 86 | 87 | class BaseStatusCommand(BaseCommand): 88 | 89 | def get_user_status(self, user): 90 | rv=[] 91 | rv.append("Jid: %s" % user.jid) 92 | rv.append("Twitterspy status: %s" 93 | % {True: 'Active', None: 'Inactive', False: 'Inactive'}[user.active]) 94 | rv.append("Service jid: %s" % user.service_jid) 95 | rv.append("Autopost: %s" 96 | % {True: 'on', None: 'off', False: 'off'}[user.auto_post]) 97 | resources = scheduling.resources(user.jid) 98 | if resources: 99 | rv.append("I see you logged in with the following resources:") 100 | for r in resources: 101 | rv.append(u" • %s" % r) 102 | else: 103 | rv.append("I don't see you logged in with any resource I'd send " 104 | "a message to. Perhaps you're dnd, xa, or negative priority.") 105 | rv.append("You are currently tracking %d topics." % len(user.tracks)) 106 | if user.has_credentials: 107 | rv.append("You're logged in to twitter as %s" % (user.username)) 108 | if user.friend_timeline_id is not None: 109 | rv.append("Friend tracking is enabled.") 110 | return "\n".join(rv) 111 | 112 | class StatusCommand(BaseStatusCommand): 113 | 114 | def __init__(self): 115 | super(StatusCommand, self).__init__('status', 'Check your status.') 116 | 117 | def __call__(self, user, prot, args): 118 | prot.send_plain(user.jid, self.get_user_status(user)) 119 | 120 | class HelpCommand(BaseCommand): 121 | 122 | def __init__(self): 123 | super(HelpCommand, self).__init__('help', 'You need help.') 124 | 125 | def __call__(self, user, prot, args): 126 | rv=[] 127 | if args and args.strip(): 128 | c=all_commands.get(args.strip().lower(), None) 129 | if c: 130 | rv.append("Help for %s:\n" % c.name) 131 | rv.append(c.extended_help) 132 | if c.aliases: 133 | rv.append("\nAliases:\n * ") 134 | "\n * ".join(c.aliases) 135 | else: 136 | rv.append("Unknown command %s." % args) 137 | else: 138 | for k in sorted(all_commands.keys()): 139 | if (not k.startswith('adm_')) or user.is_admin: 140 | rv.append('%s\t%s' % (k, all_commands[k].help)) 141 | rv.append("\nFor more help, see http://dustin.github.com/twitterspy/") 142 | prot.send_plain(user.jid, "\n".join(rv)) 143 | 144 | class OnCommand(BaseCommand): 145 | def __init__(self): 146 | super(OnCommand, self).__init__('on', 'Enable tracks.') 147 | 148 | def __call__(self, user, prot, args): 149 | user.active=True 150 | def worked(stuff): 151 | scheduling.available_user(JID(user.jid)) 152 | prot.send_plain(user.jid, "Enabled tracks.") 153 | def notWorked(e): 154 | log.err(e) 155 | prot.send_plain(user.jid, "Failed to enable. Try again.") 156 | user.save().addCallback(worked).addErrback(notWorked) 157 | 158 | class OffCommand(BaseCommand): 159 | def __init__(self): 160 | super(OffCommand, self).__init__('off', 'Disable tracks.') 161 | 162 | def __call__(self, user, prot, args): 163 | user.active=False 164 | def worked(stuff): 165 | scheduling.disable_user(user.jid) 166 | prot.send_plain(user.jid, "Disabled tracks.") 167 | def notWorked(e): 168 | log.err(e) 169 | prot.send_plain(user.jid, "Failed to disable. Try again.") 170 | user.save().addCallback(worked).addErrback(notWorked) 171 | 172 | class SearchCommand(BaseCommand): 173 | 174 | def __init__(self): 175 | super(SearchCommand, self).__init__('search', 176 | 'Perform a search query (but do not track).') 177 | 178 | def _success(self, e, jid, prot, query, rv): 179 | # log.msg("%d results found for %s" % (len(rv.results), query)) 180 | def send(r): 181 | plain = [] 182 | html = [] 183 | for eid, p, h in rv.results: 184 | plain.append(p) 185 | html.append(h) 186 | prot.send_html(jid, str(len(rv.results)) 187 | + " results for " + query 188 | + "\n\n" + "\n\n".join(plain), 189 | str(len(rv.results)) + " results for " 190 | + query + "
\n
\n" 191 | + "
\n
\n".join(html)) 192 | defer.DeferredList(rv.deferreds).addCallback(send) 193 | 194 | def _error(self, e, jid, prot): 195 | mood, good, lrr, percentage = moodiness.moodiness.current_mood() 196 | rv = [":( Problem performing search."] 197 | if percentage > 0.5: 198 | rv.append("%.1f%% of recent searches have worked (%d out of %d)" 199 | % ((percentage * 100.0), good, lrr)) 200 | elif good == 0: 201 | rv.append("This is not surprising, " 202 | "no recent searches worked for me, either (%d out of %d)" 203 | % (good, lrr)) 204 | else: 205 | rv.append("This is not surprising -- only %.1f%% work now anyway (%d out of %d)" 206 | % ((percentage * 100.0), good, lrr)) 207 | prot.send_plain(jid, "\n".join(rv)) 208 | return e 209 | 210 | def _do_search(self, query, jid, prot): 211 | rv = search_collector.SearchCollector() 212 | scheduling.getTwitterAPI().search(query, rv.gotResult, {'rpp': '3'} 213 | ).addCallback(moodiness.moodiness.markSuccess 214 | ).addErrback(moodiness.moodiness.markFailure 215 | ).addCallback(self._success, jid, prot, query, rv 216 | ).addErrback(self._error, jid, prot 217 | ).addErrback(log.err) 218 | 219 | @arg_required() 220 | def __call__(self, user, prot, args): 221 | scheduling.search_semaphore.run(self._do_search, args, user.jid, prot) 222 | 223 | class TWLoginCommand(BaseCommand): 224 | 225 | def __init__(self): 226 | super(TWLoginCommand, self).__init__('twlogin', 227 | 'Set your twitter username and password (use at your own risk)') 228 | 229 | @arg_required() 230 | def __call__(self, user, prot, args): 231 | args = args.replace(">", "").replace("<", "") 232 | username, password=args.split(' ', 1) 233 | jid = user.jid 234 | scheduling.getTwitterAPI(username, password 235 | ).direct_messages(lambda x: None).addCallback( 236 | self.__credsVerified, prot, jid, username, password, user).addErrback( 237 | self.__credsRefused, prot, jid) 238 | 239 | def __credsRefused(self, e, prot, jid): 240 | log.msg("Failed to verify creds for %s: %s" % (jid, e)) 241 | prot.send_plain(jid, 242 | ":( Your credentials were refused. " 243 | "Please try again: twlogin username password") 244 | 245 | def __credsVerified(self, x, prot, jid, username, password, user): 246 | user.username = username 247 | user.password = base64.encodestring(password) 248 | def worked(stuff): 249 | prot.send_plain(user.jid, "Added credentials for %s" 250 | % user.username) 251 | scheduling.users.set_creds(jid, username, password) 252 | def notWorked(stuff): 253 | log.err() 254 | prot.send_plain(user.jid, "Error setting credentials for %s. " 255 | "Please try again." % user.username) 256 | user.save().addCallback(worked).addErrback(notWorked) 257 | 258 | class TWLogoutCommand(BaseCommand): 259 | 260 | def __init__(self): 261 | super(TWLogoutCommand, self).__init__('twlogout', 262 | "Discard your twitter credentials.") 263 | 264 | def __call__(self, user, prot, args): 265 | user.username = None 266 | user.password = None 267 | def worked(stuff): 268 | prot.send_plain(user.jid, "You have been logged out.") 269 | scheduling.users.set_creds(user.jid, None, None) 270 | def notWorked(e): 271 | log.err(e) 272 | prot.send_plain(user.jid, "Failed to log you out. Try again.") 273 | user.save().addCallback(worked).addErrback(notWorked) 274 | 275 | class TrackCommand(BaseCommand): 276 | 277 | def __init__(self): 278 | super(TrackCommand, self).__init__('track', "Start tracking a topic.") 279 | 280 | @arg_required() 281 | def __call__(self, user, prot, args): 282 | user.track(args) 283 | if user.active: 284 | rv = "Tracking %s" % args 285 | else: 286 | rv = "Will track %s as soon as you activate again." % args 287 | 288 | def worked(stuff): 289 | if user.active: 290 | scheduling.queries.add(user.jid, args, 0) 291 | prot.send_plain(user.jid, rv) 292 | def notWorked(e): 293 | log.err(e) 294 | prot.send_plain(user.jid, ":( Failed to save your tracks. Try again.") 295 | user.save().addCallback(worked).addErrback(notWorked) 296 | 297 | class UnTrackCommand(BaseCommand): 298 | 299 | def __init__(self): 300 | super(UnTrackCommand, self).__init__('untrack', 301 | "Stop tracking a topic.") 302 | 303 | @arg_required() 304 | def __call__(self, user, prot, args): 305 | log.msg("Untracking %s for %s" % (repr(args), user.jid)) 306 | if user.untrack(args): 307 | def worked(stuff): 308 | log.msg("Untrack %s for %s successful." % (repr(args), user.jid)) 309 | scheduling.queries.untracked(user.jid, args) 310 | prot.send_plain(user.jid, "Stopped tracking %s" % args) 311 | def notWorked(e): 312 | log.msg("Untrack %s for %s failed." % (repr(args), user.jid)) 313 | log.err(e) 314 | prot.send_plain(user.jid, ":( Failed to save tracks. Try again") 315 | user.save().addCallback(worked).addErrback(notWorked) 316 | else: 317 | prot.send_plain(user.jid, 318 | "Didn't untrack %s (sure you were tracking it?)" % args) 319 | 320 | class TracksCommand(BaseCommand): 321 | 322 | def __init__(self): 323 | super(TracksCommand, self).__init__('tracks', 324 | "List the topics you're tracking.", aliases=['tracking']) 325 | 326 | def __call__(self, user, prot, args): 327 | rv = ["Currently tracking:\n"] 328 | rv.extend(sorted(user.tracks)) 329 | prot.send_plain(user.jid, "\n".join(rv)) 330 | 331 | class PostCommand(BaseCommand): 332 | 333 | def __init__(self): 334 | super(PostCommand, self).__init__('post', 335 | "Post a message to twitter.") 336 | 337 | def _posted(self, id, jid, username, prot): 338 | url = "https://twitter.com/%s/statuses/%s" % (username, id) 339 | prot.send_plain(jid, ":) Your message has been posted: %s" % url) 340 | 341 | def _failed(self, e, jid, prot): 342 | log.msg("Error updating for %s: %s" % (jid, str(e))) 343 | prot.send_plain(jid, ":( Failed to post your message. " 344 | "Your password may be wrong, or twitter may be broken.") 345 | 346 | @arg_required() 347 | def __call__(self, user, prot, args): 348 | if user.has_credentials: 349 | jid = user.jid 350 | scheduling.getTwitterAPI(user.username, user.decoded_password).update( 351 | args, 'twitterspy' 352 | ).addCallback(self._posted, jid, user.username, prot 353 | ).addErrback(self._failed, jid, prot) 354 | else: 355 | prot.send_plain(user.jid, "You must twlogin before you can post.") 356 | 357 | class FollowCommand(BaseCommand): 358 | 359 | def __init__(self): 360 | super(FollowCommand, self).__init__('follow', 361 | "Begin following a user.") 362 | 363 | def _following(self, e, jid, prot, user): 364 | prot.send_plain(jid, ":) Now following %s" % user) 365 | 366 | def _failed(self, e, jid, prot, user): 367 | log.msg("Failed a follow request %s" % repr(e)) 368 | prot.send_plain(jid, ":( Failed to follow %s" % user) 369 | 370 | @arg_required() 371 | @login_required 372 | def __call__(self, user, prot, args): 373 | scheduling.getTwitterAPI(user.username, user.decoded_password).follow( 374 | str(args)).addCallback(self._following, user.jid, prot, args 375 | ).addErrback(self._failed, user.jid, prot, args) 376 | 377 | class LeaveUser(BaseCommand): 378 | 379 | def __init__(self): 380 | super(LeaveUser, self).__init__('leave', 381 | "Stop following a user.", aliases=['unfollow']) 382 | 383 | def _left(self, e, jid, prot, user): 384 | prot.send_plain(jid, ":) No longer following %s" % user) 385 | 386 | def _failed(self, e, jid, prot, user): 387 | log.msg("Failed an unfollow request: %s", repr(e)) 388 | prot.send_plain(jid, ":( Failed to stop following %s" % user) 389 | 390 | @arg_required() 391 | @login_required 392 | def __call__(self, user, prot, args): 393 | scheduling.getTwitterAPI(user.username, user.decoded_password).leave( 394 | str(args)).addCallback(self._left, user.jid, prot, args 395 | ).addErrback(self._failed, user.jid, prot, args) 396 | 397 | class BlockCommand(BaseCommand): 398 | 399 | def __init__(self): 400 | super(BlockCommand, self).__init__('block', 401 | "Block a user.") 402 | 403 | def _blocked(self, e, jid, prot, user): 404 | prot.send_plain(jid, ":) Now blocking %s" % user) 405 | 406 | def _failed(self, e, jid, prot, user): 407 | log.msg("Failed a block request %s" % repr(e)) 408 | prot.send_plain(jid, ":( Failed to block %s" % user) 409 | 410 | @arg_required() 411 | @login_required 412 | def __call__(self, user, prot, args): 413 | scheduling.getTwitterAPI(user.username, user.decoded_password).block( 414 | str(args)).addCallback(self._blocked, user.jid, prot, args 415 | ).addErrback(self._failed, user.jid, prot, args) 416 | 417 | class UnblockCommand(BaseCommand): 418 | 419 | def __init__(self): 420 | super(UnblockCommand, self).__init__('unblock', 421 | "Unblock a user.") 422 | 423 | def _left(self, e, jid, prot, user): 424 | prot.send_plain(jid, ":) No longer blocking %s" % user) 425 | 426 | def _failed(self, e, jid, prot, user): 427 | log.msg("Failed an unblock request: %s", repr(e)) 428 | prot.send_plain(jid, ":( Failed to unblock %s" % user) 429 | 430 | @arg_required() 431 | @login_required 432 | def __call__(self, user, prot, args): 433 | scheduling.getTwitterAPI(user.username, user.decoded_password).unblock( 434 | str(args)).addCallback(self._left, user.jid, prot, args 435 | ).addErrback(self._failed, user.jid, prot, args) 436 | 437 | def must_be_on_or_off(args): 438 | return args and args.lower() in ["on", "off"] 439 | 440 | class AutopostCommand(BaseCommand): 441 | 442 | def __init__(self): 443 | super(AutopostCommand, self).__init__('autopost', 444 | "Enable or disable autopost.") 445 | 446 | @arg_required(must_be_on_or_off) 447 | def __call__(self, user, prot, args): 448 | user.auto_post = (args.lower() == "on") 449 | def worked(stuff): 450 | prot.send_plain(user.jid, "Autoposting is now %s." % (args.lower())) 451 | def notWorked(e): 452 | prot.send_plain(user.jid, "Problem saving autopost. Try again.") 453 | user.save().addCallback(worked).addErrback(notWorked) 454 | 455 | class WatchFriendsCommand(BaseCommand): 456 | 457 | def __init__(self): 458 | super(WatchFriendsCommand, self).__init__( 459 | 'watch_friends', 460 | "Enable or disable watching friends (watch_friends on|off)", 461 | aliases=['watchfriends']) 462 | 463 | def _gotFriendStatus(self, user, prot): 464 | def f(entry): 465 | user.friend_timeline_id = entry.id 466 | def worked(stuff): 467 | # Tell scheduling so the process may begin. 468 | scheduling.users.set_creds(user.jid, 469 | user.username, user.decoded_password) 470 | prot.send_plain(user.jid, ":) Starting to watch friends.") 471 | def notWorked(e): 472 | prot.send_plain(user.jid, ":( Error watching friends. Try again.") 473 | user.save().addCallback(worked).addErrback(notWorked) 474 | return f 475 | 476 | @arg_required(must_be_on_or_off) 477 | @login_required 478 | def __call__(self, user, prot, args): 479 | args = args.lower() 480 | if args == 'on': 481 | scheduling.getTwitterAPI(user.username, user.decoded_password).friends( 482 | self._gotFriendStatus(user, prot), params={'count': '1'}) 483 | elif args == 'off': 484 | user.friend_timeline_id = None 485 | # Disable the privates search. 486 | scheduling.users.set_creds(user.jid, None, None) 487 | def worked(stuff): 488 | prot.send_plain(user.jid, ":) No longer watching your friends.") 489 | def notWorked(e): 490 | prot.send_plain(user.jid, ":( Problem stopping friend watches. Try again.") 491 | user.save().addCallback(worked).addErrback(notWorked) 492 | else: 493 | prot.send_plain(user.jid, "Watch must be 'on' or 'off'.") 494 | 495 | class WhoisCommand(BaseCommand): 496 | 497 | def __init__(self): 498 | super(WhoisCommand, self).__init__('whois', 499 | 'Find out who a user is.') 500 | 501 | def _fail(self, e, prot, jid, u): 502 | prot.send_plain(user.jid, "Couldn't get info for %s" % u) 503 | 504 | def _gotUser(self, u, prot, jid): 505 | html="""Whois %(screen_name)s

507 | Name: %(name)s
508 | Home: %(url)s
509 | Where: %(location)s
510 | Friends: %(friends_count)s
512 | Followers: %(followers_count)s
514 | Recently:
515 | %(status_text)s 516 | """ 517 | params = dict(u.__dict__) 518 | params['status_text'] = u.status.text 519 | prot.send_html(jid, "(no plain text yet)", html % params) 520 | 521 | @arg_required() 522 | @login_required 523 | def __call__(self, user, prot, args): 524 | scheduling.getTwitterAPI(user.username, user.decoded_password).show_user( 525 | str(args)).addErrback(self._fail, prot, user.jid, args 526 | ).addCallback(self._gotUser, prot, user.jid) 527 | 528 | class Top10Command(BaseCommand): 529 | 530 | def __init__(self): 531 | super(Top10Command, self).__init__('top10', 532 | 'Get the top10 most common tracks.') 533 | 534 | def __call__(self, user, prot, args): 535 | def worked(top10): 536 | rv=["Top 10 most tracked topics:"] 537 | rv.append("") 538 | rv.extend(["%s (%d watchers)" % (row[1], row[0]) for row in top10]) 539 | prot.send_plain(user.jid, "\n".join(rv)) 540 | def notWorked(e): 541 | log.err(e) 542 | prot.send_plain(user.jid, "Problem grabbing top10") 543 | db.get_top10().addCallback(worked).addErrback(notWorked) 544 | 545 | class AdminServiceDistributionCommand(BaseCommand): 546 | 547 | def __init__(self): 548 | super(AdminServiceDistributionCommand, self).__init__('adm_udist', 549 | 'Find out the distribution of jid/service counts.') 550 | 551 | @admin_required 552 | def __call__(self, user, prot, args): 553 | def worked(dist): 554 | rv=["Service Distribution:"] 555 | rv.append("") 556 | rv.extend(["%s: %s" % (row[1], row[0]) for row in dist]) 557 | prot.send_plain(user.jid, "\n".join(rv)) 558 | def notWorked(e): 559 | log.err(e) 560 | prot.send_plain(user.jid, "Problem grabbing distribution") 561 | db.get_service_distribution().addCallback(worked).addErrback(notWorked) 562 | 563 | class MoodCommand(BaseCommand): 564 | 565 | def __init__(self): 566 | super(MoodCommand, self).__init__('mood', 567 | "Ask about twitterspy's mood.") 568 | 569 | def __call__(self, user, prot, args): 570 | mood, good, total, percentage = moodiness.moodiness.current_mood() 571 | if mood: 572 | rv=["My current mood is %s" % mood] 573 | rv.append("I've processed %d out of the last %d searches." 574 | % (good, total)) 575 | else: 576 | rv=["I just woke up. Ask me in a minute or two."] 577 | rv.append("I currently have %d API requests available, " 578 | "and have run out %d times." 579 | % (scheduling.available_requests, scheduling.empty_resets)) 580 | rv.append("Locks wanted: %d, locks held: %d" 581 | % (scheduling.locks_requested, scheduling.locks_acquired)) 582 | prot.send_plain(user.jid, "\n".join(rv)) 583 | 584 | class MoodDetailCommand(BaseCommand): 585 | 586 | def __init__(self): 587 | super(MoodDetailCommand, self).__init__('mood_detail', 588 | 'Detailed mood info.') 589 | 590 | def __call__(self, user, prot, args): 591 | h = moodiness.moodiness.result_counts() 592 | rv = ["Recent statuses from searches:\n"] 593 | for s,c in sorted(h.items()): 594 | rv.append("%s: %d" % (str(s), c)) 595 | prot.send_plain(user.jid, "\n".join(rv)) 596 | 597 | class UptimeCommand(BaseCommand): 598 | 599 | def __init__(self): 600 | super(UptimeCommand, self).__init__('uptime', 601 | "Ask about twitterspy's uptime.") 602 | 603 | def _pluralize(self, v, word): 604 | if v == 1: 605 | return str(v) + " " + word 606 | else: 607 | return str(v) + " " + word + "s" 608 | 609 | def _ts(self, td): 610 | rv = "" 611 | if td.days > 0: 612 | rv += self._pluralize(td.days, "day") + " " 613 | secs = td.seconds 614 | if secs >= 3600: 615 | rv += self._pluralize(int(secs / 3600), "hr") + " " 616 | secs = secs % 3600 617 | if secs >= 60: 618 | rv += self._pluralize(int(secs / 60), "min") + " " 619 | secs = secs % 60 620 | rv += self._pluralize(secs, "sec") 621 | return rv 622 | 623 | def _pluralize(self, n, w): 624 | if n == 1: 625 | return str(n) + " " + w 626 | else: 627 | return str(n) + " " + w + "s" 628 | 629 | def __call__(self, user, prot, args): 630 | time_format = "%Y/%m/%d %H:%M:%S" 631 | now = datetime.datetime.utcfromtimestamp(time.time()) 632 | 633 | started = datetime.datetime.utcfromtimestamp( 634 | protocol.presence_for(user.jid).started) 635 | 636 | start_delta = now - started 637 | 638 | rv=[] 639 | rv.append("Twitterspy Standard Time: %s" 640 | % now.strftime(time_format)) 641 | rv.append("Started at %s (%s ago)" 642 | % (started.strftime(time_format), self._ts(start_delta))) 643 | 644 | for p in protocol.presence_conns.values(): 645 | if p.connected: 646 | connected = datetime.datetime.utcfromtimestamp(p.connected) 647 | conn_delta = now - connected 648 | rv.append("Connected to %s at %s (%s ago, connected %s)" 649 | % (p.jid, connected.strftime(time_format), 650 | self._ts(conn_delta), 651 | self._pluralize(p.num_connections, "time"))) 652 | elif p.lost: 653 | lost = datetime.datetime.utcfromtimestamp(p.lost) 654 | lost_delta = now - lost 655 | rv.append("Lost connection to %s at %s (%s ago, connected %s)" 656 | % (p.jid, lost.strftime(time_format), 657 | self._ts(lost_delta), 658 | self._pluralize(p.num_connections, "time"))) 659 | else: 660 | rv.append("Not currently, nor ever connected to %s" % jid) 661 | prot.send_plain(user.jid, "\n\n".join(rv)) 662 | 663 | class AdminHangupCommand(BaseCommand): 664 | 665 | def __init__(self): 666 | super(AdminHangupCommand, self).__init__('adm_hangup', 667 | 'Disconnect an xmpp session.') 668 | 669 | @admin_required 670 | @arg_required() 671 | def __call__(self, user, prot, args): 672 | try: 673 | conn = protocol.presence_conns[args] 674 | prot.send_plain(user.jid, "Disconnecting %s" % args) 675 | reactor.callWhenRunning(conn.xmlstream.transport.loseConnection) 676 | except KeyError: 677 | prot.send_plain(user.jid, "Could not find session %s" % args) 678 | 679 | class AdminSubscribeCommand(BaseCommand): 680 | 681 | def __init__(self): 682 | super(AdminSubscribeCommand, self).__init__('adm_subscribe', 683 | 'Subscribe a user.') 684 | 685 | @admin_required 686 | @arg_required() 687 | def __call__(self, user, prot, args): 688 | prot.send_plain(user.jid, "Subscribing " + args) 689 | protocol.default_presence.subscribe(JID(args)) 690 | 691 | class AdminUserStatusCommand(BaseStatusCommand): 692 | 693 | def __init__(self): 694 | super(AdminUserStatusCommand, self).__init__('adm_status', 695 | "Check a user's status.") 696 | 697 | @admin_required 698 | @arg_required() 699 | def __call__(self, user, prot, args): 700 | def worked(u): 701 | prot.send_plain(user.jid, self.get_user_status(u)) 702 | def notWorked(e): 703 | log.err(e) 704 | prot.send_plain(user.jid, "Failed to load user: " + str(e)) 705 | db.User.by_jid(args).addCallback(worked).addErrback(notWorked) 706 | 707 | class AdminPingCommand(BaseCommand): 708 | 709 | def __init__(self): 710 | super(AdminPingCommand, self).__init__('adm_ping', 711 | 'Ping a JID') 712 | 713 | def ping(self, prot, fromjid, tojid): 714 | p = ping.Ping(prot.xmlstream, protocol.default_conn.jid, tojid) 715 | d = p.send() 716 | log.msg("Sending ping %s" % p.toXml()) 717 | def _gotPing(x): 718 | duration = time.time() - p.start_time 719 | log.msg("pong %s" % tojid) 720 | prot.send_plain(fromjid, ":) Pong (%s) - %fs" % (tojid, duration)) 721 | def _gotError(x): 722 | duration = time.time() - p.start_time 723 | log.msg("Got an error pinging %s: %s" % (tojid, x)) 724 | prot.send_plain(fromjid, ":( Error pinging %s (%fs): %s" 725 | % (tojid, duration, x.value.condition)) 726 | d.addCallback(_gotPing) 727 | d.addErrback(_gotError) 728 | return d 729 | 730 | @admin_required 731 | @arg_required() 732 | def __call__(self, user, prot, args): 733 | # For bare jids, we'll send what was requested, 734 | # but also look up the user and send it to any active resources 735 | self.ping(prot, user.jid, args) 736 | j = JID(args) 737 | if j.user and not j.resource: 738 | for rsrc in scheduling.resources(args): 739 | j.resource=rsrc 740 | self.ping(prot, user.jid, j.full()) 741 | 742 | class AdminBroadcastCommand(BaseCommand): 743 | 744 | def __init__(self): 745 | super(AdminBroadcastCommand, self).__init__('adm_broadcast', 746 | 'Broadcast a message.') 747 | 748 | def _do_broadcast(self, users, prot, jid, msg): 749 | log.msg("Administrative broadcast from %s" % jid) 750 | for j in users: 751 | log.msg("Sending message to %s" % j) 752 | prot.send_plain(j, msg) 753 | prot.send_plain(jid, "Sent message to %d users" % len(users)) 754 | 755 | @admin_required 756 | @arg_required() 757 | def __call__(self, user, prot, args): 758 | db.get_active_users().addCallback(self._do_broadcast, prot, user.jid, args) 759 | 760 | class AdminUserPresenceCommand(BaseCommand): 761 | 762 | def __init__(self): 763 | super(AdminUserPresenceCommand, self).__init__('adm_userpresence', 764 | "Find out about user presence.") 765 | 766 | @admin_required 767 | def __call__(self, user, prot, args): 768 | prot.send_plain(user.jid, "Watching %d active queries for %d active users." 769 | % (len(scheduling.queries), len(scheduling.users))) 770 | 771 | for __t in (t for t in globals().values() if isinstance(type, type(t))): 772 | if BaseCommand in __t.__mro__: 773 | try: 774 | i = __t() 775 | all_commands[i.name] = i 776 | except TypeError, e: 777 | # Ignore abstract bases 778 | log.msg("Error loading %s: %s" % (__t.__name__, str(e))) 779 | pass 780 | -------------------------------------------------------------------------------- /lib/twitterspy/xmpp_ping.py: -------------------------------------------------------------------------------- 1 | from zope.interface import implements 2 | 3 | from twisted.python import log 4 | from twisted.internet import defer 5 | from wokkel.subprotocols import XMPPHandler, IQHandlerMixin 6 | from wokkel import disco 7 | from wokkel import generic 8 | from wokkel.iwokkel import IDisco 9 | 10 | NS_PING = 'urn:xmpp:ping' 11 | PING = generic.IQ_GET + '/ping[@xmlns="' + NS_PING + '"]' 12 | 13 | class PingHandler(XMPPHandler, IQHandlerMixin): 14 | """ 15 | XMPP subprotocol handler for Ping. 16 | 17 | This protocol is described in 18 | U{XEP-0199}. 19 | """ 20 | 21 | implements(IDisco) 22 | 23 | iqHandlers = {PING: 'onPing'} 24 | 25 | def connectionInitialized(self): 26 | super(PingHandler, self).connectionInitialized() 27 | self.xmlstream.addObserver(PING, self.handleRequest) 28 | 29 | def onPing(self, iq): 30 | log.msg("Got ping from %s" % iq.getAttribute("from")) 31 | 32 | def getDiscoInfo(self, requestor, target, node): 33 | info = set() 34 | 35 | if not node: 36 | info.add(disco.DiscoFeature(NS_PING)) 37 | 38 | return defer.succeed(info) 39 | 40 | def getDiscoItems(self, requestor, target, node): 41 | return defer.succeed([]) 42 | 43 | -------------------------------------------------------------------------------- /test/expansion_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import with_statement 4 | 5 | import sys 6 | import xml 7 | 8 | from twisted.trial import unittest 9 | from twisted.internet import defer, reactor 10 | 11 | sys.path.extend(['lib', '../lib', 12 | 'lib/twitterspy', '../lib/twitterspy', 13 | 'lib/twisted-longurl/lib', '../lib/twisted-longurl/lib']) 14 | 15 | import url_expansion 16 | 17 | class SimpleService(object): 18 | 19 | def __init__(self, name, domains=None): 20 | self.name = name 21 | if domains: 22 | self.domains = domains 23 | else: 24 | self.domains = [name] 25 | 26 | class FakeHTTP(object): 27 | 28 | def __init__(self): 29 | self.d = defer.Deferred() 30 | 31 | def getPage(self, *args, **kwargs): 32 | return self.d 33 | 34 | class Result(url_expansion.BasicUrl): 35 | pass 36 | 37 | class FakeLongUrl(object): 38 | 39 | def __init__(self, r): 40 | self.result = r 41 | 42 | def expand(self, u): 43 | rv = defer.Deferred() 44 | 45 | if self.result: 46 | reactor.callWhenRunning(rv.callback, self.result) 47 | else: 48 | reactor.callWhenRunning(rv.errback, RuntimeError("http failed")) 49 | return rv 50 | 51 | class MatcherTest(unittest.TestCase): 52 | 53 | def setUp(self): 54 | self.expander = url_expansion.Expander() 55 | self.expander.cache = False 56 | self.expander._registerServices({'is.gd': 57 | SimpleService('is.gd'), 58 | 'bit.ly': 59 | SimpleService('bit.ly', 60 | ['bit.ly', 'bit.ley'])}) 61 | 62 | def testNoopExpansion(self): 63 | d = self.expander.expand("test message") 64 | def v(r): 65 | self.assertEquals('test message', r[0]) 66 | self.assertEquals(None, r[1]) 67 | d.addCallback(v) 68 | return d 69 | 70 | def testExpansion(self): 71 | self.expander.lu = FakeLongUrl(Result('Test Title', 'http://w/')) 72 | 73 | d = self.expander.expand("test http://is.gd/whatever message") 74 | def v(r): 75 | self.assertEquals('test http://w/ (from http://is.gd/whatever) message', r[0]) 76 | self.assertEquals(None, r[1]) 77 | d.addCallback(v) 78 | return d 79 | 80 | def testIdentityExpansion(self): 81 | self.expander.lu = FakeLongUrl(Result(None, 'http://is.gd/whatever')) 82 | 83 | d = self.expander.expand("test http://is.gd/whatever message") 84 | def v(r): 85 | self.assertEquals('test http://is.gd/whatever message', r[0]) 86 | self.assertEquals(None, r[1]) 87 | d.addCallback(v) 88 | return d 89 | 90 | def testExpansionWithAmpersand(self): 91 | self.expander.lu = FakeLongUrl(Result('Test Title', 'http://w/?a=1&b=2')) 92 | 93 | d = self.expander.expand("test http://is.gd/whatever message") 94 | def v(r): 95 | self.assertEquals('test http://w/?a=1&b=2 ' 96 | '(from http://is.gd/whatever) message', r[0]) 97 | self.assertEquals(None, r[1]) 98 | d.addCallback(v) 99 | return d 100 | 101 | 102 | def testFailedExpansion(self): 103 | self.expander.lu = FakeLongUrl(None) 104 | 105 | def v(r): 106 | self.assertEquals('test http://is.gd/whatever message', r[0]) 107 | self.assertEquals(None, r[1]) 108 | self.flushLoggedErrors(RuntimeError) 109 | def h(e): 110 | self.fail("Error bubbled up.") 111 | d = self.expander.expand("test http://is.gd/whatever message") 112 | d.addCallback(v) 113 | d.addErrback(h) 114 | return d 115 | 116 | def testHtmlExpansion(self): 117 | self.expander.lu = FakeLongUrl(Result('Test Title', 'http://w/')) 118 | 119 | d = self.expander.expand("test http://is.gd/whatever message", 120 | """test """ 121 | """http://is.gd/whatever message""") 122 | def v(r): 123 | self.assertEquals('test http://w/ (from http://is.gd/whatever) message', r[0]) 124 | self.assertEquals("""test """ 125 | """http://w/ message""", r[1]) 126 | d.addCallback(v) 127 | return d 128 | -------------------------------------------------------------------------------- /test/mood_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | 5 | from twisted.trial import unittest 6 | from twisted.internet import defer, reactor 7 | from twisted.python import failure 8 | from twisted import web 9 | 10 | sys.path.extend(['lib', '../lib', 11 | 'lib/twitterspy', '../lib/twitterspy', 12 | 'lib/twitty-twister/twittytwister', '../lib/twitty-twister', 13 | 'lib/twisted-longurl/lib', '../lib/twisted-longurl', 14 | 'lib/wokkel', '../lib/wokkel']) 15 | 16 | import moodiness 17 | 18 | def webError(n): 19 | return failure.Failure(web.error.Error(n)) 20 | 21 | class MoodiTest(unittest.TestCase): 22 | 23 | def setUp(self): 24 | self.m = moodiness.Moodiness() 25 | 26 | for i in range(25): 27 | self.m.markSuccess("ignored") 28 | for i in range(25): 29 | self.m.markFailure("not an exception") 30 | for i in range(50): 31 | self.m.markFailure(webError('503')) 32 | 33 | def testMoodCounts(self): 34 | h = self.m.result_counts() 35 | self.assertEquals(25, h[True]) 36 | self.assertEquals(25, h[False]) 37 | self.assertEquals(50, h['503']) 38 | 39 | def testMood(self): 40 | mood, good, total, percentage = self.m.current_mood() 41 | self.assertTrue(mood in ('annoyed', 'dismayed', 'depressed', 'worried')) 42 | self.assertEquals(25, good) 43 | self.assertEquals(100, total) 44 | self.assertEquals(0.25, percentage) 45 | 46 | 47 | def testWeirdFailure(self): 48 | r = ['503', '503', '503', '503', 49 | '503', '503', '503', '503', '503', '503', '503', '503', '503', 50 | '503', '503', '503', '503', True, '503', '503', '503', '503', 51 | '503', '503', '503', '503', '503', '503', '503', '503', '503', 52 | True, True, True, True, True, True, True, True, True, True, 53 | True, True, True, True, '400', True, True, True, True, True, 54 | True, True, True, True, True, True, True, True, True, True, 55 | True, True, True, True, True, True, True, True, True, True, 56 | True, True, True, True, True, True, True] 57 | 58 | self.m = moodiness.Moodiness() 59 | self.m.recent_results = r 60 | 61 | mood, good, total, percentage = self.m.current_mood() 62 | self.assertTrue(mood in ('frustrated', 'annoyed', 'anxious', 'grumpy')) 63 | self.assertEquals(47, good) 64 | self.assertEquals(78, total) 65 | self.assertAlmostEquals(0.60, percentage, .01) 66 | -------------------------------------------------------------------------------- /test/scheduling_test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import xml 3 | 4 | from twisted.trial import unittest 5 | from twisted.internet import defer, reactor 6 | 7 | sys.path.extend(['lib', '../lib', 8 | 'lib/twitterspy', '../lib/twitterspy', 9 | 'lib/wokkel/', '../lib/wokkel/', 10 | 'lib/twisted-longurl/lib', '../lib/twisted-longurl/lib/', 11 | 'lib/twitty-twister/twittytwister', '../lib/twitty-twister/twittytwister']) 12 | 13 | import cache 14 | import scheduling 15 | 16 | class FakeTwitterAPI(object): 17 | 18 | def __init__(self, res): 19 | self.res = res 20 | 21 | def search(self, query, cb, params): 22 | def f(): 23 | for r in self.res: 24 | cb(res) 25 | return defer.succeed("yay") 26 | reactor.callWhenRunning(f) 27 | 28 | class FakeCache(object): 29 | 30 | def get(self, x): 31 | return defer.succeed([0, None]) 32 | 33 | def set(self, k, v): 34 | return defer.succeed(None) 35 | 36 | class QueryRegistryTest(unittest.TestCase): 37 | 38 | started = 0 39 | stopped = 0 40 | 41 | def setUp(self): 42 | import twisted 43 | twisted.internet.base.DelayedCall.debug = True 44 | super(QueryRegistryTest, self).setUp() 45 | 46 | cache.mc = FakeCache() 47 | self.patch(scheduling.Query, '_doStart', self.trackStarted) 48 | self.patch(scheduling.Query, 'start', lambda *x: self.fail("unexpected start")) 49 | self.patch(scheduling.Query, 'stop', self.trackStopped) 50 | 51 | self.qr = scheduling.QueryRegistry(lambda x: FakeTwitterAPI(['a'])) 52 | self.assertEquals(0, len(self.qr)) 53 | self.qr.add('dustin@localhost', 'test query') 54 | 55 | def trackStarted(self, *args): 56 | self.started += 1 57 | 58 | def trackStopped(self, *args): 59 | self.stopped += 1 60 | 61 | def testTracking(self): 62 | self.assertEquals(1, len(self.qr)) 63 | self.assertEquals(1, self.started) 64 | 65 | def testUntracking(self): 66 | self.qr.untracked('dustin@localhost', 'test query') 67 | self.assertEquals(0, len(self.qr)) 68 | self.assertEquals(1, self.stopped) 69 | 70 | def testRemove(self): 71 | self.qr.add('dustin@localhost', 'test query two') 72 | self.assertEquals(2, len(self.qr)) 73 | self.assertEquals(2, self.started) 74 | self.qr.remove('dustin@localhost') 75 | self.assertEquals(0, len(self.qr)) 76 | self.assertEquals(2, self.stopped) 77 | 78 | def testRemoveTwo(self): 79 | self.qr.add('dustin2@localhost', 'test query two') 80 | self.assertEquals(2, len(self.qr)) 81 | self.assertEquals(2, self.started) 82 | self.qr.remove('dustin@localhost') 83 | self.assertEquals(1, len(self.qr)) 84 | self.assertEquals(1, self.stopped) 85 | 86 | def testRemoveUser(self): 87 | self.qr.add('dustin@localhost', 'test query two') 88 | self.assertEquals(2, len(self.qr)) 89 | self.assertEquals(2, self.started) 90 | self.qr.remove_user('', ['dustin@localhost']) 91 | self.assertEquals(0, len(self.qr)) 92 | self.assertEquals(2, self.stopped) 93 | 94 | def testRemoveUserTwo(self): 95 | self.qr.add('dustin@localhost', 'test query two') 96 | self.qr.add('dustin2@localhost', 'test query two') 97 | self.assertEquals(2, len(self.qr)) 98 | self.assertEquals(2, self.started) 99 | self.qr.remove_user('', ['dustin@localhost']) 100 | self.assertEquals(1, len(self.qr)) 101 | self.assertEquals(1, self.stopped) 102 | 103 | class JidSetTest(unittest.TestCase): 104 | 105 | def testIteration(self): 106 | js = scheduling.JidSet() 107 | js.add('dustin@localhost/r1') 108 | js.add('dustin@localhost/r2') 109 | js.add('dustin@elsewhere/r1') 110 | 111 | self.assertEquals(3, len(js)) 112 | 113 | self.assertEquals(2, len(js.bare_jids())) 114 | 115 | self.assertTrue('dustin@localhost', js.bare_jids()) 116 | self.assertTrue('dustin@elsewhere', js.bare_jids()) 117 | -------------------------------------------------------------------------------- /test/search_collector_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | 3 | import sys 4 | import xml 5 | 6 | from twisted.trial import unittest 7 | from twisted.internet import defer, reactor 8 | 9 | sys.path.extend(['lib', '../lib', 10 | 'lib/twitterspy', '../lib/twitterspy', 11 | 'lib/twisted-longurl/lib', '../lib/twisted-longurl/lib']) 12 | 13 | import search_collector, url_expansion 14 | 15 | class FakeAuthor(object): 16 | 17 | def __init__(self, name, uri): 18 | self.name = name 19 | self.uri = uri 20 | 21 | class FakeEntry(object): 22 | 23 | def __init__(self, i, author, title, content): 24 | self.id = i 25 | self.author = author 26 | self.title = title 27 | self.content = content 28 | 29 | class FakeUrlExpander(object): 30 | 31 | def __init__(self): 32 | self.expectations = set() 33 | 34 | def instantError(self, plain, html): 35 | return defer.fail(RuntimeError("failed " + plain)) 36 | 37 | def instantSuccess(self, plain, html): 38 | return defer.succeed((plain + " m", html + " m")) 39 | 40 | def expand(self, plain, html): 41 | if plain in self.expectations: 42 | return self.instantSuccess(plain, html) 43 | else: 44 | return self.instantError(plain, html) 45 | 46 | class SearchCollectorTest(unittest.TestCase): 47 | 48 | def setUp(self): 49 | url_expansion.expander = FakeUrlExpander() 50 | 51 | def doSomeStuff(self, sc): 52 | sc.gotResult(FakeEntry('blah:14', 53 | FakeAuthor('dustin author', 'http://w/'), 54 | 'Some Title 14', 55 | 'Some Content 14')) 56 | 57 | sc.gotResult(FakeEntry('blah:11', 58 | FakeAuthor('dustin author', 'http://w/'), 59 | 'Some Title 11', 60 | 'Some Content 11')) 61 | 62 | sc.gotResult(FakeEntry('blah:13', 63 | FakeAuthor('dustin author', 'http://w/'), 64 | 'Some Title 13', 65 | 'Some Content 13')) 66 | 67 | def testSimpleNoMatches(self): 68 | sc = search_collector.SearchCollector() 69 | self.doSomeStuff(sc) 70 | 71 | self.assertEquals(3, len(sc.deferreds)) 72 | dl = defer.DeferredList(sc.deferreds) 73 | 74 | def verify(r): 75 | self.assertEquals([11, 13, 14], [e[0] for e in sc.results]) 76 | self.assertEquals("dustin: Some Title 11", sc.results[0][1]) 77 | self.flushLoggedErrors(RuntimeError) 78 | 79 | dl.addCallback(verify) 80 | 81 | return dl 82 | 83 | def testSomeMatches(self): 84 | url_expansion.expander.expectations.add('dustin: Some Title 11') 85 | sc = search_collector.SearchCollector() 86 | self.doSomeStuff(sc) 87 | 88 | self.assertEquals(3, len(sc.deferreds)) 89 | dl = defer.DeferredList(sc.deferreds) 90 | 91 | def verify(r): 92 | self.assertEquals([11, 13, 14], [e[0] for e in sc.results]) 93 | self.assertEquals("dustin: Some Title 11 m", sc.results[0][1]) 94 | self.assertEquals("dustin: Some Title 13", sc.results[1][1]) 95 | self.flushLoggedErrors(RuntimeError) 96 | 97 | dl.addCallback(verify) 98 | 99 | return dl 100 | -------------------------------------------------------------------------------- /twitterspy.conf.sample: -------------------------------------------------------------------------------- 1 | [general] 2 | loop_sleep: 60 3 | watch_freq: 1 4 | personal_freq: 3 5 | admins: you@example.com 6 | expand: True 7 | 8 | # See the documentation for db config options: 9 | # http://dustin.github.com/twitterspy/dev.html 10 | [db] 11 | type: couch 12 | host: localhost 13 | 14 | [xmpp] 15 | jid: twitterspy@example.com/bot 16 | pass: y0urb0tzp4ssw0rd 17 | -------------------------------------------------------------------------------- /twitterspy.start: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Shell script for running the jabber bot. I'd rather use something like 4 | # launchd, but that's unavailable to me on my servers. 5 | 6 | ulimit -v 200000 7 | ulimit -m 200000 8 | 9 | while : 10 | do 11 | twistd -l log/twitterspy.log -ny twitterspy.tac 12 | sleep 5 13 | done 14 | 15 | -------------------------------------------------------------------------------- /twitterspy.tac: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.insert(0, "lib/twitty-twister/twittytwister") 3 | sys.path.insert(0, "lib/wokkel") 4 | sys.path.insert(0, 'lib/twisted-longurl/lib') 5 | sys.path.insert(0, "lib") 6 | 7 | import ConfigParser 8 | 9 | from twisted.application import service 10 | from twisted.internet import task, reactor 11 | from twisted.words.protocols.jabber import jid 12 | from twisted.web import client 13 | from wokkel.client import XMPPClient 14 | from wokkel.generic import VersionHandler 15 | from wokkel.keepalive import KeepAlive 16 | from wokkel.disco import DiscoHandler 17 | import twitter 18 | 19 | from twitterspy import db 20 | from twitterspy import cache 21 | from twitterspy import config 22 | from twitterspy import protocol 23 | from twitterspy import xmpp_ping 24 | from twitterspy import scheduling 25 | from twitterspy import moodiness 26 | from twitterspy import url_expansion 27 | from twitterspy import adhoc_commands 28 | 29 | # Set the user agent for twitter 30 | twitter.Twitter.agent = "twitterspy" 31 | 32 | client.HTTPClientFactory.noisy = False 33 | 34 | application = service.Application("twitterspy") 35 | 36 | def build_client(section): 37 | host = None 38 | try: 39 | host = config.CONF.get(section, 'host') 40 | except ConfigParser.NoSectionError: 41 | pass 42 | 43 | j = jid.internJID(config.CONF.get(section, 'jid')) 44 | 45 | xmppclient = XMPPClient(j, config.CONF.get(section, 'pass'), host) 46 | 47 | xmppclient.logTraffic = False 48 | 49 | # Stream handling protocols for twitterspy 50 | protocols = [protocol.TwitterspyPresenceProtocol, 51 | protocol.TwitterspyMessageProtocol] 52 | 53 | for p in protocols: 54 | handler=p(j) 55 | handler.setHandlerParent(xmppclient) 56 | 57 | DiscoHandler().setHandlerParent(xmppclient) 58 | VersionHandler('twitterspy', config.VERSION).setHandlerParent(xmppclient) 59 | xmpp_ping.PingHandler().setHandlerParent(xmppclient) 60 | adhoc_commands.AdHocHandler().setHandlerParent(xmppclient) 61 | KeepAlive().setHandlerParent(xmppclient) 62 | xmppclient.setServiceParent(application) 63 | 64 | return xmppclient 65 | 66 | 67 | cache.connect() 68 | db.initialize() 69 | 70 | build_client('xmpp') 71 | try: 72 | build_client('xmpp_secondary') 73 | except ConfigParser.NoSectionError: 74 | pass 75 | 76 | task.LoopingCall(moodiness.moodiness).start(60, now=False) 77 | task.LoopingCall(scheduling.resetRequests).start(scheduling.REQUEST_PERIOD, 78 | now=False) 79 | 80 | # If the expansion services are loaded, expansion will take effect 81 | if (config.CONF.has_option('general', 'expand') 82 | and config.CONF.getboolean('general', 'expand')): 83 | 84 | # Load the url expansion services now, and refresh it every seven days. 85 | task.LoopingCall(url_expansion.expander.loadServices).start(86400 * 7) 86 | 87 | --------------------------------------------------------------------------------