├── .coveragerc ├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.md ├── quicktest.py ├── settings_test.py ├── setup.py ├── travis_django_version.py └── twitter_api ├── __init__.py ├── admin.py ├── api.py ├── decorators.py ├── factories.py ├── fields.py ├── migrations ├── 0001_initial.py └── __init__.py ├── models.py ├── parser.py ├── sql ├── status.postgresql_psycopg2.sql └── user.postgresql_psycopg2.sql └── tests.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | *migrations* 4 | *admin.py 5 | 6 | [report] 7 | exclude_lines = 8 | def __repr__ 9 | def __unicode__ 10 | def parse_args 11 | pragma: no cover 12 | raise NotImplementedError 13 | if __name__ == .__main__.: -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /dist 2 | *.pyc 3 | *.egg-info 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | env: 5 | - DJANGO=1.4 DB=postgres 6 | - DJANGO=1.5 DB=postgres 7 | - DJANGO=1.6 DB=postgres 8 | - DJANGO=1.7 DB=postgres 9 | before_script: 10 | - mysql -e 'create database django;' 11 | - psql -c 'create database django;' -U postgres 12 | install: 13 | - if [[ $DB == mysql ]]; then pip install mysql-python; fi 14 | - if [[ $DB == postgres ]]; then pip install psycopg2; fi 15 | - DJANGO_VER=$(./travis_django_version.py $DJANGO) 16 | - pip install $DJANGO_VER 17 | - pip install simplejson 18 | - pip install factory_boy 19 | - pip install coveralls 20 | - pip install mock 21 | - pip install . 22 | script: 23 | - django-admin.py --version 24 | - coverage run --source=twitter_api quicktest.py twitter_api 25 | after_success: 26 | - coveralls 27 | notifications: 28 | email: 29 | recipients: 30 | - ramusus@gmail.com 31 | on_success: change 32 | on_failure: change 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011, ramusus and contributors. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of Django nor the names of its contributors may be used 15 | to endorse or promote products derived from this software without 16 | specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include LICENSE 3 | include MANIFEST.in 4 | include quicktest.py 5 | include settings_test.py 6 | recursive-include twitter_api * -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Django Twitter API 2 | 3 | [![Build Status](https://travis-ci.org/ramusus/django-twitter-api.png?branch=master)](https://travis-ci.org/ramusus/django-twitter-api) [![Coverage Status](https://coveralls.io/repos/ramusus/django-twitter-api/badge.png?branch=master)](https://coveralls.io/r/ramusus/django-twitter-api) 4 | 5 | Application for interacting with Twitter API objects using Django model interface 6 | 7 | ## Installation 8 | 9 | pip install django-twitter-api 10 | 11 | Add into `settings.py` lines: 12 | 13 | INSTALLED_APPS = ( 14 | ... 15 | 'oauth_tokens', 16 | 'm2m_history', 17 | 'taggit', 18 | 'twitter_api', 19 | ) 20 | 21 | # oauth-tokens settings 22 | OAUTH_TOKENS_HISTORY = True # to keep in DB expired access tokens 23 | OAUTH_TOKENS_TWITTER_CLIENT_ID = '' # application ID 24 | OAUTH_TOKENS_TWITTER_CLIENT_SECRET = '' # application secret key 25 | OAUTH_TOKENS_TWITTER_USERNAME = '' # user login 26 | OAUTH_TOKENS_TWITTER_PASSWORD = '' # user password 27 | 28 | ## Usage examples 29 | 30 | ### Simple API request 31 | 32 | >>> from twitter_api.utils import api 33 | >>> response = api('get_status', 327926550815207424) 34 | >>> response.text 35 | '@mrshoranweyhey Thanks for the love! How about a follow for a follow? :) ^LF' 36 | >>> response.source_url 37 | 'http://www.exacttarget.com/social' 38 | >>> response = api('get_user', 'BarackObama') 39 | >>> response.id, response.name 40 | (813286, 'Barack Obama') 41 | 42 | ### Fetch status by ID 43 | 44 | >>> from twitter_api.models import Status 45 | >>> status = Status.remote.fetch(327926550815207424) 46 | >>> status 47 | 48 | >>> status.in_reply_to_status 49 | 50 | 51 | ### Fetch user by ID and user name 52 | 53 | >>> from twitter_api.models import User 54 | >>> User.remote.fetch(813286) 55 | 56 | >>> User.remote.fetch('BarackObama') 57 | 58 | 59 | ### Fetch statuses of user 60 | 61 | >>> from models import User 62 | >>> user = User.remote.fetch(813286) 63 | >>> user.fetch_statuses(count=30) 64 | [, 65 | , 66 | , 67 | ...] 68 | 69 | ### Fetch followers of user 70 | 71 | >>> from twitter_api.models import User 72 | >>> user = User.remote.fetch(813286) 73 | >>> user.fetch_followers(all=True) 74 | [, , , '...(remaining elements truncated)...'] 75 | 76 | ### Fetch retweets of status 77 | 78 | >>> from twitter_api.models import Status 79 | >>> status = Status.remote.fetch(329231054282055680) 80 | >>> status.fetch_retweets() 81 | [, 82 | , 83 | , 84 | ...] 85 | 86 | ### Fetch replies of status 87 | 88 | >>> from twitter_api.models import Status 89 | >>> status = Status.remote.fetch(536859483851735040) 90 | >>> status.fetch_replies() 91 | [, 92 | , 93 | , 94 | ...] 95 | >>> status.replies_count 96 | 6 97 | -------------------------------------------------------------------------------- /quicktest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import argparse 4 | from django.conf import settings 5 | 6 | ''' 7 | QuickDjangoTest module for testing in Travis CI https://travis-ci.org 8 | Changes log: 9 | * 2014-10-24 updated for compatibility with Django 1.7 10 | * 2014-11-03 different databases support: sqlite3, mysql, postgres 11 | ''' 12 | 13 | class QuickDjangoTest(object): 14 | """ 15 | A quick way to run the Django test suite without a fully-configured project. 16 | 17 | Example usage: 18 | 19 | >>> QuickDjangoTest('app1', 'app2') 20 | 21 | Based on a script published by Lukasz Dziedzia at: 22 | http://stackoverflow.com/questions/3841725/how-to-launch-tests-for-django-reusable-app 23 | """ 24 | DIRNAME = os.path.dirname(__file__) 25 | INSTALLED_APPS = ( 26 | 'django.contrib.auth', 27 | 'django.contrib.contenttypes', 28 | 'django.contrib.sessions', 29 | 'django.contrib.admin', 30 | ) 31 | 32 | def __init__(self, *args, **kwargs): 33 | self.apps = args 34 | 35 | # Get the version of the test suite 36 | self.version = self.get_test_version() 37 | 38 | # Call the appropriate one 39 | if self.version == '1.7': 40 | self._tests_1_7() 41 | elif self.version == '1.2': 42 | self._tests_1_2() 43 | else: 44 | self._tests_old() 45 | 46 | def get_test_version(self): 47 | """ 48 | Figure out which version of Django's test suite we have to play with. 49 | """ 50 | from django import VERSION 51 | if VERSION[0] == 1 and VERSION[1] >= 7: 52 | return '1.7' 53 | elif VERSION[0] == 1 and VERSION[1] >= 2: 54 | return '1.2' 55 | else: 56 | return 57 | 58 | def get_database(self): 59 | test_db = os.environ.get('DB', 'sqlite') 60 | if test_db == 'mysql': 61 | database = { 62 | 'ENGINE': 'django.db.backends.mysql', 63 | 'NAME': 'django', 64 | 'USER': 'root', 65 | } 66 | elif test_db == 'postgres': 67 | database = { 68 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', 69 | 'USER': 'postgres', 70 | 'NAME': 'django', 71 | 'OPTIONS': { 72 | 'autocommit': True, 73 | } 74 | } 75 | else: 76 | database = { 77 | 'ENGINE': 'django.db.backends.sqlite3', 78 | 'NAME': os.path.join(self.DIRNAME, 'database.db'), 79 | 'USER': '', 80 | 'PASSWORD': '', 81 | 'HOST': '', 82 | 'PORT': '', 83 | } 84 | return {'default': database} 85 | 86 | def get_custom_settings(self): 87 | try: 88 | from settings_test import * 89 | settings_test = dict(locals()) 90 | del settings_test['self'] 91 | if 'INSTALLED_APPS' in settings_test: 92 | del settings_test['INSTALLED_APPS'] 93 | except ImportError: 94 | settings_test = {} 95 | INSTALLED_APPS = [] 96 | 97 | return INSTALLED_APPS, settings_test 98 | 99 | def _tests_old(self): 100 | """ 101 | Fire up the Django test suite from before version 1.2 102 | """ 103 | INSTALLED_APPS, settings_test = self.get_custom_settings() 104 | 105 | settings.configure(DEBUG = True, 106 | DATABASE_ENGINE = 'sqlite3', 107 | DATABASE_NAME = os.path.join(self.DIRNAME, 'database.db'), 108 | INSTALLED_APPS = self.INSTALLED_APPS + INSTALLED_APPS + self.apps, 109 | **settings_test 110 | ) 111 | from django.test.simple import run_tests 112 | failures = run_tests(self.apps, verbosity=1) 113 | if failures: 114 | sys.exit(failures) 115 | 116 | def _tests_1_2(self): 117 | """ 118 | Fire up the Django test suite developed for version 1.2 and up 119 | """ 120 | INSTALLED_APPS, settings_test = self.get_custom_settings() 121 | 122 | settings.configure( 123 | DEBUG = True, 124 | DATABASES = self.get_database(), 125 | INSTALLED_APPS = self.INSTALLED_APPS + INSTALLED_APPS + self.apps, 126 | **settings_test 127 | ) 128 | 129 | from django.test.simple import DjangoTestSuiteRunner 130 | failures = DjangoTestSuiteRunner().run_tests(self.apps, verbosity=1) 131 | if failures: 132 | sys.exit(failures) 133 | 134 | def _tests_1_7(self): 135 | """ 136 | Fire up the Django test suite developed for version 1.7 and up 137 | """ 138 | INSTALLED_APPS, settings_test = self.get_custom_settings() 139 | 140 | settings.configure( 141 | DEBUG = True, 142 | DATABASES = self.get_database(), 143 | MIDDLEWARE_CLASSES = ('django.middleware.common.CommonMiddleware', 144 | 'django.middleware.csrf.CsrfViewMiddleware'), 145 | INSTALLED_APPS = self.INSTALLED_APPS + INSTALLED_APPS + self.apps, 146 | **settings_test 147 | ) 148 | 149 | from django.test.simple import DjangoTestSuiteRunner 150 | import django 151 | django.setup() 152 | failures = DjangoTestSuiteRunner().run_tests(self.apps, verbosity=1) 153 | if failures: 154 | sys.exit(failures) 155 | 156 | 157 | if __name__ == '__main__': 158 | """ 159 | What do when the user hits this file from the shell. 160 | 161 | Example usage: 162 | 163 | $ python quicktest.py app1 app2 164 | 165 | """ 166 | parser = argparse.ArgumentParser( 167 | usage="[args]", 168 | description="Run Django tests on the provided applications." 169 | ) 170 | parser.add_argument('apps', nargs='+', type=str) 171 | args = parser.parse_args() 172 | QuickDjangoTest(*args.apps) 173 | -------------------------------------------------------------------------------- /settings_test.py: -------------------------------------------------------------------------------- 1 | INSTALLED_APPS = ( 2 | 'oauth_tokens', 3 | 'm2m_history', 4 | 'taggit', 5 | ) 6 | 7 | OAUTH_TOKENS_TWITTER_CLIENT_ID = 'NLKrDQAE6YcSi76b0PGSg' 8 | OAUTH_TOKENS_TWITTER_CLIENT_SECRET = '4D8TBznBjiJWlRE00G4qETLNNmfFadiKbREDrmNSDE' 9 | OAUTH_TOKENS_TWITTER_USERNAME = 'baranus1@mail.ru' 10 | OAUTH_TOKENS_TWITTER_PASSWORD = 'jcej9EIAQrrptDBy' 11 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from setuptools import find_packages, setup 4 | 5 | setup( 6 | name='django-twitter-api', 7 | version=__import__('twitter_api').__version__, 8 | description='Django implementation for Twitter API', 9 | long_description=open('README.md').read(), 10 | author='ramusus', 11 | author_email='ramusus@gmail.com', 12 | url='https://github.com/ramusus/django-twitter-api', 13 | download_url='http://pypi.python.org/pypi/django-twitter-api', 14 | license='BSD', 15 | packages=find_packages(), 16 | include_package_data=True, 17 | zip_safe=False, # because we're including media that Django needs 18 | install_requires=[ 19 | 'django', 20 | 'django-annoying', 21 | 'django-picklefield', 22 | 'django-m2m-history', 23 | 'django-oauth-tokens>=0.5.1', 24 | 'django-m2m-history>=0.2.0', 25 | 'tweepy', 26 | ], 27 | classifiers=[ 28 | 'Development Status :: 4 - Beta', 29 | 'Environment :: Web Environment', 30 | 'Intended Audience :: Developers', 31 | 'License :: OSI Approved :: BSD License', 32 | 'Operating System :: OS Independent', 33 | 'Programming Language :: Python', 34 | 'Topic :: Software Development :: Libraries :: Python Modules', 35 | ], 36 | ) 37 | -------------------------------------------------------------------------------- /travis_django_version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | 4 | version = sys.argv[1] 5 | if version.startswith('http'): 6 | print(version) 7 | else: 8 | next_version = float(version) + 0.1 9 | print('Django>=%s,<%.1f' % (version, next_version)) -------------------------------------------------------------------------------- /twitter_api/__init__.py: -------------------------------------------------------------------------------- 1 | VERSION = (0, 2, 10) 2 | __version__ = '.'.join(map(str, VERSION)) 3 | -------------------------------------------------------------------------------- /twitter_api/admin.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from django.contrib import admin 3 | from models import Status, User 4 | 5 | 6 | class TwitterModelAdmin(admin.ModelAdmin): 7 | 8 | def get_readonly_fields(self, request, obj=None): 9 | if obj: 10 | return [field.name for field in obj._meta.fields] 11 | return [] 12 | 13 | 14 | class StatusAdmin(TwitterModelAdmin): 15 | list_display = ['id', 'author', 'text'] 16 | 17 | 18 | class UserAdmin(TwitterModelAdmin): 19 | exclude = ('followers',) 20 | search_fields = ('name', 'screen_name') 21 | 22 | 23 | admin.site.register(Status, StatusAdmin) 24 | admin.site.register(User, UserAdmin) 25 | -------------------------------------------------------------------------------- /twitter_api/api.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import sys 3 | 4 | from django.conf import settings 5 | from oauth_tokens.api import ApiAbstractBase, Singleton 6 | from oauth_tokens.models import AccessToken 7 | from tweepy import TweepError as TwitterError 8 | import tweepy 9 | 10 | __all__ = ['api_call', 'TwitterError'] 11 | 12 | TWITTER_CLIENT_ID = getattr(settings, 'OAUTH_TOKENS_TWITTER_CLIENT_ID', None) 13 | TWITTER_CLIENT_SECRET = getattr(settings, 'OAUTH_TOKENS_TWITTER_CLIENT_SECRET', None) 14 | 15 | @property 16 | def code(self): 17 | return self[0][0]['code'] if 'code' in self[0][0] else 0 18 | 19 | TwitterError.code = code 20 | 21 | 22 | class TwitterApi(ApiAbstractBase): 23 | 24 | __metaclass__ = Singleton 25 | 26 | provider = 'twitter' 27 | error_class = TwitterError 28 | sleep_repeat_error_messages = [ 29 | 'Failed to send request:' 30 | ] 31 | 32 | def get_consistent_token(self): 33 | return getattr(settings, 'TWITTER_API_ACCESS_TOKEN', None) 34 | 35 | def get_api(self, token): 36 | delimeter = AccessToken.objects.get_token_class(self.provider).delimeter 37 | auth = tweepy.OAuthHandler(TWITTER_CLIENT_ID, TWITTER_CLIENT_SECRET) 38 | auth.set_access_token(*token.split(delimeter)) 39 | return tweepy.API(auth, wait_on_rate_limit=True, retry_count=3, retry_delay=1, retry_errors=set([401, 404, 500, 503])) 40 | 41 | def get_api_response(self, *args, **kwargs): 42 | return getattr(self.api, self.method)(*args, **kwargs) 43 | 44 | def handle_error_no_active_tokens(self, e, *args, **kwargs): 45 | if self.used_access_tokens and self.api: 46 | 47 | # check if all tokens are blocked by rate limits response 48 | try: 49 | rate_limit_status = self.api.rate_limit_status() 50 | except self.error_class, e: 51 | # handle rate limit on rate_limit_status request -> wait 15 min and repeat main request 52 | if self.get_error_code(e) == 88: 53 | self.used_access_tokens = [] 54 | return self.sleep_repeat_call(seconds=60 * 15, *args, **kwargs) 55 | else: 56 | raise 57 | 58 | # TODO: wrong logic, path is different completelly sometimes 59 | method = '/%s' % self.method.replace('_', '/') 60 | status = [methods for methods in rate_limit_status['resources'].values() if method in methods][0][method] 61 | if status['remaining'] == 0: 62 | secs = (datetime.fromtimestamp(status['reset']) - datetime.now()).seconds 63 | self.used_access_tokens = [] 64 | return self.sleep_repeat_call(seconds=secs, *args, **kwargs) 65 | else: 66 | return self.repeat_call(*args, **kwargs) 67 | else: 68 | return super(TwitterApi, self).handle_error_no_active_tokens(e, *args, **kwargs) 69 | 70 | def handle_error_code_88(self, e, *args, **kwargs): 71 | # Rate limit exceeded 72 | self.logger.warning("Rate limit exceeded: %s, method: %s recursion count: %d" % 73 | (e, self.method, self.recursion_count)) 74 | token = AccessToken.objects.get_token_class(self.provider).delimeter.join( 75 | [self.api.auth.access_token, self.api.auth.access_token_secret]) 76 | self.used_access_tokens += [token] 77 | return self.repeat_call(*args, **kwargs) 78 | 79 | # def handle_error_code_63(self, e, *args, **kwargs): 80 | # # User has been suspended. 81 | # self.refresh_tokens() 82 | # return self.repeat_call(*args, **kwargs) 83 | 84 | 85 | def api_call(*args, **kwargs): 86 | api = TwitterApi() 87 | return api.call(*args, **kwargs) 88 | -------------------------------------------------------------------------------- /twitter_api/decorators.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from django.utils.functional import wraps 3 | from django.db.models import Min 4 | 5 | 6 | def opt_arguments(func): 7 | ''' 8 | Meta-decorator for ablity use decorators with optional arguments 9 | from here http://www.ellipsix.net/blog/2010/08/more-python-voodoo-optional-argument-decorators.html 10 | ''' 11 | def meta_wrapper(*args, **kwargs): 12 | if len(args) == 1 and callable(args[0]): 13 | # No arguments, this is the decorator 14 | # Set default values for the arguments 15 | return func(args[0]) 16 | else: 17 | def meta_func(inner_func): 18 | return func(inner_func, *args, **kwargs) 19 | return meta_func 20 | return meta_wrapper 21 | 22 | @opt_arguments 23 | def fetch_all(func, max_count): 24 | """ 25 | Class method decorator for fetching all items. Add parameter `all=False` for decored method. 26 | If `all` is True, method runs as many times as it returns any results. 27 | Decorator receive 2 parameters: 28 | * integer `max_count` - max number of items method able to return 29 | Usage: 30 | 31 | @fetch_all(max_count=200) 32 | def fetch_something(self, ..., *kwargs): 33 | .... 34 | """ 35 | def wrapper(self, all=False, return_instances=None, *args, **kwargs): 36 | if all: 37 | if return_instances is None: 38 | return_instances = [] 39 | kwargs['count'] = max_count 40 | instances = func(self, *args, **kwargs) 41 | instances_count = len(instances) 42 | min_id = instances.aggregate(minid=Min('id'))['minid'] 43 | return_instances += instances 44 | 45 | if instances_count > 1: 46 | kwargs['max_id'] = min_id 47 | return wrapper(self, all=True, return_instances=return_instances, *args, **kwargs) 48 | else: 49 | return self.model.objects.filter(id__in=[instance.id for instance in return_instances]) 50 | else: 51 | return func(self, *args, **kwargs) 52 | 53 | return wraps(func)(wrapper) 54 | -------------------------------------------------------------------------------- /twitter_api/factories.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from django.utils import timezone 4 | 5 | import factory 6 | import models 7 | 8 | 9 | class UserFactory(factory.DjangoModelFactory): 10 | 11 | id = factory.Sequence(lambda n: n) 12 | screen_name = factory.Sequence(lambda n: n) 13 | created_at = factory.LazyAttribute(lambda o: timezone.now()) 14 | entities = {} 15 | 16 | favorites_count = factory.LazyAttribute(lambda o: random.randint(0, 1000)) 17 | followers_count = factory.LazyAttribute(lambda o: random.randint(0, 1000)) 18 | friends_count = factory.LazyAttribute(lambda o: random.randint(0, 1000)) 19 | listed_count = factory.LazyAttribute(lambda o: random.randint(0, 1000)) 20 | statuses_count = factory.LazyAttribute(lambda o: random.randint(0, 1000)) 21 | utc_offset = factory.LazyAttribute(lambda o: random.randint(0, 1000)) 22 | 23 | class Meta: 24 | model = models.User 25 | 26 | 27 | class StatusFactory(factory.DjangoModelFactory): 28 | 29 | id = factory.Sequence(lambda n: n) 30 | created_at = factory.LazyAttribute(lambda o: timezone.now()) 31 | entities = {} 32 | 33 | author = factory.SubFactory(UserFactory) 34 | favorites_count = factory.LazyAttribute(lambda o: random.randint(0, 1000)) 35 | retweets_count = factory.LazyAttribute(lambda o: random.randint(0, 1000)) 36 | replies_count = factory.LazyAttribute(lambda o: random.randint(0, 1000)) 37 | 38 | class Meta: 39 | model = models.Status 40 | -------------------------------------------------------------------------------- /twitter_api/fields.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from django.db import models 3 | from django.core import validators 4 | from django.utils.translation import ugettext_lazy as _ 5 | from annoying.fields import JSONField 6 | from picklefield.fields import PickledObjectField 7 | import re 8 | 9 | class PositiveSmallIntegerRangeField(models.PositiveSmallIntegerField): 10 | ''' 11 | Range integer field with max_value and min_value properties 12 | from here http://stackoverflow.com/questions/849142/how-to-limit-the-maximum-value-of-a-numeric-field-in-a-django-model 13 | ''' 14 | def __init__(self, verbose_name=None, name=None, min_value=None, max_value=None, **kwargs): 15 | self.min_value, self.max_value = min_value, max_value 16 | models.IntegerField.__init__(self, verbose_name, name, **kwargs) 17 | 18 | def formfield(self, **kwargs): 19 | defaults = {'min_value': self.min_value, 'max_value':self.max_value} 20 | defaults.update(kwargs) 21 | return super(PositiveSmallIntegerRangeField, self).formfield(**defaults) 22 | 23 | comma_separated_string_list_re = re.compile(u'^(?u)[\w#\[\], ]+$') 24 | validate_comma_separated_string_list = validators.RegexValidator(comma_separated_string_list_re, _(u'Enter values separated by commas.'), 'invalid') 25 | 26 | class CommaSeparatedCharField(models.CharField): 27 | ''' 28 | Field for comma-separated strings 29 | TODO: added max_number validator 30 | ''' 31 | default_validators = [validate_comma_separated_string_list] 32 | description = _("Comma-separated strings") 33 | 34 | def formfield(self, **kwargs): 35 | defaults = { 36 | 'error_messages': { 37 | 'invalid': _(u'Enter values separated by commas.'), 38 | } 39 | } 40 | defaults.update(kwargs) 41 | return super(CommaSeparatedCharField, self).formfield(**defaults) 42 | 43 | try: 44 | from south.modelsinspector import add_introspection_rules 45 | add_introspection_rules([], ["^facebook_api\.fields"]) 46 | add_introspection_rules([], ["^annoying\.fields"]) 47 | except ImportError: 48 | pass -------------------------------------------------------------------------------- /twitter_api/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | import m2m_history.fields 6 | import annoying.fields 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name='Status', 17 | fields=[ 18 | ('id', models.BigIntegerField(serialize=False, primary_key=True)), 19 | ('created_at', models.DateTimeField()), 20 | ('lang', models.CharField(max_length=10)), 21 | ('entities', annoying.fields.JSONField()), 22 | ('fetched', models.DateTimeField(null=True, verbose_name='Fetched', blank=True)), 23 | ('text', models.TextField()), 24 | ('favorited', models.BooleanField(default=False)), 25 | ('retweeted', models.BooleanField(default=False)), 26 | ('truncated', models.BooleanField(default=False)), 27 | ('source', models.CharField(max_length=100)), 28 | ('source_url', models.URLField(null=True)), 29 | ('favorites_count', models.PositiveIntegerField()), 30 | ('retweets_count', models.PositiveIntegerField()), 31 | ('replies_count', models.PositiveIntegerField(null=True)), 32 | ('place', annoying.fields.JSONField(null=True)), 33 | ('contributors', annoying.fields.JSONField(null=True)), 34 | ('coordinates', annoying.fields.JSONField(null=True)), 35 | ('geo', annoying.fields.JSONField(null=True)), 36 | ], 37 | options={ 38 | 'abstract': False, 39 | }, 40 | bases=(models.Model,), 41 | ), 42 | migrations.CreateModel( 43 | name='User', 44 | fields=[ 45 | ('id', models.BigIntegerField(serialize=False, primary_key=True)), 46 | ('created_at', models.DateTimeField()), 47 | ('lang', models.CharField(max_length=10)), 48 | ('entities', annoying.fields.JSONField()), 49 | ('fetched', models.DateTimeField(null=True, verbose_name='Fetched', blank=True)), 50 | ('screen_name', models.CharField(unique=True, max_length=50, verbose_name='Screen name')), 51 | ('name', models.CharField(max_length=100, verbose_name='Name')), 52 | ('description', models.TextField(verbose_name='Description')), 53 | ('location', models.CharField(max_length=100, verbose_name='Location')), 54 | ('time_zone', models.CharField(max_length=100, null=True, verbose_name='Time zone')), 55 | ('contributors_enabled', models.BooleanField(default=False, verbose_name='Contributors enabled')), 56 | ('default_profile', models.BooleanField(default=False, verbose_name='Default profile')), 57 | ('default_profile_image', models.BooleanField(default=False, verbose_name='Default profile image')), 58 | ('follow_request_sent', models.BooleanField(default=False, verbose_name='Follow request sent')), 59 | ('following', models.BooleanField(default=False, verbose_name='Following')), 60 | ('geo_enabled', models.BooleanField(default=False, verbose_name='Geo enabled')), 61 | ('is_translator', models.BooleanField(default=False, verbose_name='Is translator')), 62 | ('notifications', models.BooleanField(default=False, verbose_name='Notifications')), 63 | ('profile_use_background_image', models.BooleanField(default=False, verbose_name='Profile use background image')), 64 | ('protected', models.BooleanField(default=False, verbose_name='Protected')), 65 | ('verified', models.BooleanField(default=False, verbose_name='Verified')), 66 | ('profile_background_image_url', models.URLField(max_length=300, null=True)), 67 | ('profile_background_image_url_https', models.URLField(max_length=300, null=True)), 68 | ('profile_background_tile', models.BooleanField(default=False)), 69 | ('profile_background_color', models.CharField(max_length=6)), 70 | ('profile_banner_url', models.URLField(max_length=300, null=True)), 71 | ('profile_image_url', models.URLField(max_length=300, null=True)), 72 | ('profile_image_url_https', models.URLField(max_length=300)), 73 | ('url', models.URLField(max_length=300, null=True)), 74 | ('profile_link_color', models.CharField(max_length=6)), 75 | ('profile_sidebar_border_color', models.CharField(max_length=6)), 76 | ('profile_sidebar_fill_color', models.CharField(max_length=6)), 77 | ('profile_text_color', models.CharField(max_length=6)), 78 | ('favorites_count', models.PositiveIntegerField()), 79 | ('followers_count', models.PositiveIntegerField()), 80 | ('friends_count', models.PositiveIntegerField()), 81 | ('listed_count', models.PositiveIntegerField()), 82 | ('statuses_count', models.PositiveIntegerField()), 83 | ('utc_offset', models.IntegerField(null=True)), 84 | ('followers', m2m_history.fields.ManyToManyHistoryField(to='twitter_api.User')), 85 | ], 86 | options={ 87 | 'abstract': False, 88 | }, 89 | bases=(models.Model,), 90 | ), 91 | migrations.AddField( 92 | model_name='status', 93 | name='author', 94 | field=models.ForeignKey(related_name='statuses', to='twitter_api.User'), 95 | preserve_default=True, 96 | ), 97 | migrations.AddField( 98 | model_name='status', 99 | name='favorites_users', 100 | field=m2m_history.fields.ManyToManyHistoryField(related_name='favorites', to='twitter_api.User'), 101 | preserve_default=True, 102 | ), 103 | migrations.AddField( 104 | model_name='status', 105 | name='in_reply_to_status', 106 | field=models.ForeignKey(related_name='replies', to='twitter_api.Status', null=True), 107 | preserve_default=True, 108 | ), 109 | migrations.AddField( 110 | model_name='status', 111 | name='in_reply_to_user', 112 | field=models.ForeignKey(related_name='replies', to='twitter_api.User', null=True), 113 | preserve_default=True, 114 | ), 115 | migrations.AddField( 116 | model_name='status', 117 | name='retweeted_status', 118 | field=models.ForeignKey(related_name='retweets', to='twitter_api.Status', null=True), 119 | preserve_default=True, 120 | ), 121 | ] 122 | -------------------------------------------------------------------------------- /twitter_api/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramusus/django-twitter-api/dec2f3c15c332f7e7a524d92dc3e15468aa2d85e/twitter_api/migrations/__init__.py -------------------------------------------------------------------------------- /twitter_api/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import logging 3 | import re 4 | import collections 5 | from datetime import datetime 6 | 7 | import tweepy 8 | from django.db import models 9 | from django.db.models.fields import FieldDoesNotExist 10 | from django.utils import timezone 11 | from m2m_history.fields import ManyToManyHistoryField 12 | 13 | from . import fields 14 | from .api import TwitterError, api_call 15 | from .decorators import fetch_all 16 | from .parser import get_replies 17 | 18 | try: 19 | from django.db.transaction import atomic 20 | except ImportError: 21 | from django.db.transaction import commit_on_success as atomic 22 | 23 | try: 24 | from django.db.models.related import RelatedObject as ForeignObjectRel 25 | except: 26 | # django 1.8 + 27 | from django.db.models.fields.related import ForeignObjectRel 28 | 29 | __all__ = ['User', 'Status', 'TwitterContentError', 'TwitterModel', 'TwitterManager', 'UserManager'] 30 | 31 | log = logging.getLogger('twitter_api') 32 | 33 | 34 | class TwitterContentError(Exception): 35 | pass 36 | 37 | 38 | class TwitterManager(models.Manager): 39 | 40 | """ 41 | Twitter Manager for RESTful CRUD operations 42 | """ 43 | 44 | def __init__(self, methods=None, remote_pk=None, *args, **kwargs): 45 | if methods and len(methods.items()) < 1: 46 | raise ValueError('Argument methods must contains at least 1 specified method') 47 | 48 | self.methods = methods or {} 49 | self.remote_pk = remote_pk or ('id',) 50 | if not isinstance(self.remote_pk, tuple): 51 | self.remote_pk = (self.remote_pk,) 52 | 53 | super(TwitterManager, self).__init__(*args, **kwargs) 54 | 55 | def get_by_url(self, url): 56 | """ 57 | Return object by url 58 | """ 59 | m = re.findall(r'(?:https?://)?(?:www\.)?twitter\.com/([^/]+)/?', url) 60 | if not len(m): 61 | raise ValueError("Url should be started with https://twitter.com/") 62 | 63 | return self.get_by_slug(m[0]) 64 | 65 | def get_by_slug(self, slug): 66 | """ 67 | Return object by slug 68 | """ 69 | # TODO: change to self.get method 70 | return self.model.remote.fetch(slug) 71 | 72 | def get_or_create_from_instance(self, instance): 73 | remote_pk_dict = {} 74 | for field_name in self.remote_pk: 75 | remote_pk_dict[field_name] = getattr(instance, field_name) 76 | 77 | try: 78 | instance_old = self.model.objects.get(**remote_pk_dict) 79 | instance._substitute(instance_old) 80 | instance.save() 81 | except self.model.DoesNotExist: 82 | instance.save() 83 | log.debug('Fetch and create new object %s with remote pk %s' % (self.model, remote_pk_dict)) 84 | 85 | return instance 86 | 87 | # def get_or_create_from_resource(self, resource): 88 | # 89 | # instance = self.model() 90 | # instance.parse(dict(resource)) 91 | # 92 | # return self.get_or_create_from_instance(instance) 93 | 94 | def api_call(self, method, *args, **kwargs): 95 | if method in self.methods: 96 | method = self.methods[method] 97 | return api_call(method, *args, **kwargs) 98 | 99 | def fetch(self, *args, **kwargs): 100 | """ 101 | Retrieve and save object to local DB 102 | """ 103 | result = self.get(*args, **kwargs) 104 | if isinstance(result, collections.Iterable): 105 | return self.filter(pk__in=[self.get_or_create_from_instance(instance).pk for instance in result]) 106 | else: 107 | return self.get_or_create_from_instance(result) 108 | 109 | def get(self, *args, **kwargs): 110 | """ 111 | Retrieve objects from remote server 112 | """ 113 | method = kwargs.pop('method', 'get') 114 | extra_fields = kwargs.pop('extra_fields', {}) 115 | extra_fields['fetched'] = timezone.now() 116 | response = self.api_call(method, *args, **kwargs) 117 | 118 | return self.parse_response(response, extra_fields) 119 | 120 | def parse_response(self, response, extra_fields=None): 121 | # if response is None: 122 | # return [] 123 | # el 124 | if isinstance(response, (list, tuple)): 125 | return self.parse_response_list(response, extra_fields) 126 | elif isinstance(response, tweepy.models.Model): 127 | return self.parse_response_object(response, extra_fields) 128 | else: 129 | raise TwitterContentError('Twitter response should be list or dict, not %s' % response) 130 | 131 | def parse_response_object(self, resource, extra_fields=None): 132 | 133 | instance = self.model() 134 | # important to do it before calling parse method 135 | if extra_fields: 136 | instance.__dict__.update(extra_fields) 137 | instance.set_tweepy(resource) 138 | instance.parse() 139 | 140 | return instance 141 | 142 | def parse_response_list(self, response_list, extra_fields=None): 143 | 144 | instances = [] 145 | for response in response_list: 146 | 147 | if not isinstance(response, tweepy.models.Model): 148 | log.error("Resource %s is not dictionary" % response) 149 | continue 150 | 151 | instance = self.parse_response_object(response, extra_fields) 152 | instances += [instance] 153 | 154 | return instances 155 | 156 | 157 | class TwitterTimelineManager(TwitterManager): 158 | 159 | """ 160 | Manager class, child of OdnoklassnikiManager for fetching objects with arguments `after`, `before` 161 | """ 162 | timeline_cut_fieldname = 'created_at' 163 | timeline_force_ordering = True 164 | 165 | def get_timeline_date(self, instance): 166 | return getattr(instance, self.timeline_cut_fieldname, datetime(1970, 1, 1).replace(tzinfo=timezone.utc)) 167 | 168 | @atomic 169 | def get(self, *args, **kwargs): 170 | """ 171 | Retrieve objects and return result list with respect to parameters: 172 | * 'after' - excluding all items before. 173 | * 'before' - excluding all items after. 174 | """ 175 | after = kwargs.pop('after', None) 176 | before = kwargs.pop('before', None) 177 | 178 | if before and not after: 179 | raise ValueError("Attribute `before` should be specified with attribute `after`") 180 | if before and before < after: 181 | raise ValueError("Attribute `before` should be later, than attribute `after`") 182 | 183 | result = super(TwitterTimelineManager, self).get(*args, **kwargs) 184 | if not isinstance(result, collections.Iterable): 185 | return result 186 | 187 | if self.timeline_force_ordering and result: 188 | result.sort(key=self.get_timeline_date, reverse=True) 189 | 190 | instances = [] 191 | for instance in result: 192 | 193 | timeline_date = self.get_timeline_date(instance) 194 | 195 | if timeline_date and isinstance(timeline_date, datetime): 196 | 197 | if after and after > timeline_date: 198 | break 199 | 200 | if before and before < timeline_date: 201 | continue 202 | 203 | instances += [instance] 204 | 205 | return instances 206 | 207 | 208 | class UserManager(TwitterManager): 209 | 210 | def get_followers_ids_for_user(self, user, all=False, count=5000, **kwargs): 211 | # https://dev.twitter.com/docs/api/1.1/get/followers/ids 212 | if all: 213 | cursor = tweepy.Cursor(user.tweepy._api.followers_ids, id=user.pk, count=count) 214 | return list(cursor.items()) 215 | else: 216 | raise NotImplementedError("This method implemented only with argument all=True") 217 | 218 | def fetch_followers_for_user(self, user, all=False, count=200, **kwargs): 219 | # https://dev.twitter.com/docs/api/1.1/get/followers/list 220 | # in docs default count is 20, but maximum is 200 221 | if all is False: 222 | raise NotImplementedError("This method implemented only with argument all=True") 223 | 224 | # TODO: make optimization: break cursor iteration after getting already 225 | # existing user and switch to ids REST method 226 | ids = [] 227 | cursor = tweepy.Cursor(user.tweepy._api.followers, id=user.pk, count=count) 228 | for instance in cursor.items(): 229 | instance = self.parse_response_object(instance) 230 | instance = self.get_or_create_from_instance(instance) 231 | ids += [instance.pk] 232 | 233 | initial = user.followers.versions.count() == 0 234 | 235 | user.followers = ids 236 | 237 | if initial: 238 | user.followers.get_queryset_through().update(time_from=None) 239 | user.followers.versions.update(added_count=0) 240 | 241 | return user.followers.all() 242 | 243 | def get_or_create_from_instance(self, instance): 244 | try: 245 | instance_old = self.model.objects.get(screen_name=instance.screen_name) 246 | if instance_old.id == instance.id: 247 | instance._substitute(instance_old) 248 | instance.save() 249 | else: 250 | # perhaps we already have old User with the same screen_name, but different id 251 | try: 252 | self.fetch(instance_old.pk) 253 | except TwitterError, e: 254 | if e.code == 34: 255 | instance._substitute(instance_old) 256 | instance_old.delete() 257 | instance.save() 258 | else: 259 | raise 260 | return instance 261 | except self.model.DoesNotExist: 262 | return super(UserManager, self).get_or_create_from_instance(instance) 263 | 264 | 265 | class StatusManager(TwitterTimelineManager): 266 | 267 | @fetch_all(max_count=200) 268 | def fetch_for_user(self, user, count=20, **kwargs): 269 | # https://dev.twitter.com/docs/api/1.1/get/statuses/user_timeline 270 | kwargs['extra_fields'] = {'user_id': user.pk} 271 | kwargs['count'] = count 272 | kwargs['id'] = user.pk 273 | return self.fetch(method='user_timeline', **kwargs) 274 | 275 | def fetch_retweets(self, status, count=100, **kwargs): 276 | # https://dev.twitter.com/docs/api/1.1/get/statuses/retweets/%3Aid 277 | kwargs['count'] = count 278 | kwargs['id'] = status.pk 279 | return self.fetch(method='retweets', **kwargs) 280 | 281 | def fetch_replies(self, status, **kwargs): 282 | instances = Status.objects.none() 283 | 284 | replies_ids = get_replies(status) 285 | for id in replies_ids: 286 | instance = Status.remote.fetch(id) 287 | instances |= Status.objects.filter(pk=instance.pk) 288 | 289 | status.replies_count = instances.count() 290 | status.save() 291 | 292 | return instances 293 | 294 | 295 | class TwitterModel(models.Model): 296 | 297 | objects = models.Manager() 298 | 299 | class Meta: 300 | abstract = True 301 | 302 | def __init__(self, *args, **kwargs): 303 | super(TwitterModel, self).__init__(*args, **kwargs) 304 | 305 | # different lists for saving related objects 306 | self._external_links_post_save = [] 307 | self._foreignkeys_pre_save = [] 308 | self._external_links_to_add = [] 309 | 310 | def save(self, *args, **kwargs): 311 | """ 312 | Save all related instances before or after current instance 313 | """ 314 | for field, instance in self._foreignkeys_pre_save: 315 | instance = instance.__class__.remote.get_or_create_from_instance(instance) 316 | setattr(self, field, instance) 317 | self._foreignkeys_pre_save = [] 318 | 319 | try: 320 | super(TwitterModel, self).save(*args, **kwargs) 321 | except Exception as e: 322 | import sys 323 | raise type(e), type(e)(e.message + ' while saving %s' % self.__dict__), sys.exc_info()[2] 324 | 325 | for field, instance in self._external_links_post_save: 326 | # set foreignkey to the main instance 327 | setattr(instance, field, self) 328 | instance.__class__.remote.get_or_create_from_instance(instance) 329 | self._external_links_post_save = [] 330 | 331 | for field, instance in self._external_links_to_add: 332 | # if there is already connected instances, then continue, because it's hard to check for duplicates 333 | if getattr(self, field).count(): 334 | continue 335 | getattr(self, field).add(instance) 336 | self._external_links_to_add = [] 337 | 338 | def _substitute(self, old_instance): 339 | """ 340 | Substitute new user with old one while updating in method Manager.get_or_create_from_instance() 341 | Can be overrided in child models 342 | """ 343 | self.pk = old_instance.pk 344 | 345 | def parse(self): 346 | """ 347 | Parse API response and define fields with values 348 | """ 349 | for key, value in self._response.items(): 350 | if key == '_api': 351 | continue 352 | 353 | try: 354 | field, model, direct, m2m = self._meta.get_field_by_name(key) 355 | except FieldDoesNotExist: 356 | log.debug('Field with name "%s" doesn\'t exist in the model %s' % (key, type(self))) 357 | continue 358 | 359 | if isinstance(field, ForeignObjectRel) and value: 360 | for item in value: 361 | rel_instance = field.model.remote.parse_response_object(item) 362 | self._external_links_post_save += [(field.field.name, rel_instance)] 363 | else: 364 | if isinstance(field, (models.BooleanField)): 365 | value = bool(value) 366 | 367 | elif isinstance(field, (models.OneToOneField, models.ForeignKey)) and value: 368 | rel_instance = field.rel.to.remote.parse_response_object(value) 369 | value = rel_instance 370 | if isinstance(field, models.ForeignKey): 371 | self._foreignkeys_pre_save += [(key, rel_instance)] 372 | 373 | elif isinstance(field, (fields.CommaSeparatedCharField, models.CommaSeparatedIntegerField)) and isinstance(value, list): 374 | value = ','.join([unicode(v) for v in value]) 375 | 376 | elif isinstance(field, (models.CharField, models.TextField)) and value: 377 | if isinstance(value, (str, unicode)): 378 | value = value.strip() 379 | 380 | elif isinstance(field, (models.DateTimeField)): 381 | value = value.replace(tzinfo=timezone.utc) 382 | 383 | setattr(self, key, value) 384 | 385 | def _get_foreignkeys_for_fields(self, *args): 386 | 387 | for field_name in args: 388 | model = self._meta.get_field(field_name).rel.to 389 | try: 390 | id = int(self._response.pop(field_name + '_id', None)) 391 | setattr(self, field_name, model.objects.get(pk=id)) 392 | except model.DoesNotExist: 393 | try: 394 | self._foreignkeys_pre_save += [(field_name, model.remote.get(id))] 395 | except tweepy.TweepError: 396 | pass 397 | except TypeError: 398 | pass 399 | 400 | 401 | class TwitterBaseModel(TwitterModel): 402 | 403 | _tweepy_model = None 404 | _response = None 405 | 406 | id = models.BigIntegerField(primary_key=True) 407 | created_at = models.DateTimeField() 408 | lang = models.CharField(max_length=10) 409 | entities = fields.JSONField() 410 | 411 | fetched = models.DateTimeField(u'Fetched', null=True, blank=True) 412 | 413 | class Meta: 414 | abstract = True 415 | 416 | def set_tweepy(self, model): 417 | self._tweepy_model = model 418 | self._response = dict(self._tweepy_model.__dict__) 419 | 420 | def save(self, *args, **kwargs): 421 | if len(self.lang) > 10: 422 | self.lang = '' 423 | super(TwitterBaseModel, self).save(*args, **kwargs) 424 | 425 | @property 426 | def tweepy(self): 427 | if not self._tweepy_model: 428 | # get fresh instance with the same ID, set tweepy object and refresh attributes 429 | instance = self.__class__.remote.get(self.pk) 430 | self.set_tweepy(instance.tweepy) 431 | self.parse() 432 | return self._tweepy_model 433 | 434 | def parse(self): 435 | self._response.pop('id_str', None) 436 | super(TwitterBaseModel, self).parse() 437 | 438 | def get_url(self): 439 | return 'https://twitter.com/%s' % self.slug 440 | 441 | 442 | class User(TwitterBaseModel): 443 | 444 | screen_name = models.CharField(u'Screen name', max_length=50, unique=True) 445 | 446 | name = models.CharField(u'Name', max_length=100) 447 | description = models.TextField(u'Description') 448 | location = models.CharField(u'Location', max_length=100) 449 | time_zone = models.CharField(u'Time zone', max_length=100, null=True) 450 | 451 | contributors_enabled = models.BooleanField(u'Contributors enabled', default=False) 452 | default_profile = models.BooleanField(u'Default profile', default=False) 453 | default_profile_image = models.BooleanField(u'Default profile image', default=False) 454 | follow_request_sent = models.BooleanField(u'Follow request sent', default=False) 455 | following = models.BooleanField(u'Following', default=False) 456 | geo_enabled = models.BooleanField(u'Geo enabled', default=False) 457 | is_translator = models.BooleanField(u'Is translator', default=False) 458 | notifications = models.BooleanField(u'Notifications', default=False) 459 | profile_use_background_image = models.BooleanField(u'Profile use background image', default=False) 460 | protected = models.BooleanField(u'Protected', default=False) 461 | verified = models.BooleanField(u'Verified', default=False) 462 | 463 | profile_background_image_url = models.URLField(max_length=300, null=True) 464 | profile_background_image_url_https = models.URLField(max_length=300, null=True) 465 | profile_background_tile = models.BooleanField(default=False) 466 | profile_background_color = models.CharField(max_length=6) 467 | profile_banner_url = models.URLField(max_length=300, null=True) 468 | profile_image_url = models.URLField(max_length=300, null=True) 469 | profile_image_url_https = models.URLField(max_length=300) 470 | url = models.URLField(max_length=300, null=True) 471 | 472 | profile_link_color = models.CharField(max_length=6) 473 | profile_sidebar_border_color = models.CharField(max_length=6) 474 | profile_sidebar_fill_color = models.CharField(max_length=6) 475 | profile_text_color = models.CharField(max_length=6) 476 | 477 | favorites_count = models.PositiveIntegerField() 478 | followers_count = models.PositiveIntegerField() 479 | friends_count = models.PositiveIntegerField() 480 | listed_count = models.PositiveIntegerField() 481 | statuses_count = models.PositiveIntegerField() 482 | utc_offset = models.IntegerField(null=True) 483 | 484 | followers = ManyToManyHistoryField('User', versions=True) 485 | 486 | objects = models.Manager() 487 | remote = UserManager(methods={ 488 | 'get': 'get_user', 489 | }) 490 | 491 | def __unicode__(self): 492 | return self.name 493 | 494 | def save(self, *args, **kwargs): 495 | if self.friends_count < 0: 496 | log.warning('Negative value friends_count=%s set to 0 for user ID %s' % (self.friends_count, self.id)) 497 | self.friends_count = 0 498 | super(User, self).save(*args, **kwargs) 499 | 500 | @property 501 | def slug(self): 502 | return self.screen_name 503 | 504 | def parse(self): 505 | self._response['favorites_count'] = self._response.pop('favourites_count', None) 506 | self._response.pop('status', None) 507 | super(User, self).parse() 508 | 509 | def fetch_followers(self, **kwargs): 510 | return User.remote.fetch_followers_for_user(user=self, **kwargs) 511 | 512 | def get_followers_ids(self, **kwargs): 513 | return User.remote.get_followers_ids_for_user(user=self, **kwargs) 514 | 515 | def fetch_statuses(self, **kwargs): 516 | return Status.remote.fetch_for_user(user=self, **kwargs) 517 | 518 | 519 | class Status(TwitterBaseModel): 520 | 521 | author = models.ForeignKey('User', related_name='statuses') 522 | 523 | text = models.TextField() 524 | 525 | favorited = models.BooleanField(default=False) 526 | retweeted = models.BooleanField(default=False) 527 | truncated = models.BooleanField(default=False) 528 | 529 | source = models.CharField(max_length=100) 530 | source_url = models.URLField(null=True) 531 | 532 | favorites_count = models.PositiveIntegerField() 533 | retweets_count = models.PositiveIntegerField() 534 | replies_count = models.PositiveIntegerField(null=True) 535 | 536 | in_reply_to_status = models.ForeignKey('Status', null=True, related_name='replies') 537 | in_reply_to_user = models.ForeignKey('User', null=True, related_name='replies') 538 | 539 | favorites_users = ManyToManyHistoryField('User', related_name='favorites') 540 | retweeted_status = models.ForeignKey('Status', null=True, related_name='retweets') 541 | 542 | place = fields.JSONField(null=True) 543 | # format the next fields doesn't clear 544 | contributors = fields.JSONField(null=True) 545 | coordinates = fields.JSONField(null=True) 546 | geo = fields.JSONField(null=True) 547 | 548 | objects = models.Manager() 549 | remote = StatusManager(methods={ 550 | 'get': 'get_status', 551 | }) 552 | 553 | def __unicode__(self): 554 | return u'%s: %s' % (self.author, self.text) 555 | 556 | @property 557 | def slug(self): 558 | return '/%s/status/%d' % (self.author.screen_name, self.pk) 559 | 560 | def _substitute(self, old_instance): 561 | super(Status, self)._substitute(old_instance) 562 | self.replies_count = old_instance.replies_count 563 | 564 | def parse(self): 565 | self._response['favorites_count'] = self._response.pop('favorite_count', 0) 566 | self._response['retweets_count'] = self._response.pop('retweet_count', 0) 567 | 568 | self._response.pop('user', None) 569 | self._response.pop('in_reply_to_screen_name', None) 570 | self._response.pop('in_reply_to_user_id_str', None) 571 | self._response.pop('in_reply_to_status_id_str', None) 572 | 573 | self._get_foreignkeys_for_fields('in_reply_to_status', 'in_reply_to_user') 574 | 575 | super(Status, self).parse() 576 | 577 | def fetch_retweets(self, **kwargs): 578 | return Status.remote.fetch_retweets(status=self, **kwargs) 579 | 580 | def fetch_replies(self, **kwargs): 581 | return Status.remote.fetch_replies(status=self, **kwargs) 582 | -------------------------------------------------------------------------------- /twitter_api/parser.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | from bs4 import BeautifulSoup 4 | from oauth_tokens.models import AccessToken 5 | 6 | HEADERS = { 7 | 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:33.0) Gecko/20100101 Firefox/33.0', 8 | 'Accept_Language': 'en' 9 | } 10 | IDS_RE = re.compile('data-tweet-id="(\d+)"') 11 | 12 | 13 | def get_replies(status): 14 | "Return all replies ids of tweet" 15 | 16 | replies_ids = set() 17 | 18 | url = 'https://twitter.com/i/%s/conversation/%s' % (status.author.screen_name, status.pk) 19 | ar = AccessToken.objects.get_token('twitter').auth_request 20 | 21 | headers = dict(HEADERS) 22 | headers['X-Requested-With'] = 'XMLHttpRequest' 23 | 24 | resp = ar.authorized_request(url=status.get_url(), headers=headers) 25 | params = {'max_position': BeautifulSoup(resp.content).find('div', **{'class': 'stream-container'})['data-min-position']} 26 | 27 | while True: 28 | r = ar.authorized_request(url=url, params=params, headers=headers) 29 | response = r.json() 30 | if 'descendants' in response: 31 | response = response['descendants'] 32 | 33 | ids = IDS_RE.findall(response['items_html']) 34 | [replies_ids.add(id) for id in ids] 35 | 36 | if response['has_more_items'] and len(ids): 37 | params = {'max_position': response['min_position']} 38 | else: 39 | break 40 | 41 | return list(replies_ids) 42 | -------------------------------------------------------------------------------- /twitter_api/sql/status.postgresql_psycopg2.sql: -------------------------------------------------------------------------------- 1 | --twitter_api_status_favorites_users 2 | 3 | CREATE UNIQUE INDEX twitter_api_status_favorites_users_time_from_3col_uniq 4 | ON twitter_api_status_favorites_users (status_id, user_id, time_from) 5 | WHERE time_from IS NOT NULL; 6 | 7 | CREATE UNIQUE INDEX twitter_api_status_favorites_users_time_from_2col_uniq 8 | ON twitter_api_status_favorites_users (status_id, user_id) 9 | WHERE time_from IS NULL; 10 | 11 | CREATE UNIQUE INDEX twitter_api_status_favorites_users_time_to_3col_uniq 12 | ON twitter_api_status_favorites_users (status_id, user_id, time_to) 13 | WHERE time_to IS NOT NULL; 14 | 15 | CREATE UNIQUE INDEX twitter_api_status_favorites_users_time_to_2col_uniq 16 | ON twitter_api_status_favorites_users (status_id, user_id) 17 | WHERE time_to IS NULL; 18 | -------------------------------------------------------------------------------- /twitter_api/sql/user.postgresql_psycopg2.sql: -------------------------------------------------------------------------------- 1 | --twitter_api_user_followers 2 | 3 | CREATE UNIQUE INDEX twitter_api_user_followers_time_from_3col_uniq 4 | ON twitter_api_user_followers (from_user_id, to_user_id, time_from) 5 | WHERE time_from IS NOT NULL; 6 | 7 | CREATE UNIQUE INDEX twitter_api_user_followers_time_from_2col_uniq 8 | ON twitter_api_user_followers (from_user_id, to_user_id) 9 | WHERE time_from IS NULL; 10 | 11 | CREATE UNIQUE INDEX twitter_api_user_followers_time_to_3col_uniq 12 | ON twitter_api_user_followers (from_user_id, to_user_id, time_to) 13 | WHERE time_to IS NOT NULL; 14 | 15 | CREATE UNIQUE INDEX twitter_api_user_followers_time_to_2col_uniq 16 | ON twitter_api_user_followers (from_user_id, to_user_id) 17 | WHERE time_to IS NULL; 18 | -------------------------------------------------------------------------------- /twitter_api/tests.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import time 3 | from datetime import datetime, timedelta 4 | 5 | from django.test import TestCase 6 | from django.utils import six, timezone 7 | from django.utils.timezone import is_aware 8 | import mock 9 | import tweepy 10 | 11 | from .api import api_call, TwitterApi, TwitterError 12 | from .factories import UserFactory, StatusFactory 13 | from .models import User, Status 14 | from .parser import get_replies 15 | 16 | 17 | STATUS_ID = 327926550815207424 18 | USER_ID = 813286 19 | USER_SCREEN_NAME = 'BarackObama' 20 | USER1_ID = 18807761 21 | USER1_SCREEN_NAME = 'voronezh' 22 | STATUS_MANY_REPLIES_ID = 538755896063832064 23 | STATUS_MANY_RETWEETS_ID = 329231054282055680 24 | 25 | 26 | class TwitterApiTest(TestCase): 27 | 28 | def raise_rate_limit(*a, **kw): 29 | raise tweepy.TweepError([{u'message': u'Rate limit exceeded', u'code': 88}]) 30 | 31 | def get_rate_limit_status(*a, **kw): 32 | return {u'rate_limit_context': {u'access_token': u''}, 33 | u'resources': { 34 | u'trends': {u'/trends/available': {u'limit': 15, 35 | u'remaining': 0, 36 | u'reset': time.time() + 100}}}} 37 | 38 | @mock.patch('tweepy.API.trends_available', side_effect=raise_rate_limit) 39 | @mock.patch('tweepy.API.rate_limit_status', side_effect=get_rate_limit_status) 40 | @mock.patch('twitter_api.api.TwitterApi.sleep_repeat_call') 41 | def test_rate_limit(self, sleep, rate_limit_status, trends_available): 42 | 43 | api_call('trends_available') 44 | 45 | self.assertTrue(trends_available.called) 46 | self.assertTrue(rate_limit_status.called) 47 | self.assertTrue(sleep.called) 48 | self.assertEqual(sleep.call_count, 1) 49 | self.assertGreater(sleep.call_args_list[0], 98) 50 | 51 | def test_user_screen_name_unique(self): 52 | 53 | # created user with unexisting pk 54 | user_wrong = UserFactory(pk=102732226, screen_name=USER_SCREEN_NAME) 55 | 56 | self.assertEqual(User.objects.count(), 1) 57 | self.assertEqual(User.objects.filter(pk=user_wrong.pk).count(), 1) 58 | 59 | user = User.remote.fetch(USER_SCREEN_NAME) 60 | 61 | self.assertEqual(User.objects.count(), 1) 62 | self.assertEqual(User.objects.filter(pk=user.pk).count(), 1) 63 | 64 | def test_request_error(self): 65 | 66 | with self.assertRaises(TwitterError): 67 | response = api_call('get_status') 68 | 69 | def test_api_instance_singleton(self): 70 | 71 | self.assertEqual(id(TwitterApi()), id(TwitterApi())) 72 | 73 | def test_request(self): 74 | 75 | response = api_call('get_status', STATUS_ID) 76 | self.assertEqual(response.text, '@mrshoranweyhey Thanks for the love! How about a follow for a follow? :) ^LF') 77 | self.assertEqual(response.source_url, 'http://www.exacttarget.com/social') 78 | 79 | response = api_call('get_user', USER_SCREEN_NAME) 80 | self.assertEqual(response.id, USER_ID) 81 | self.assertEqual(response.name, 'Barack Obama') 82 | 83 | def test_tweepy_properties(self): 84 | 85 | instance = User.remote.fetch(USER_ID) 86 | 87 | self.assertEqual(instance.screen_name, USER_SCREEN_NAME) 88 | self.assertIsInstance(instance.tweepy, tweepy.models.User) 89 | self.assertEqual(instance.tweepy.id_str, str(USER_ID)) 90 | 91 | def test_fetch_negative_friends_count(self): 92 | 93 | user = User.remote.fetch(961050745) 94 | # problem with saving User id = 2577412818 95 | user.fetch_statuses(count=100) 96 | 97 | def test_fetch_status(self): 98 | 99 | self.assertEqual(Status.objects.count(), 0) 100 | 101 | instance = Status.remote.fetch(STATUS_ID) 102 | 103 | self.assertEqual(Status.objects.count(), 2) 104 | 105 | self.assertEqual(instance.id, STATUS_ID) 106 | self.assertEqual(instance.source, 'SocialEngage') 107 | self.assertEqual(instance.source_url, 'http://www.exacttarget.com/social') 108 | self.assertEqual(instance.text, '@mrshoranweyhey Thanks for the love! How about a follow for a follow? :) ^LF') 109 | self.assertEqual(instance.in_reply_to_status_id, 327912852486762497) 110 | self.assertEqual(instance.in_reply_to_user_id, 1323314442) 111 | self.assertEqual(instance.in_reply_to_status, Status.objects.get(id=327912852486762497)) 112 | self.assertEqual(instance.in_reply_to_user, User.objects.get(id=1323314442)) 113 | self.assertIsInstance(instance.created_at, datetime) 114 | self.assertTrue(is_aware(instance.created_at)) 115 | 116 | def test_fetch_user(self): 117 | 118 | instance = User.remote.fetch(USER_ID) 119 | 120 | self.assertEqual(instance.screen_name, USER_SCREEN_NAME) 121 | self.assertEqual(instance.id, USER_ID) 122 | self.assertEqual(instance.name, 'Barack Obama') 123 | self.assertEqual(instance.location, 'Washington, DC') 124 | self.assertEqual(instance.verified, True) 125 | self.assertEqual(instance.lang, 'en') 126 | self.assertGreater(instance.followers_count, 30886141) 127 | self.assertGreater(instance.friends_count, 600000) 128 | self.assertGreater(instance.listed_count, 192107) 129 | 130 | instance1 = User.remote.fetch(USER_SCREEN_NAME) 131 | self.assertEqual(instance.name, instance1.name) 132 | self.assertEqual(User.objects.count(), 1) 133 | 134 | def test_fetch_user_statuses(self): 135 | 136 | instance = UserFactory(id=USER_ID) 137 | 138 | self.assertEqual(Status.objects.count(), 0) 139 | 140 | instances = instance.fetch_statuses(count=30) 141 | 142 | self.assertEqual(instances.count(), 30) 143 | self.assertEqual(instances.count(), Status.objects.filter(author=instance).count()) 144 | 145 | # test `all` argument 146 | instances = instance.fetch_statuses(all=True, exclude_replies=True) 147 | 148 | self.assertGreater(instances.count(), 3100) 149 | self.assertLess(instances.count(), 4000) 150 | self.assertEqual(instances.count(), Status.objects.filter(author=instance).count()) 151 | 152 | # test `after` argument 153 | after = timezone.now() - timedelta(20) 154 | instances_after = instance.fetch_statuses(all=True, after=after) 155 | self.assertLess(instances_after.count(), instances.count()) 156 | self.assertEqual(instances_after.filter(created_at__lt=after).count(), 0) 157 | 158 | # test `before` argument 159 | before = instances_after.order_by('created_at')[instances_after.count() / 2].created_at 160 | instances_before =instance.fetch_statuses(all=True, after=after, before=before) 161 | self.assertLess(instances_before.count(), instances_after.count()) 162 | self.assertEqual(instances_before.filter(created_at__gt=before).count(), 0) 163 | 164 | def test_fetch_user_followers(self): 165 | 166 | instance = UserFactory(id=USER1_ID) 167 | 168 | self.assertEqual(User.objects.count(), 1) 169 | instances = instance.fetch_followers(all=True) 170 | self.assertGreater(instances.count(), 870) 171 | self.assertLess(instances.count(), 2000) 172 | self.assertIsInstance(instances[0], User) 173 | self.assertEqual(instances.count(), User.objects.count() - 1) 174 | 175 | def test_fetch_user_followers_ids(self): 176 | 177 | instance = UserFactory(id=USER1_ID) 178 | 179 | self.assertEqual(User.objects.count(), 1) 180 | 181 | ids = instance.get_followers_ids(all=True) 182 | 183 | self.assertGreater(len(ids), 1000) 184 | self.assertLess(len(ids), 2000) 185 | self.assertIsInstance(ids[0], six.integer_types) 186 | self.assertEqual(User.objects.count(), 1) 187 | 188 | def test_fetch_status_retweets(self): 189 | 190 | instance = StatusFactory(id=STATUS_MANY_RETWEETS_ID) 191 | 192 | self.assertEqual(Status.objects.count(), 1) 193 | 194 | instances = instance.fetch_retweets() 195 | 196 | self.assertGreaterEqual(instances.count(), 6) 197 | self.assertEqual(instances.count(), Status.objects.count() - 1) 198 | 199 | def test_get_replies(self): 200 | """ 201 | Check what ids[0] < ids[1] < ids[2] ... 202 | this also check what there is no duplicates 203 | """ 204 | status = Status.remote.fetch(STATUS_MANY_REPLIES_ID) 205 | ids = get_replies(status) 206 | 207 | self.assertListEqual(ids, sorted(ids)) 208 | self.assertEqual(len(ids), len(set(ids))) 209 | 210 | def test_status_fetch_replies(self): 211 | status = Status.remote.fetch(STATUS_MANY_REPLIES_ID) 212 | 213 | self.assertEqual(Status.objects.count(), 1) 214 | self.assertEqual(status.replies_count, None) 215 | 216 | replies = status.fetch_replies() 217 | 218 | self.assertGreater(replies.count(), 200) 219 | self.assertEqual(replies.count(), status.replies_count) 220 | self.assertEqual(replies.count(), Status.objects.count() - 1) 221 | 222 | self.assertEqual(replies[0].in_reply_to_status, status) 223 | 224 | status = Status.remote.fetch(STATUS_MANY_REPLIES_ID) 225 | self.assertEqual(replies.count(), status.replies_count) 226 | --------------------------------------------------------------------------------