├── .gitignore ├── AUTHORS ├── MANIFEST.in ├── README.rst ├── setup.cfg ├── setup.py └── socialfeedsparser ├── __init__.py ├── admin.py ├── contrib ├── __init__.py ├── facebook │ ├── __init__.py │ ├── settings.py │ └── source.py ├── instagram │ ├── __init__.py │ ├── settings.py │ └── source.py ├── linkedin │ ├── __init__.py │ ├── settings.py │ └── source.py ├── parsers.py └── twitter │ ├── __init__.py │ ├── settings.py │ └── source.py ├── management ├── __init__.py └── commands │ ├── __init__.py │ └── collect_social_feeds.py ├── managers.py ├── migrations ├── 0001_initial.py ├── 0002_auto_20150701_0024.py ├── 0003_channel_name.py ├── 0004_channel_user_secret_user_token.py ├── 0005_auto_20160415_1020.py └── __init__.py ├── models.py ├── settings.py ├── templates └── socialfeedsparser │ └── socialfeedsparser_list.html ├── templatetags ├── __init__.py └── socialfeedsparser_tags.py ├── urls.py ├── utils.py └── views.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | *.pot 3 | *.pyc 4 | local_settings.py 5 | build 6 | dist 7 | django_spokeaboutus.egg-info/* 8 | .project 9 | .pydevproject 10 | .idea 11 | tox.ini 12 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Tomasz Roszko 2 | Guillaume Pousseo 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS 2 | include LICENSE.txt 3 | include README.rst 4 | recursive-include spokeaboutus/templates * 5 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ########################## 2 | django-social-feeds-parser 3 | ########################## 4 | 5 | django-social-feeds-parser is a django application to store and display feed coming from social medias such as: 6 | 7 | * facebook 8 | * twitter 9 | * instagram 10 | 11 | This library is a fork of Tomasz Roszko's django-spokeaboutus 12 | 13 | ******* 14 | Install 15 | ******* 16 | 17 | From Github 18 | 19 | .. code-block:: shell-session 20 | 21 | pip install -e git+https://github.com/revsquare/django-social-feeds-parser.git#egg=django_socialfeeds-parser 22 | 23 | Install python-linkedin if needed: 24 | 25 | .. code-block:: shell-session 26 | 27 | pip install -e git+https://github.com/ozgur/python-linkedin.git@master#egg=python-linkedin 28 | 29 | ***** 30 | Setup 31 | ***** 32 | 33 | settings.py 34 | =========== 35 | 36 | .. code-block:: python 37 | 38 | INSTALED_APPS = ( 39 | ... 40 | 'socialfeedsparser', 41 | ... 42 | ) 43 | 44 | Configure the social media source you want to trigger. You can of course add you own custom channel source. 45 | 46 | .. code-block:: python 47 | 48 | SOCIALFEEDSPARSER_SOURCE = ( 49 | 'socialfeedsparser.contrib.twitter', 50 | 'socialfeedsparser.contrib.facebook', 51 | 'socialfeedsparser.contrib.instagram', 52 | 'socialfeedsparser.contrib.linkedin', 53 | 'your.socialfeedsparser.source', 54 | ) 55 | 56 | For each service you add, you will need to configure their API accesses: 57 | 58 | 59 | Facebook 60 | -------- 61 | 62 | .. code-block:: python 63 | 64 | SOCIALFEEDSPARSER_FACEBOOK_CLIENT_ID = "your app client_id" 65 | SOCIALFEEDSPARSER_FACEBOOK_CLIENT_SECRET = "your app client secret" 66 | 67 | Twitter 68 | ------- 69 | 70 | .. code-block:: python 71 | 72 | SOCIALFEEDSPARSER_TWITTER_CONSUMER_KEY = "your app consumer key" 73 | SOCIALFEEDSPARSER_TWITTER_CONSUMER_SECRET = "your app consumer secret key" 74 | SOCIALFEEDSPARSER_TWITTER_ACCESS_TOKEN = "your app access token" 75 | SOCIALFEEDSPARSER_TWITTER_ACCESS_TOKEN_SECRET = "your app access token secret" 76 | 77 | Instagram 78 | --------- 79 | 80 | .. code-block:: python 81 | 82 | SOCIALFEEDSPARSER_INSTAGRAM_ACCESS_TOKEN = "your app access token" 83 | 84 | LinkedIn 85 | --------- 86 | 87 | .. code-block:: python 88 | 89 | SOCIALFEEDSPARSER_LINKEDIN_API_KEY = "your application api key" 90 | SOCIALFEEDSPARSER_LINKEDIN_API_SECRET = "your application api secret" 91 | SOCIALFEEDSPARSER_LINKEDIN_RETURN_URL = 'http://your.domain.com/social-feeds-parser/linkedin-save-token/' 92 | SOCIALFEEDSPARSER_LINKEDIN_PERMISSIONS = ( 93 | 'rw_company_admin', 94 | 'r_basicprofile', 95 | # 'r_fullprofile', 96 | 'r_emailaddress', 97 | # 'r_network', 98 | # 'r_contactinfo', 99 | 'w_share', 100 | # 'rw_groups', 101 | # 'w_messages' 102 | ) 103 | 104 | 105 | urls.py 106 | ======= 107 | 108 | .. code-block:: python 109 | 110 | urlpatterns = patterns('', 111 | ... 112 | url(r'^social-feeds-parser/', include('socialfeedsparser.urls')), 113 | ... 114 | ) 115 | 116 | database 117 | ======== 118 | 119 | .. code-block:: shell-session 120 | 121 | python manage.py syncdb 122 | python manage.py migrate 123 | 124 | 125 | ***************** 126 | Configure sources 127 | ***************** 128 | 129 | Each query you setup for a social media is called a "channel". They are configurable from the admin. You can wether parse a user or page feed. Or even use a search query (this functionnality won't work for facebook as the ability to search for posts has been remove from its API). 130 | 131 | ******************* 132 | Collecting messages 133 | ******************* 134 | 135 | Run this command (you can of course add it to a cronjob or a scheduled broker): 136 | 137 | .. code-block:: shell-session 138 | 139 | python manage.py collect_social_feeds 140 | 141 | 142 | ************ 143 | Templatetags 144 | ************ 145 | 146 | A simple template tag is provided to display your content in a widget. You can overwrite it by adding your own 'socialfeedsparser/socialfeed_widget.html' file or by setting up a file path in the SOCIALFEEDSPARSER_TAG_TEMPLATE of your settings. You can alternatively pass the template path as an argument in the template tag in case you have several or if they differ depending on the source. 147 | 148 | You can also pass the number of items to display in the template tag. 149 | 150 | The first argument to pass is the channel instance you want to display. 151 | 152 | .. code-block:: html 153 | 154 | {% load socialfeedsparser_tags %} 155 | ... 156 | {% socialfeed_display channel 5 'widgets/twitter.html' %} 157 | 158 | ***** 159 | Other 160 | ***** 161 | 162 | 163 | channel.get_posts 164 | ================= 165 | 166 | You can trigger the published posts by order and descending publication date for a channel instance by using the 'get_posts' method. By default it will return 10 posts. You can change this number by passing it as an argument. For exemple, if you want 5 posts: 167 | 168 | 169 | .. code-block:: python 170 | 171 | channel.get_posts(5) 172 | 173 | 174 | post.linkified_content 175 | ====================== 176 | 177 | You can use this method to make all urls, hashtags or arobased user names in a message clickable as links: 178 | 179 | 180 | .. code-block:: html 181 | 182 | {{ post.content }} 183 | 184 | "This #hashtag is not linkified." 185 | 186 | {{ post.linkified_content }} 187 | 188 | "This ##hashtag is linkified for twitter." 189 | 190 | 191 | **** 192 | TODO 193 | **** 194 | 195 | * use celery to process news 196 | * write tests 197 | * support python3.4 for twitter and instagram 198 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.rst 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | from setuptools import find_packages 3 | from distutils.core import setup 4 | import sys 5 | try: 6 | reload(sys).setdefaultencoding('Utf-8') 7 | except NameError: # fix for python >= 3.4.x 8 | from importlib import reload 9 | reload(sys) 10 | 11 | setup( 12 | name='django-social-feeds-parser', 13 | version='0.5', 14 | packages=find_packages(), 15 | include_package_data=True, 16 | license='BSD License', 17 | description='A simple Django app to store and display what social media talk about site', 18 | long_description=open('README.rst').read(), 19 | url='https://github.com/RevSquare/django-social-feeds-parser', 20 | author='Tomasz Roszko, Guillaume Pousseo', 21 | author_email='tomaszroszko@revsquare.com, guillaumepousseo@revsquare.com', 22 | zip_safe=False, 23 | classifiers=[ 24 | 'Environment :: Web Environment', 25 | 'Framework :: Django', 26 | 'Intended Audience :: Developers', 27 | 'License :: OSI Approved :: BSD License', 28 | 'Operating System :: OS Independent', 29 | 'Programming Language :: Python', 30 | 'Programming Language :: Python :: 2.6', 31 | 'Programming Language :: Python :: 2.7', 32 | 'Topic :: Internet :: WWW/HTTP', 33 | 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 34 | ], 35 | install_requires=[ 36 | 'facebook-sdk==1.0.0', 37 | 'python-instagram==0.8.0.', 38 | 'tweepy==3.3.0' 39 | ], 40 | dependency_links=[ 41 | 'https://github.com/ozgur/python-linkedin.git@master#egg=python-linkedin', 42 | ], 43 | ) 44 | -------------------------------------------------------------------------------- /socialfeedsparser/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tomaszroszko' 2 | -------------------------------------------------------------------------------- /socialfeedsparser/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.template.defaultfilters import truncatewords 3 | from django.utils.timezone import now 4 | from django.utils.translation import ugettext_lazy as _ 5 | 6 | from .models import Post, Channel 7 | 8 | 9 | def get_messages(modeladmin, request, queryset): 10 | """ 11 | Collects messages from selected sources. 12 | """ 13 | for source in queryset: 14 | sc = source.source_class(spoke_source=source) 15 | sc.collect_messages() 16 | source.updated = now() 17 | source.save() 18 | 19 | get_messages.short_description = _('Get Messages from selected sources') 20 | 21 | 22 | class ChannelAdmin(admin.ModelAdmin): 23 | """ 24 | Admin class for the Channel model. 25 | """ 26 | list_display = ('query', 'name', 'source', 'query_type', 'updated', 'is_active', 'show_linkedin_token_renew_link') 27 | list_filter = ('query', 'source', 'query_type', 'updated', 'is_active') 28 | exclude = ('user_secret', 'user_token',) 29 | actions = [get_messages] 30 | radio_fields = {"query_type": admin.HORIZONTAL} 31 | 32 | def show_linkedin_token_renew_link(self, obj): 33 | return '%s' % (obj.token_renew_link, _('Click to renew')) if obj.source == 'linkedin' else '' 34 | show_linkedin_token_renew_link.allow_tags = True 35 | show_linkedin_token_renew_link.short_description = _('Renew Token') 36 | 37 | 38 | class PostAdmin(admin.ModelAdmin): 39 | """ 40 | Admin class for the Post model. 41 | """ 42 | list_display = ('channel', 'author', 'content_admin', 'date', 43 | 'is_active', 'order') 44 | list_filter = ('is_active', 'channel') 45 | list_editable = ('is_active', 'order', 'author', 'date') 46 | 47 | def content_admin(self, obj): 48 | return truncatewords(obj.content, 20) 49 | content_admin.short_description = _('Post content') 50 | 51 | 52 | admin.site.register(Channel, ChannelAdmin) 53 | admin.site.register(Post, PostAdmin) 54 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RevSquare/django-social-feeds-parser/00ce07ebdadfd42fb224195cfe1594f8ce7d7bf4/socialfeedsparser/contrib/__init__.py -------------------------------------------------------------------------------- /socialfeedsparser/contrib/facebook/__init__.py: -------------------------------------------------------------------------------- 1 | from .source import FacebookSource 2 | 3 | SOCIALFEEDSPARSER_SOURCE = FacebookSource 4 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/facebook/settings.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | FACEBOOK_CLIENT_ID = settings.SOCIALFEEDSPARSER_FACEBOOK_CLIENT_ID 4 | FACEBOOK_CLIENT_SECRET = settings.SOCIALFEEDSPARSER_FACEBOOK_CLIENT_SECRET 5 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/facebook/source.py: -------------------------------------------------------------------------------- 1 | import facebook as fbsdk 2 | import json 3 | 4 | from socialfeedsparser.contrib.parsers import ChannelParser, PostParser 5 | from socialfeedsparser.settings import SOCIALFEEDSPARSER_TIMEOUT 6 | from .settings import FACEBOOK_CLIENT_ID, FACEBOOK_CLIENT_SECRET 7 | try: 8 | from urllib2 import urlopen 9 | from urllib import urlencode 10 | except (ImportError): # for python >= 3.4 11 | from urllib.request import urlopen 12 | from urllib.parse import urlencode 13 | 14 | 15 | def get_app_access_token(app_id, app_secret): 16 | """Get the access_token for the app. 17 | 18 | This token can be used for insights and creating test users. 19 | 20 | app_id = retrieved from the developer page 21 | app_secret = retrieved from the developer page 22 | 23 | Returns the application access_token. 24 | 25 | """ 26 | 27 | # Get an app access token 28 | args = {'grant_type': 'client_credentials', 29 | 'client_id': app_id, 30 | 'client_secret': app_secret} 31 | file = urlopen("https://graph.facebook.com/oauth/access_token?" + 32 | urlencode(args)) 33 | file_readed = file.read().decode('utf8') 34 | values = json.loads(file_readed) 35 | result = values.get('access_token') 36 | file.close() 37 | return result 38 | 39 | FACEBOOK_ACCESS_TOKEN = get_app_access_token( 40 | FACEBOOK_CLIENT_ID, FACEBOOK_CLIENT_SECRET) 41 | 42 | 43 | class FacebookSource(ChannelParser): 44 | """ 45 | Collect class for Facebook. 46 | """ 47 | 48 | name = 'Facebook' 49 | slug = 'facebook' 50 | 51 | def get_messages_user(self, feed_id, count=20): 52 | """ 53 | Return posts from user or page feed. 54 | 55 | :param feed_id: id of the page or user to parse. 56 | :type item: str 57 | 58 | :param count: number of items to retrieve (default 20). 59 | :type item: int 60 | """ 61 | try: 62 | ret = self.get_api().get_connections(feed_id.encode('utf-8'), 'feed')['data'] 63 | except(fbsdk.GraphAPIError): 64 | ret = self.get_api().get_connections(feed_id, 'feed')['data'] 65 | 66 | return ret 67 | 68 | def get_messages_search(self, search): 69 | """ 70 | Return posts by search param. 71 | THIS API FEATURE HAS BEEN DISABLED BY FACEBOOK 72 | 73 | :param search: search string to search for on twitter. 74 | :type item: str 75 | """ 76 | return {} 77 | 78 | def get_api(self): 79 | """ 80 | Return authenticated connections with fb. 81 | """ 82 | api = fbsdk.GraphAPI(FACEBOOK_ACCESS_TOKEN, timeout=float(SOCIALFEEDSPARSER_TIMEOUT)) 83 | return api 84 | 85 | def prepare_message(self, message): 86 | """ 87 | Convert tweets to standard message. 88 | 89 | :param message: message entry to convert. 90 | :type item: dict 91 | """ 92 | l = 'http://www.facebook.com/permalink.php?id=%s&v=wall&story_fbid=%s' \ 93 | % (message['id'], message['id'].split('_')[1]) 94 | return PostParser( 95 | uid=message['id'], 96 | author=message.get('name', ''), 97 | author_uid=message['id'], 98 | content=message.get('message', '') or message.get('description', ''), 99 | date=message['created_time'], 100 | image=message.get('picture', None), 101 | link=l 102 | ) 103 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/instagram/__init__.py: -------------------------------------------------------------------------------- 1 | from .source import InstagramSource 2 | 3 | SOCIALFEEDSPARSER_SOURCE = InstagramSource 4 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/instagram/settings.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | INSTAGRAM_ACCESS_TOKEN = settings.SOCIALFEEDSPARSER_INSTAGRAM_ACCESS_TOKEN 4 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/instagram/source.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | 3 | from instagram.client import InstagramAPI 4 | 5 | from .settings import INSTAGRAM_ACCESS_TOKEN 6 | from socialfeedsparser.contrib.parsers import ChannelParser, PostParser 7 | 8 | 9 | class InstagramSource(ChannelParser): 10 | """ 11 | Collect class for Instagram. 12 | """ 13 | 14 | name = 'Instagram' 15 | slug = 'instagram' 16 | 17 | def get_messages_user(self, user_id): 18 | """ 19 | Return posts from user feed. 20 | 21 | :param user_id: user id of the feed to parse. 22 | :type item: str 23 | 24 | :param count: number of items to retrieve (default 20). 25 | :type item: int 26 | """ 27 | api = self.get_api() 28 | user = api.user_search(q=user_id)[0].id 29 | return api.user_recent_media(user_id=user)[0] 30 | 31 | def get_messages_search(self, search): 32 | """ 33 | Return posts by search param. 34 | 35 | :param search: search string to search for on Instagram. 36 | :type item: str 37 | """ 38 | api = self.get_api() 39 | return api.tag_recent_media(tag_name=search)[0] 40 | 41 | def get_api(self): 42 | """ 43 | Return authenticated connections with Instagram. 44 | """ 45 | api = InstagramAPI(access_token=INSTAGRAM_ACCESS_TOKEN) 46 | return api 47 | 48 | def prepare_message(self, message): 49 | """ 50 | Convert posts to standard message. 51 | 52 | :param message: message entry to convert. 53 | :type item: dict 54 | """ 55 | return PostParser( 56 | uid=hashlib.sha224(message.id).hexdigest()[:50], 57 | author=message.user.username, 58 | author_uid=message.user.username, 59 | content=message.caption, 60 | date=message.created_time, 61 | image=message.images['standard_resolution'].url, 62 | link=message.link 63 | ) 64 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/linkedin/__init__.py: -------------------------------------------------------------------------------- 1 | from .source import LinkendInSource 2 | 3 | SOCIALFEEDSPARSER_SOURCE = LinkendInSource 4 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/linkedin/settings.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | LINKEDIN_API_KEY = settings.SOCIALFEEDSPARSER_LINKEDIN_API_KEY 4 | LINKEDIN_API_SECRET = settings.SOCIALFEEDSPARSER_LINKEDIN_API_SECRET 5 | LINKEDIN_RETURN_URL = settings.SOCIALFEEDSPARSER_LINKEDIN_RETURN_URL 6 | LINKEDIN_PERMISSIONS = settings.SOCIALFEEDSPARSER_LINKEDIN_PERMISSIONS 7 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/linkedin/source.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from linkedin import linkedin 3 | 4 | from socialfeedsparser.contrib.parsers import ChannelParser, PostParser 5 | 6 | 7 | class LinkendInSource(ChannelParser): 8 | """ 9 | Collect class for Facebook. 10 | """ 11 | 12 | name = 'LinkedIn' 13 | slug = 'linkedin' 14 | 15 | def get_messages_user(self, universal_names, count=20): 16 | """ 17 | Return posts from user or page feed. 18 | 19 | :param feed_id: id of the page or user to parse. 20 | :type item: str 21 | 22 | :param count: number of items to retrieve (default 20). 23 | :type item: int 24 | """ 25 | return self.get_api().get_company_updates(self.channel.query)['values'] 26 | 27 | def get_messages_search(self, search): 28 | """ 29 | Return posts by search param. 30 | THIS API FEATURE HAS BEEN DISABLED BY FACEBOOK 31 | 32 | :param search: search string to search for on twitter. 33 | :type item: str 34 | """ 35 | return {} 36 | 37 | def get_api(self): 38 | """ 39 | Return linkedin.linkedin.LinkedInApplication 40 | """ 41 | return linkedin.LinkedInApplication(token=self.channel.user_token) 42 | 43 | def prepare_message(self, message): 44 | """ 45 | Convert linkedin post to standard message. 46 | 47 | :param message: message entry to convert. 48 | :type item: dict 49 | """ 50 | try: 51 | share = message['updateContent']['companyStatusUpdate']['share'] 52 | l = 'https://www.linkedin.com/hp/updates?topic=%s' % message['updateKey'].split('-')[2] 53 | except KeyError: 54 | raise 55 | 56 | return PostParser( 57 | uid=message['updateKey'].split('-')[2], 58 | author=message['updateContent']['company']['name'], 59 | author_uid=message['updateContent']['company']['id'], 60 | content=share.get('comment', ''), 61 | date=datetime.datetime.fromtimestamp( 62 | share['timestamp']/1000.0 63 | ).strftime('%Y-%m-%d %H:%M:%S'), 64 | image=share['content'].get( 65 | 'submittedImageUrl', None) or share['content'].get('thumbnailUrl', None), 66 | link=l 67 | ) 68 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/parsers.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import os 3 | try: 4 | from urllib2 import urlopen 5 | except(ImportError): # for python >= 3.4 6 | from urllib.request import urlopen 7 | 8 | from django.core.files.base import ContentFile 9 | from django.db.utils import IntegrityError 10 | 11 | 12 | class ChannelParser(object): 13 | """ 14 | Base parser class for all sources (twitter, fb...) 15 | """ 16 | name = None 17 | slug = None 18 | 19 | def __init__(self, channel): 20 | """ 21 | :param channel: models.Channel instance to collect messages for. 22 | :type item: int 23 | """ 24 | self.channel = channel 25 | 26 | def collect_messages(self): 27 | """ 28 | Retrieves and saves message for the models.Channel instance. 29 | """ 30 | messages = self.get_messages() 31 | for message in messages: 32 | try: 33 | parsed_message = self.prepare_message(message) 34 | except KeyError: 35 | continue 36 | parsed_message.save(channel=self.channel) 37 | 38 | def get_messages(self): 39 | """ 40 | Get a list of messages to process. 41 | """ 42 | messages = [] 43 | searches = map(lambda x: x.strip(), 44 | self.channel.query.split(',')) 45 | 46 | for search in searches: 47 | if self.channel.query_type == self.channel.FEED: 48 | messages.extend(self.get_messages_user(search)) 49 | else: 50 | messages.extend(self.get_messages_search(search)) 51 | 52 | return messages 53 | 54 | def get_messages_user(self, user_id): 55 | """ 56 | Must return list of messages from social channel. Depending on search string. 57 | Overwrite to get list of messages from user account. 58 | 59 | :param user_id: user id of the feed to parse. 60 | :type item: str 61 | """ 62 | raise NotImplementedError 63 | 64 | def get_messages_search(self, search): 65 | """ 66 | Must return list of messages from social channel. Depending on search string. 67 | Overwrite to get list of messages that are match to search string. 68 | 69 | :param search: search string to search for on twitter. 70 | :type item: str 71 | """ 72 | raise NotImplementedError 73 | 74 | def prepare_message(self, message): 75 | """ 76 | Convert returned datas into standard message. 77 | 78 | :param message: message entry to convert. 79 | :type item: dict 80 | """ 81 | raise NotImplementedError 82 | 83 | 84 | class PostParser(object): 85 | """ 86 | Manages the formating of posts into database compatible objects for the models.Post class. 87 | """ 88 | def __init__(self, uid, author='', author_uid=None, content=None, image=None, date=None, link=None): 89 | """ 90 | :param uid: ;unique id of the post in the source. 91 | :type item: str 92 | 93 | :param author: Verbose name of the author who poster the message. 94 | :type item: str 95 | 96 | :param author_uid: id of the author who poster the message. 97 | :type item: str 98 | 99 | :param about_us: post content. 100 | :type item: str 101 | 102 | :param image: image url of the post. 103 | :type item: str 104 | 105 | :param date: date of publication of the post. 106 | :type item: str 107 | 108 | :param link: unique url of the post. 109 | :type item: str 110 | """ 111 | self.uid = uid 112 | self.author = author 113 | self.author_uid = author_uid 114 | self.content = content 115 | self.image = image 116 | self.date = date 117 | self.link = link 118 | 119 | def save(self, channel): 120 | """ 121 | Return a Post model class instance. 122 | 123 | :param channel: models.Channel instance to collect messages for. 124 | :type item: int 125 | """ 126 | from socialfeedsparser.models import Post 127 | import warnings 128 | 129 | try: 130 | sau = Post.objects.get(source_uid=self.uid, channel=channel) 131 | except Post.DoesNotExist: 132 | sau = Post( 133 | source_uid=self.uid, 134 | channel=channel, 135 | author=self.author[:49], 136 | author_uid=self.author_uid, 137 | content=self.content, 138 | date=self.date, 139 | link=self.link 140 | ) 141 | 142 | with warnings.catch_warnings(): # ignoring db warnings, dirty 143 | warnings.simplefilter("ignore") 144 | if self.image: 145 | base_file_name = os.path.basename(self.image) 146 | try: 147 | file_name = hashlib.sha224(base_file_name).hexdigest()[:50] 148 | except(TypeError): 149 | file_name = hashlib.sha224(base_file_name.encode('utf-8')).hexdigest()[:50] 150 | downloaded = urlopen(self.image).read() 151 | image_file = ContentFile(downloaded, name=file_name) 152 | sau.image.save(file_name, image_file) 153 | sau.save() 154 | except IntegrityError: 155 | pass 156 | return sau 157 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/twitter/__init__.py: -------------------------------------------------------------------------------- 1 | from .source import TwitterSource 2 | 3 | SOCIALFEEDSPARSER_SOURCE = TwitterSource 4 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/twitter/settings.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | TWITTER_CONSUMER_KEY = settings.SOCIALFEEDSPARSER_TWITTER_CONSUMER_KEY 4 | TWITTER_CONSUMER_SECRET = settings.SOCIALFEEDSPARSER_TWITTER_CONSUMER_SECRET 5 | TWITTER_ACCESS_TOKEN = settings.SOCIALFEEDSPARSER_TWITTER_ACCESS_TOKEN 6 | TWITTER_ACCESS_TOKENS_SECRET = settings.SOCIALFEEDSPARSER_TWITTER_ACCESS_TOKEN_SECRET 7 | -------------------------------------------------------------------------------- /socialfeedsparser/contrib/twitter/source.py: -------------------------------------------------------------------------------- 1 | import tweepy 2 | 3 | from socialfeedsparser.contrib.parsers import ChannelParser, PostParser 4 | from socialfeedsparser.settings import SOCIALFEEDSPARSER_TIMEOUT 5 | from .settings import (TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET, 6 | TWITTER_ACCESS_TOKEN, TWITTER_ACCESS_TOKENS_SECRET) 7 | 8 | 9 | class TwitterSource(ChannelParser): 10 | """ 11 | Collect class for tweeter. 12 | """ 13 | 14 | name = 'Twitter' 15 | slug = 'twitter' 16 | 17 | def get_messages_user(self, screen_name, count=20): 18 | """ 19 | Return tweets from user feed. 20 | 21 | :param screen_name: screen name of the user feed to parse. 22 | :type item: str 23 | 24 | :param count: number of items to retrieve (default 20). 25 | :type item: int 26 | """ 27 | return self.get_api().user_timeline( 28 | screen_name=screen_name, count=count, 29 | include_entities=True, include_rts=True) 30 | 31 | def get_messages_search(self, search): 32 | """ 33 | Return tweets by search param. 34 | 35 | :param search: search string to search for on twitter. 36 | :type item: str 37 | """ 38 | return self.get_api().search( 39 | q=search, 40 | count=self.spoke_source.limit) 41 | 42 | def get_api(self): 43 | """ 44 | Return authenticated connections with twitter. 45 | """ 46 | oauth = tweepy.OAuthHandler( 47 | consumer_key=TWITTER_CONSUMER_KEY, 48 | consumer_secret=TWITTER_CONSUMER_SECRET) 49 | oauth.set_access_token(TWITTER_ACCESS_TOKEN, TWITTER_ACCESS_TOKENS_SECRET) 50 | return tweepy.API(oauth, timeout=SOCIALFEEDSPARSER_TIMEOUT) 51 | 52 | def prepare_message(self, message): 53 | """ 54 | Convert tweets to standard message. 55 | 56 | :param message: message entry to convert. 57 | :type item: dict 58 | """ 59 | # TODO: add getting images from tweet 60 | # https://dev.twitter.com/docs/tweet-entities 61 | 62 | return PostParser( 63 | uid=message.id_str, 64 | author=message.user.name, 65 | author_uid=message.user.screen_name, 66 | content=message.text, 67 | date=message.created_at, 68 | image=message.user.profile_image_url, 69 | link='https://twitter.com/%s/status/%s' % ( 70 | message.user.screen_name, message.id_str) 71 | ) 72 | -------------------------------------------------------------------------------- /socialfeedsparser/management/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'martaroszko' 2 | -------------------------------------------------------------------------------- /socialfeedsparser/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'martaroszko' 2 | -------------------------------------------------------------------------------- /socialfeedsparser/management/commands/collect_social_feeds.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | from django.utils.timezone import now 3 | 4 | from socialfeedsparser.models import Channel 5 | 6 | 7 | class Command(BaseCommand): 8 | """ 9 | Command used to get messages for the instances stored in models.Channel. 10 | 11 | usage: 12 | 13 | python manage.py collect_info_about_us 14 | """ 15 | help = 'Collect messages from social services' 16 | 17 | def handle(self, *args, **options): 18 | channels = Channel.objects.to_update() 19 | for channel in channels: 20 | self.stdout.write(u'Processing source: "%s"' % channel) 21 | sc = channel.source_class(channel=channel) 22 | sc.collect_messages() 23 | channel.updated = now() 24 | channel.save() 25 | -------------------------------------------------------------------------------- /socialfeedsparser/managers.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class ChannelManager(models.Manager): 5 | """ 6 | Manager class for the SpokeSource model. 7 | """ 8 | 9 | def published(self): 10 | """ 11 | Returns published sources. 12 | """ 13 | return self.all().filter(is_active=True) 14 | 15 | def to_update(self): 16 | """ 17 | Return sources to update. 18 | """ 19 | queryset = self.published() 20 | return [channel for channel in queryset if channel.can_update()] 21 | 22 | 23 | class PostManager(models.Manager): 24 | """ 25 | Manager class for the SpokeAboutUs model. 26 | """ 27 | 28 | def published(self): 29 | """ 30 | Returns published posts. 31 | """ 32 | return self.all().filter(is_active=True, channel__is_active=True) 33 | -------------------------------------------------------------------------------- /socialfeedsparser/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='Channel', 15 | fields=[ 16 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 17 | ('source', models.CharField(default=(b'facebook', b'Facebook'), max_length=50, verbose_name='Social media', choices=[(b'facebook', b'Facebook'), (b'twitter', b'Twitter')])), 18 | ('limit', models.IntegerField(null=True, verbose_name='Limit', blank=True)), 19 | ('query', models.CharField(help_text='Enter a search query or user/page id.', max_length=255, verbose_name='Query')), 20 | ('query_type', models.CharField(default=b'feed', help_text='Note: search is not applicable for Facebook.', max_length=5, verbose_name='Search for:', choices=[(b'feed', 'feed'), (b'search', 'search')])), 21 | ('periodicity', models.IntegerField(default=60, help_text='Collecting messages periodicy. (In minutes)', verbose_name='Periodicy')), 22 | ('is_active', models.BooleanField(default=True, verbose_name='Is Active')), 23 | ('updated', models.DateTimeField(null=True, verbose_name='Last Updated', blank=True)), 24 | ], 25 | options={ 26 | 'verbose_name': 'Social feed channel', 27 | 'verbose_name_plural': 'Social feed channels', 28 | }, 29 | bases=(models.Model,), 30 | ), 31 | migrations.CreateModel( 32 | name='Post', 33 | fields=[ 34 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 35 | ('source_uid', models.CharField(verbose_name='ID in the social media source', max_length=255, editable=False)), 36 | ('link', models.CharField(max_length=255, null=True, verbose_name='Link', blank=True)), 37 | ('author', models.CharField(max_length=50, verbose_name='Author')), 38 | ('content', models.TextField(verbose_name='Post content')), 39 | ('image', models.ImageField(upload_to=b'socialfeedsparser', null=True, verbose_name='Image', blank=True)), 40 | ('date', models.DateTimeField(null=True, verbose_name='Date', blank=True)), 41 | ('order', models.IntegerField(default=0, verbose_name='Order')), 42 | ('is_active', models.BooleanField(default=True, verbose_name='Is Active')), 43 | ('like_count', models.PositiveIntegerField(null=True, verbose_name='Like count', blank=True)), 44 | ('channel', models.ForeignKey(to='socialfeedsparser.Channel')), 45 | ], 46 | options={ 47 | 'ordering': ('order',), 48 | 'verbose_name': 'Social feed post', 49 | 'verbose_name_plural': 'Social feed posts', 50 | }, 51 | bases=(models.Model,), 52 | ), 53 | ] 54 | -------------------------------------------------------------------------------- /socialfeedsparser/migrations/0002_auto_20150701_0024.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('socialfeedsparser', '0001_initial'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='post', 16 | name='author_uid', 17 | field=models.CharField(default='', max_length=50, verbose_name='Author id'), 18 | preserve_default=False, 19 | ), 20 | migrations.AlterField( 21 | model_name='post', 22 | name='author', 23 | field=models.CharField(max_length=50, verbose_name='Author name'), 24 | preserve_default=True, 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /socialfeedsparser/migrations/0003_channel_name.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('socialfeedsparser', '0002_auto_20150701_0024'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='channel', 16 | name='name', 17 | field=models.CharField(default=b'', max_length=100, verbose_name="Channel's name", blank=True), 18 | preserve_default=True, 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /socialfeedsparser/migrations/0004_channel_user_secret_user_token.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('socialfeedsparser', '0003_channel_name'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='channel', 16 | name='user_secret', 17 | field=models.TextField(null=True, blank=True, verbose_name="User Secret"), 18 | preserve_default=False, 19 | ), 20 | migrations.AddField( 21 | model_name='channel', 22 | name='user_token', 23 | field=models.TextField(null=True, blank=True, verbose_name="User Token"), 24 | preserve_default=False, 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /socialfeedsparser/migrations/0005_auto_20160415_1020.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('socialfeedsparser', '0004_channel_user_secret_user_token'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterUniqueTogether( 15 | name='post', 16 | unique_together=set([('source_uid', 'channel')]), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /socialfeedsparser/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RevSquare/django-social-feeds-parser/00ce07ebdadfd42fb224195cfe1594f8ce7d7bf4/socialfeedsparser/migrations/__init__.py -------------------------------------------------------------------------------- /socialfeedsparser/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | try: 3 | from django.utils.importlib import import_module 4 | except ImportError: 5 | from django.utils.module_loading import import_module 6 | from django.utils.translation import ugettext_lazy as _ 7 | from django.utils.timezone import now 8 | from linkedin import linkedin 9 | 10 | from .managers import PostManager, ChannelManager 11 | from .settings import SOCIALFEEDSPARSER_SOURCE 12 | from .contrib.linkedin.settings import LINKEDIN_API_KEY, LINKEDIN_API_SECRET, \ 13 | LINKEDIN_RETURN_URL, LINKEDIN_PERMISSIONS 14 | 15 | # load all sources 16 | from .utils import get_source, linkify_url, linkify_hashes, linkify_arobase 17 | 18 | SOURCE = [import_module(s).SOCIALFEEDSPARSER_SOURCE for s in SOCIALFEEDSPARSER_SOURCE] 19 | SOURCE_CHOICES = [(s.slug, s.name) for s in SOURCE] 20 | 21 | 22 | class Channel(models.Model): 23 | """ 24 | Model storing rules on how to parse a source. 25 | """ 26 | FEED = 'feed' 27 | SEARCH = 'search' 28 | QUERY_TYPE = ( 29 | (FEED, _('feed')), 30 | (SEARCH, _('search')) 31 | ) 32 | 33 | name = models.CharField(_('Channel\'s name'), max_length=100, blank=True, default='') 34 | source = models.CharField(_('Social media'), max_length=50, choices=SOURCE_CHOICES, default=SOURCE_CHOICES[0]) 35 | limit = models.IntegerField(_('Limit'), null=True, blank=True) 36 | query = models.CharField(_('Query'), max_length=255, help_text=_('Enter a search query or user/page id.')) 37 | query_type = models.CharField(_('Search for:'), choices=QUERY_TYPE, default=FEED, max_length=5, 38 | help_text=_('Note: search is not applicable for Facebook.')) 39 | 40 | periodicity = models.IntegerField(_('Periodicy'), default=60, 41 | help_text=_('Collecting messages periodicy. (In minutes)')) 42 | is_active = models.BooleanField(_('Is Active'), default=True) 43 | updated = models.DateTimeField(_('Last Updated'), null=True, blank=True) 44 | user_secret = models.TextField(_('User Secret'), null=True, blank=True) 45 | user_token = models.TextField(_('User Token'), null=True, blank=True) 46 | 47 | objects = ChannelManager() 48 | 49 | class Meta: 50 | verbose_name = _('Social feed channel') 51 | verbose_name_plural = _('Social feed channels') 52 | 53 | def __unicode__(self): 54 | return '%s - %s' % (self.get_source_display(), self.name or self.query) 55 | 56 | def __str__(self): 57 | return self.__unicode__() 58 | 59 | def can_update(self): 60 | """ 61 | Return True if source is possible to update 62 | """ 63 | current = now() 64 | return self.is_active and (self.updated is None or current > self.updated) 65 | 66 | def get_posts(self, count=10): 67 | """ 68 | Returns a the list post for a channel: 69 | 70 | :param count: number of items to display. 71 | :type item: int 72 | """ 73 | return self.post_set.published().order_by('order', '-date')[:count] 74 | 75 | @property 76 | def source_class(self): 77 | """ 78 | Returns the source class of the source instance as a property. 79 | """ 80 | return get_source(self.source) 81 | 82 | @property 83 | def posts(self): 84 | """ 85 | Returns the 10 first posts of instance as a property. 86 | """ 87 | return self.get_posts() 88 | 89 | @property 90 | def token_renew_link(self): 91 | ret = '' 92 | if self.source == 'linkedin': 93 | authentication = linkedin.LinkedInAuthentication( 94 | LINKEDIN_API_KEY, LINKEDIN_API_SECRET, LINKEDIN_RETURN_URL, 95 | LINKEDIN_PERMISSIONS) 96 | # Optionally one can send custom "state" value that will be returned from OAuth server 97 | # It can be used to track your user state or something else (it's up to you) 98 | # Be aware that this value is sent to OAuth server AS IS - make sure to encode or hash it 99 | # authorization.state = 'your_encoded_message' 100 | ret = authentication.authorization_url # open this url on your browser 101 | linkedin.LinkedInApplication(authentication) 102 | return ret 103 | 104 | 105 | class Post(models.Model): 106 | """ 107 | Model storing posts of stored sources in SpokeSource. 108 | """ 109 | source_uid = models.CharField(_('ID in the social media source'), max_length=255, editable=False) 110 | channel = models.ForeignKey(Channel) 111 | link = models.CharField(_('Link'), null=True, blank=True, max_length=255) 112 | 113 | author = models.CharField(_('Author name'), max_length=50) 114 | author_uid = models.CharField(_('Author id'), max_length=50) 115 | content = models.TextField(_('Post content')) 116 | image = models.ImageField( 117 | _('Image'), upload_to='socialfeedsparser', null=True, blank=True) 118 | date = models.DateTimeField(_('Date'), null=True, blank=True) 119 | order = models.IntegerField(_('Order'), default=0) 120 | is_active = models.BooleanField(_('Is Active'), default=True) 121 | like_count = models.PositiveIntegerField(_('Like count'), null=True, 122 | blank=True) 123 | 124 | objects = PostManager() 125 | 126 | class Meta: 127 | verbose_name = _('Social feed post') 128 | verbose_name_plural = _('Social feed posts') 129 | ordering = ('order',) 130 | unique_together = (("source_uid", "channel"), ) 131 | 132 | def __unicode__(self): 133 | return u'%s - %s' % (self.channel.__unicode__(), self.author) 134 | 135 | def __str__(self): 136 | return self.__unicode__() 137 | 138 | @property 139 | def linkified_content(self): 140 | message = linkify_url(self.content) 141 | 142 | if self.channel.source in ('twitter', 'Twitter',): 143 | message = linkify_hashes(message) 144 | message = linkify_arobase(message) 145 | 146 | return message 147 | -------------------------------------------------------------------------------- /socialfeedsparser/settings.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | PAGINATE_BY = getattr(settings, 'SOCIALFEEDSPARSER_PAGINATE_BY', 6) 4 | 5 | DEFAULT_SOURCE = ( 6 | 'socialfeedsparser.contrib.twitter', 7 | 'socialfeedsparser.contrib.facebook', 8 | 'socialfeedsparser.contrib.instagram', 9 | ) 10 | 11 | SOCIALFEEDSPARSER_TIMEOUT = getattr(settings, 'SOCIALFEEDSPARSER_TIMEOUT', 60) 12 | SOCIALFEEDSPARSER_SOURCE = getattr(settings, 'SOCIALFEEDSPARSER_SOURCE', DEFAULT_SOURCE) 13 | SOCIALFEEDSPARSER_TAG_TEMPLATE = getattr(settings, 'SOCIALFEEDSPARSER_TAG_TEMPLATE', '') 14 | -------------------------------------------------------------------------------- /socialfeedsparser/templates/socialfeedsparser/socialfeedsparser_list.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 | {% for object in object_list %} 5 | {{ object }} 6 | {% endfor %} 7 | {% endblock %} -------------------------------------------------------------------------------- /socialfeedsparser/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tomaszroszko' 2 | -------------------------------------------------------------------------------- /socialfeedsparser/templatetags/socialfeedsparser_tags.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Template tags 4 | """ 5 | from django import template 6 | from django.utils.safestring import mark_safe 7 | 8 | from socialfeedsparser.models import Channel 9 | from socialfeedsparser.settings import settings 10 | 11 | register = template.Library() 12 | 13 | 14 | @register.simple_tag(takes_context=True) 15 | def socialfeed_display(context, channel, count=10, 16 | template=settings.SOCIALFEEDSPARSER_TAG_TEMPLATE): 17 | """ 18 | Returns a simple list of post for a channel: 19 | 20 | :param channel: models.Channel instance to return messages for. 21 | :type item: obj 22 | 23 | :param count: number of items to display. 24 | :type item: int 25 | 26 | :param template: specifies a template to use. 27 | :type item: str 28 | 29 | Usage exemple: 30 | 31 | {% socialfeed_display channel 5 'widgets/twitter.html' %} 32 | """ 33 | assert isinstance(channel, Channel) 34 | 35 | if not template: 36 | template = 'socialfeedsparser/socialfeed_widget.html' 37 | 38 | tmpl = template.loader.get_template(template) 39 | context.update({ 40 | 'channel': channel, 41 | 'posts': channel.get_posts(count) 42 | }) 43 | content = tmpl.render(template.Context(context)) 44 | 45 | return mark_safe(content) 46 | -------------------------------------------------------------------------------- /socialfeedsparser/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | 3 | from .views import PostList, save_linkedin_token 4 | from .settings import PAGINATE_BY 5 | 6 | urlpatterns = [ 7 | url(r'^$', PostList.as_view(paginate_by=PAGINATE_BY), name="social_feeds_parser"), 8 | url(r'^linkedin-save-token/', save_linkedin_token, name='linkendin_save_token'), 9 | ] 10 | -------------------------------------------------------------------------------- /socialfeedsparser/utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | url_regex = re.compile(r""" 5 | [^\s] # not whitespace 6 | [a-zA-Z0-9:/\-]+ # the protocol and domain name 7 | \.(?!\.) # A literal '.' not followed by another 8 | [\w\-\./\?=&%~#]+ # country and path components 9 | [^\s] # not whitespace""", re.VERBOSE) 10 | hashtag_regex = re.compile(r""" 11 | \# # a hashmark 12 | [^\s]* # not whitespace repeated""", re.VERBOSE) 13 | arobase_regex = re.compile(r""" 14 | \@ # a hashmark 15 | [^\s]* # not whitespace repeated""", re.VERBOSE) 16 | 17 | 18 | def linkify_url(message): 19 | """ 20 | Reformats twitter message to replace urls by cliquable links. 21 | 22 | :param message: message to parse. 23 | :type item: str 24 | """ 25 | for url in url_regex.findall(message): 26 | if url.endswith('.'): 27 | url = url[:-1] 28 | if 'http://' not in url and 'https://' not in url: 29 | href = 'http://' + url 30 | else: 31 | href = url 32 | message = message.replace(url, '%s' % (href, url)) 33 | 34 | return message 35 | 36 | 37 | def linkify_hashes(message): 38 | """ 39 | Reformats twitter message to replace hashes by cliquable links. 40 | 41 | :param message: message to parse. 42 | :type item: str 43 | """ 44 | hashtags = hashtag_regex.findall(message) 45 | for hashtag in hashtags: 46 | cleaned_hash = re.sub(r'[^a-zA-Z0-9]+', '', hashtag) 47 | formated_hashtag = '#%s' % cleaned_hash 48 | message = message \ 49 | .replace(formated_hashtag, 50 | '#%s'\ 51 | % (cleaned_hash, cleaned_hash)) 52 | 53 | return message 54 | 55 | 56 | def linkify_arobase(message): 57 | """ 58 | Reformats twitter message to replace hashes by cliquable links. 59 | 60 | :param message: message to parse. 61 | :type item: str 62 | """ 63 | arobases = arobase_regex.findall(message) 64 | for arobase in arobases: 65 | cleaned_arobase = re.sub(r'[^a-zA-Z0-9]+', '', arobase) 66 | formated_arobase = '@%s' % cleaned_arobase 67 | message = message \ 68 | .replace(formated_arobase, 69 | '@%s'\ 70 | % (cleaned_arobase, cleaned_arobase)) 71 | 72 | return message 73 | 74 | 75 | def linkify(message): 76 | """ 77 | Reformats twitter message to replace urls, hashes and arrobased strings 78 | by cliquable links. 79 | 80 | :param message: message to parse. 81 | :type item: str 82 | """ 83 | message = linkify_url(message) 84 | message = linkify_hashes(message) 85 | message = linkify_arobase(message) 86 | 87 | return message 88 | 89 | 90 | def get_source(slug): 91 | """ 92 | Return source class from given slug 93 | 94 | :param slug: slug to parse. 95 | :type item: str 96 | """ 97 | from .models import SOURCE 98 | for cls in SOURCE: 99 | if cls.slug == slug: 100 | return cls 101 | -------------------------------------------------------------------------------- /socialfeedsparser/views.py: -------------------------------------------------------------------------------- 1 | from django.contrib import messages 2 | from django.shortcuts import redirect 3 | from django.utils.translation import ugettext_lazy as _ 4 | from django.views.generic import ListView 5 | from linkedin import linkedin 6 | 7 | from .contrib.linkedin.settings import LINKEDIN_API_KEY, LINKEDIN_API_SECRET, \ 8 | LINKEDIN_RETURN_URL, LINKEDIN_PERMISSIONS 9 | 10 | from .models import Channel, Post 11 | 12 | 13 | class PostList(ListView): 14 | """ 15 | List view for the Post model instances. 16 | """ 17 | model = Post 18 | 19 | 20 | def save_linkedin_token(request): 21 | authentication = linkedin.LinkedInAuthentication( 22 | LINKEDIN_API_KEY, LINKEDIN_API_SECRET, LINKEDIN_RETURN_URL, 23 | LINKEDIN_PERMISSIONS) 24 | authentication.authorization_code = request.GET['code'] 25 | token = authentication.get_access_token() 26 | Channel.objects.filter(source='linkedin').update( 27 | user_secret=authentication.authorization_code, 28 | user_token=token.access_token) 29 | messages.success(request, _('LinkedIn Channel Token updated successfully')) 30 | return redirect('admin:socialfeedsparser_channel_changelist') 31 | --------------------------------------------------------------------------------