| Tweets Stored: | 15 |~{{ status.tweet_count }} | 16 |
|---|---|
| Earliest: | 19 |{{ status.earliest }} | 20 |
| Latest: | 23 |{{ status.latest }} | 24 |
| Average Rate: | 27 |{{ status.avg_rate|floatformat }} tweets / second | 28 |
Current filter terms:
34 | {% for term in status.terms %}
35 | {{ term }}
36 | {% endfor %}
37 |
Recent Twitter streaming processes:
41 || Status | 45 |Process | 46 |API Key | 47 |Started | 48 |Last Heartbeat | 49 |Tweet Rate (t/s) | 50 |Memory | 51 |Errors | 52 ||
|---|---|---|---|---|---|---|---|---|
| 61 | 62 | {{ stream.get_status_display }} 63 | 64 | | 65 |{{ stream.hostname }}:{{ stream.process_id }} | 66 |{{ stream.keys }} | 67 |{{ stream.created_at|naturaltime }} | 68 |{{ stream.last_heartbeat|naturaltime }} | 69 |{{ stream.tweet_rate|floatformat }} | 70 |{{ stream.memory_usage }} | 71 | {% if stream.error_count > 0 %} 72 |{{ stream.error_count }} | 73 | {% else %} 74 |{{ stream.error_count }} | 75 | {% endif %} 76 |
No recent Twitter streaming processes.
82 | {% endif %} 83 | 84 | -------------------------------------------------------------------------------- /twitter_stream/fields.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django import forms 3 | from django.core import exceptions 4 | import math 5 | 6 | class PositiveBigIntegerField(models.BigIntegerField): 7 | description = "Positive Big integer" 8 | 9 | def formfield(self, **kwargs): 10 | defaults = {'min_value': 0, 11 | 'max_value': models.BigIntegerField.MAX_BIGINT * 2 - 1} 12 | defaults.update(kwargs) 13 | return super(PositiveBigIntegerField, self).formfield(**defaults) 14 | 15 | def db_type(self, connection): 16 | if 'mysql' in connection.__class__.__module__: 17 | return 'bigint UNSIGNED' 18 | return super(PositiveBigIntegerField, self).db_type(connection) 19 | 20 | 21 | class PositiveBigAutoField(models.AutoField): 22 | description = "Unsigned Big Integer" 23 | empty_strings_allowed = False 24 | MAX_BIGINT = 9223372036854775807 25 | 26 | def db_type(self, connection): 27 | if 'mysql' in connection.__class__.__module__: 28 | return 'bigint UNSIGNED AUTO_INCREMENT' 29 | 30 | return super(PositiveBigAutoField, self).db_type(connection) 31 | 32 | 33 | default_error_messages = { 34 | 'invalid': "'%(value)s' value must be an integer.", 35 | } 36 | 37 | def get_prep_value(self, value): 38 | if value is None: 39 | return None 40 | return int(value) 41 | 42 | def get_prep_lookup(self, lookup_type, value): 43 | if ((lookup_type == 'gte' or lookup_type == 'lt') 44 | and isinstance(value, float)): 45 | value = math.ceil(value) 46 | return super(PositiveBigAutoField, self).get_prep_lookup(lookup_type, value) 47 | 48 | def to_python(self, value): 49 | if value is None: 50 | return value 51 | try: 52 | return int(value) 53 | except (TypeError, ValueError): 54 | raise exceptions.ValidationError( 55 | self.error_messages['invalid'], 56 | code='invalid', 57 | params={'value': value}, 58 | ) 59 | 60 | def formfield(self, **kwargs): 61 | defaults = {'min_value': 0, 62 | 'max_value': PositiveBigAutoField.MAX_BIGINT * 2 - 1, 63 | 'form_class': forms.IntegerField } 64 | defaults.update(kwargs) 65 | return super(PositiveBigAutoField, self).formfield(**defaults) 66 | 67 | 68 | class PositiveBigAutoForeignKey(models.ForeignKey): 69 | """A special foriegn key field for positive big auto fields""" 70 | 71 | def db_type(self, connection): 72 | # The database column type of a ForeignKey is the column type 73 | # of the field to which it points. An exception is if the ForeignKey 74 | # points to an AutoField/PositiveIntegerField/PositiveSmallIntegerField, 75 | # in which case the column type is simply that of an IntegerField. 76 | # If the database needs similar types for key fields however, the only 77 | # thing we can do is making AutoField an IntegerField. 78 | rel_field = self.related_field 79 | if isinstance(rel_field, PositiveBigAutoField): 80 | return PositiveBigIntegerField().db_type(connection=connection) 81 | return rel_field.db_type(connection=connection) 82 | try: 83 | # If we are using south, we need some rules to use these fields 84 | from south.modelsinspector import add_introspection_rules 85 | add_introspection_rules([], ["^twitter_stream\.fields\.PositiveBigAutoField"]) 86 | add_introspection_rules([], ["^twitter_stream\.fields\.PositiveBigIntegerField"]) 87 | add_introspection_rules([], ["^twitter_stream\.fields\.PositiveBigAutoForeignKey"]) 88 | except ImportError: 89 | pass 90 | -------------------------------------------------------------------------------- /twitter_stream/views.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | import json 3 | from django.conf import settings 4 | from django.utils import timezone 5 | from django.template import RequestContext 6 | from django.template.loader import render_to_string 7 | from django.views import generic 8 | from django.contrib.admin.views.decorators import staff_member_required 9 | from jsonview.decorators import json_view 10 | from twitter_stream.models import FilterTerm, StreamProcess 11 | from swapper import load_model 12 | from django.db import models 13 | 14 | 15 | def _render_to_string_request(request, template, dictionary): 16 | """ 17 | Wrapper around render_to_string that includes the request context 18 | This is necessary to get all of the TEMPLATE_CONTEXT_PROCESSORS 19 | activated in the template. 20 | """ 21 | context = RequestContext(request, dictionary) 22 | return render_to_string(template, context_instance=context) 23 | 24 | 25 | def stream_status(): 26 | terms = FilterTerm.objects.filter(enabled=True) 27 | processes = StreamProcess.get_current_stream_processes() 28 | running = False 29 | for p in processes: 30 | if p.status == StreamProcess.STREAM_STATUS_RUNNING: 31 | running = True 32 | break 33 | 34 | Tweet = load_model("twitter_stream", "Tweet") 35 | tweet_count = Tweet.count_approx() 36 | earliest_time = Tweet.get_earliest_created_at() 37 | latest_time = Tweet.get_latest_created_at() 38 | 39 | avg_rate = None 40 | if earliest_time is not None and latest_time is not None: 41 | avg_rate = float(tweet_count) / (latest_time - earliest_time).total_seconds() 42 | 43 | # Get the tweets / minute over the past 10 minutes 44 | tweet_counts = [] 45 | if latest_time is not None: 46 | latest_time_minute = latest_time.replace(second=0, microsecond=0) 47 | 48 | if settings.DATABASES['default']['ENGINE'].endswith('mysql'): 49 | drop_seconds = "created_at - INTERVAL SECOND(created_at) SECOND" 50 | elif settings.DATABASES['default']['ENGINE'].endswith('postgresql_psycopg2'): 51 | drop_seconds = "date_trunc('minute', created_at)" 52 | else: 53 | drop_seconds = "created_at" 54 | 55 | tweet_counts = Tweet.objects.extra(select={ 56 | 'time': drop_seconds 57 | }) \ 58 | .filter(created_at__gt=latest_time_minute - timedelta(minutes=20)) \ 59 | .values('time') \ 60 | .order_by('time') \ 61 | .annotate(tweets=models.Count('id')) 62 | 63 | tweet_counts = list(tweet_counts) 64 | 65 | for row in tweet_counts: 66 | row['time'] = row['time'].isoformat() 67 | 68 | return { 69 | 'running': running, 70 | 'terms': [t.term for t in terms], 71 | 'processes': processes, 72 | 'tweet_count': tweet_count, 73 | 'earliest': earliest_time, 74 | 'latest': latest_time, 75 | 'avg_rate': avg_rate, 76 | 'timeline': tweet_counts 77 | } 78 | 79 | 80 | 81 | class StatusView(generic.TemplateView): 82 | template_name = 'twitter_stream/status.html' 83 | 84 | def get_context_data(self, **kwargs): 85 | status = stream_status() 86 | status['timeline'] = json.dumps(status['timeline']) 87 | return { 88 | 'status': status 89 | } 90 | 91 | status = staff_member_required(StatusView.as_view()) 92 | 93 | @staff_member_required 94 | @json_view 95 | def json_status(request, task=None): 96 | """ 97 | Returns a JSON representation of the status, with 98 | HTML conveniently included. 99 | """ 100 | 101 | status = stream_status() 102 | 103 | display = _render_to_string_request(request, 'twitter_stream/status_display.html', { 104 | 'status': status 105 | }) 106 | 107 | return { 108 | 'display': display, 109 | 'timeline': status['timeline'] 110 | } 111 | 112 | -------------------------------------------------------------------------------- /twitter_stream/management/commands/stream_from_file.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from optparse import make_option 3 | from logging.config import dictConfig 4 | 5 | import time 6 | 7 | from django.core.management.base import BaseCommand 8 | import signal 9 | 10 | from twitter_stream import models 11 | from twitter_stream import utils 12 | from twitter_stream import settings 13 | 14 | # Setup logging if not already configured 15 | logger = logging.getLogger(__name__) 16 | 17 | if not logger.handlers: 18 | dictConfig({ 19 | "version": 1, 20 | "disable_existing_loggers": False, 21 | "handlers": { 22 | "twitter_stream": { 23 | "level": "DEBUG", 24 | "class": "logging.StreamHandler", 25 | }, 26 | }, 27 | "twitter_stream": { 28 | "handlers": ["twitter_stream"], 29 | "level": "DEBUG" 30 | } 31 | }) 32 | 33 | 34 | class Command(BaseCommand): 35 | """ 36 | Streams tweets from an existing file. The file should 37 | be pretty-printed JSON dump from the streaming API. 38 | 39 | Example usage: 40 | python manage.py stream_from_file tweets.json 41 | python manage.py stream_from_file tweets.json --limit 100000 42 | python manage.py stream_from_file tweets.json --rate-limit 25 --poll-interval 25 43 | """ 44 | 45 | option_list = BaseCommand.option_list + ( 46 | make_option( 47 | '--poll-interval', 48 | action='store', 49 | dest='poll_interval', 50 | default=10, 51 | type=int, 52 | help='Seconds between tweet inserts.' 53 | ), 54 | make_option( 55 | '--rate-limit', 56 | action='store', 57 | dest='rate_limit', 58 | default=None, 59 | type=float, 60 | help='Rate to read in tweets.' 61 | ), 62 | make_option( 63 | '--limit', 64 | action='store', 65 | dest='limit', 66 | default=None, 67 | type=int, 68 | help='Limit the number of tweets read.' 69 | ) 70 | ) 71 | args = '