45 | | Count: | ${data['count']} |
46 |
47 | | Users: | ${data['usercount']} |
48 | | User percentiles: | ${sparkline.sparkify(data["userspercentiles"])} |
49 | | ${str(data["userspercentiles"])} |
50 |
51 | | Has hashtag: | ${"{:>9}".format(str(data["hashtagcount"]))} (${percentage(data["hashtagcount"], data["count"])}) |
52 | | Hashtags: | ${"{:>9}".format(str(data["hashtags"]))} |
53 | | Hashtags percentiles: | ${sparkline.sparkify(data["hashtagspercentiles"])} |
54 | | ${str(data["hashtagspercentiles"])} |
55 |
56 | | Has URL: | ${"{:>9}".format(str(data["urlcount"]))} (${percentage(data["urlcount"], data["count"])}) |
57 | | URLs: | ${"{:>9}".format(str(data["urls"]))} |
58 | | URLs percentiles: | ${sparkline.sparkify(data["urlspercentiles"])} |
59 | | ${str(data["urlspercentiles"])} |
60 |
61 | | Has Image URL: | ${"{:>9}".format(str(data["imageurlcount"]))} (${percentage(data["imageurlcount"], data["count"])}) |
62 | | Image URLs: | ${"{:>9}".format(str(data["imageurls"]))} |
63 | | Image URLs percentiles: | ${sparkline.sparkify(data["imageurlspercentiles"])} |
64 | | ${str(data["imageurlspercentiles"])} |
65 |
66 | | Originals: | ${"{:>9}".format(str(data["originalcount"]))} (${percentage(data["originalcount"], data["count"])}) |
67 | | Retweets: | ${"{:>9}".format(str(data["retweetcount"]))} (${percentage(data["retweetcount"], data["count"])}) |
68 | | Quotes: | ${"{:>9}".format(str(data["quotecount"]))} (${percentage(data["quotecount"], data["count"])}) |
69 | | Replies: | ${"{:>9}".format(str(data["replycount"]))} (${percentage(data["replycount"], data["count"])}) |
70 | | Geo: | ${"{:>9}".format(str(data["geocount"]))} (${percentage(data["geocount"], data["count"])}) |
71 | | Earliest: | ${str(data["earliest"])} |
72 | | Latest: | ${str(data["latest"])} |
73 | | Duration: | ${str(parser.parse(data["latest"]) - parser.parse(data["earliest"]))} |
74 | | Top users: | ${sparkline.sparkify([u["value"] for u in data["topusers"]])} |
75 | % for user in data["topusers"]:
76 | | ${user["value"]} | ${user["name"]} |
77 | % endfor
78 | | Top hashtags: | ${sparkline.sparkify([u["value"] for u in data["tophashtags"]])} |
79 | % for hashtag in data["tophashtags"]:
80 | | ${hashtag["value"]} | ${hashtag["name"]} |
81 | % endfor
82 | | Top URLs: | ${sparkline.sparkify([u["value"] for u in data["topurls"]])} |
83 | % for url in data["topurls"]:
84 | | ${url["value"]} | ${url["name"]} |
85 | % endfor
86 | | Top Image URLs: | ${sparkline.sparkify([u["value"] for u in data["topimageurls"]])}
87 |
(Note: Firefox may not display these images, because of content blocking. You can turn off Enhanced Tracking Protection for this domain to allow the thumbnails to load. The links to the images should still work even when the thumbnails don't.) |
88 | % for imageurl in data["topimageurls"]:
89 | | ${imageurl["value"]} | ![](${imageurl["name"].replace("http:", "https:")}) |
90 | % endfor
91 |
92 |
93 |
--------------------------------------------------------------------------------
/d3wordcloud.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | import optparse
4 | import re
5 | import dateutil.parser
6 | from profiler import TimeProfiler
7 | import pytz
8 | import d3output
9 |
10 | opt_parser = optparse.OptionParser()
11 | opt_parser.add_option("-t", "--timezone", type=str, default="",
12 | help="output timezone (e.g. 'America/New_York' or 'local'; default: UTC)")
13 | opt_parser.add_option("-w", "--maxwords", dest="maxwords", type="int",
14 | help="maximum number of words to display (default: 25)", default=25)
15 | opt_parser.add_option("-i", "--interval", dest="intervalStr", type="str",
16 | help="interval for grouping timestamps, in seconds, minutes or hours, e.g. 15M (default: 1H)",
17 | default="1H")
18 | opt_parser.add_option("-s", "--start", type=str, default=None,
19 | help="start date/time")
20 | opt_parser.add_option("-e", "--end", type=str, default=None,
21 | help="end date/time")
22 | opt_parser.add_option("-o", "--output", dest="output", type="str",
23 | help="html | csv | json (default: html)", default="html")
24 | opt_parser.add_option("-p", "--template", dest="template", type="str",
25 | help="name of template in utils/template (default: wordcloud.html)", default="wordcloud.html")
26 |
27 | opts, args = opt_parser.parse_args()
28 |
29 | tzname = opts.timezone
30 | # determine output time zone
31 | if tzname == "":
32 | tz = pytz.UTC
33 | elif tzname == "local":
34 | tz = get_localzone() # system timezone, from tzlocal
35 | else:
36 | tz = pytz.timezone(tzname)
37 |
38 | maxwords = opts.maxwords
39 | intervalStr = opts.intervalStr
40 | output = opts.output
41 |
42 | start = opts.start
43 | end = opts.end
44 | if opts.start:
45 | start = tz.localize(dateutil.parser.parse(start + "0001-01-01 00:00:00"[len(start):]))
46 | if opts.end:
47 | end = tz.localize(dateutil.parser.parse(end + "9999-12-31 23:11:59"[len(end):]))
48 |
49 | # from https://gist.github.com/uogbuji/705383
50 | GRUBER_URLINTEXT_PAT = re.compile(ur'(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?\xab\xbb\u201c\u201d\u2018\u2019]))')
51 |
52 | class WordcloudTimeProfiler(TimeProfiler):
53 | def __init__(self, opts):
54 | TimeProfiler.__init__(self, opts)
55 | self.timeslices = {}
56 | self.stop_words = set(line.strip().lower() for line in open("stopwords/stop-words_english_6_en.txt"))
57 |
58 | def process(self, tweet):
59 | created_at = dateutil.parser.parse(tweet["created_at"])
60 | if ((self.start is None) or (created_at >= self.start)) and ((self.end is None)
61 | or (created_at <= self.end)):
62 | timeslice = TimeProfiler.process(self, tweet)
63 | if not timeslice in self.timeslices:
64 | self.timeslices[timeslice] = {}
65 | word_counts = self.timeslices[timeslice]
66 | text = tweet["text"]
67 | # remove hashtags and user names
68 | text = re.sub("(^|[^\w])[@#]\w*", "\g<1>", text)
69 | # remove urls
70 | text = re.sub(GRUBER_URLINTEXT_PAT, " ", text)
71 | # trim punctuation next to space
72 | text = re.sub(ur"[^\w\s]+(\s|$)|(^|\s)[^\w\s]+", " ", text, re.UNICODE)
73 | # replace internal punctuation, except apostrophes
74 | text = re.sub(ur"[^\w\s\']", " ", text, re.UNICODE)
75 | for word in text.split():
76 | word = word.lower()
77 | if len(word) < 3: continue
78 | if len(word) > 15: continue
79 | if word in self.stop_words: continue
80 | if word.startswith("rt"): continue
81 | if not re.match("^[a-z]", word, re.IGNORECASE): continue
82 | # remove final 's
83 | word = re.sub("\'s$", "", word)
84 | if len(word) > 0:
85 | word_counts[word] = word_counts.get(word, 0) + 1
86 |
87 | def report(self):
88 | data = TimeProfiler.report(self)
89 | data["profile"]["start"] = str(self.start)
90 | data["profile"]["end"] = str(self.end)
91 | for value in data["values"]:
92 | thisslice = self.timeslices[value["name"]]
93 | # sort words by value
94 | sorted_words = thisslice.keys()
95 | sorted_words.sort(lambda a, b: cmp(thisslice[b], thisslice[a]))
96 | top_words = sorted_words[0:maxwords]
97 | words = []
98 | for word in top_words:
99 | words.append({
100 | "text": word,
101 | "count": thisslice[word]
102 | })
103 | value["words"] = words
104 | return data
105 |
106 | profiler = WordcloudTimeProfiler({
107 | "tz": tz,
108 | "output": "json",
109 | "aggregate": True,
110 | "intervalStr": intervalStr,
111 | "start": start,
112 | "end": end})
113 |
114 | profiler.gettweets(opts, args)
115 |
116 | data = profiler.report()
117 |
118 | if opts.output == "html":
119 | d3output.embed(opts.template, data)
120 | else:
121 | print(data)
122 |
--------------------------------------------------------------------------------
/d3cotags.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | import json
4 | import itertools
5 | import optparse
6 | import d3output # local module
7 | from profiler import Profiler # local module
8 | from profiler import LinkNodesProfiler # local module
9 | from collections import Counter
10 |
11 | class CotagsProfiler(LinkNodesProfiler):
12 | def __init__(self, opts):
13 | LinkNodesProfiler.__init__(self, opts)
14 | self.savetweets = []
15 | self.counts = Counter()
16 | self.keepers = set()
17 |
18 | def process(self, tweet):
19 | Profiler.process(self, tweet)
20 | # gather a list of the tags in this tweet, lowercased
21 | savetweet = []
22 | for tag in tweet["entities"]["hashtags"]:
23 | t = tag["text"].lower()
24 | savetweet.append(t)
25 | # and increment count for this tag
26 | self.counts[t] += 1
27 | # add tag list to savetweets
28 | self.savetweets.append(savetweet)
29 |
30 | def report(self):
31 | # for tags below the threshold, replace with "-OTHER"
32 | # which is not necessary if threshold is 0
33 | if self.threshold > 0:
34 | countkeys = self.counts.keys()
35 | for countkey in countkeys:
36 | if self.counts[countkey] < self.threshold:
37 | # for a tag whose count is below the threshold, transfer its
38 | # count to tag "-OTHER" and delete it
39 | if self.keepother:
40 | self.counts["-OTHER"] += self.counts[countkey]
41 | del self.counts[countkey]
42 | else:
43 | # otherwise add it to list of keepers
44 | self.keepers.add(countkey)
45 | if self.keepother:
46 | self.keepers.add("-OTHER")
47 | # keepers now has a complete set of surviving tags
48 |
49 | # now process hashtags in tweets again, replacing any tag not in keepers with -OTHER
50 | self.counts = Counter()
51 | for savetweet in self.savetweets:
52 |
53 | # cleantags gathers unique, lower-cased tags for this tweet
54 | cleantags = set()
55 |
56 | for tag in savetweet:
57 | if self.threshold == 0 or tag in self.keepers:
58 | cleantags.add(tag)
59 | else:
60 | if self.keepother:
61 | cleantags.add("-OTHER")
62 |
63 | # sort tags and remove tags that are in the exclude set
64 | cleantags = sorted(cleantags.difference(self.exclude))
65 |
66 | # generate all pairs
67 | for c in itertools.combinations(cleantags, 2):
68 | self.addlink(c[0], c[1])
69 | if self.reciprocal:
70 | self.addlink(c[1], c[0])
71 |
72 | # if this tag is the only one we're including from this tweet,
73 | # then there won't be any combinations, and so it won't have
74 | # been added to self.nodes by addlink: so add it.
75 |
76 | # add to tweet count for this tag
77 | for tag in cleantags:
78 | if tag in self.nodes:
79 | self.nodes[tag]["tweetcount"] += 1
80 | else:
81 | self.addsingle(tag)
82 |
83 | data = LinkNodesProfiler.report(self)
84 | return data;
85 |
86 |
87 | opt_parser = optparse.OptionParser()
88 | opt_parser.add_option("-o", "--output", dest="output", type="str",
89 | help="html | json (default: html)", default="html")
90 | opt_parser.add_option("-e", "--exclude", type=str, default="",
91 | help="comma-separated list of hashtags to exclude")
92 | opt_parser.add_option("-t", "--threshold", type=int, default=0,
93 | help="threshold below which to treat hashtags as 'other'")
94 | opt_parser.add_option("-r", "--reciprocal", action="store_true", default=False,
95 | help="add reciprocal links for each pair")
96 | opt_parser.add_option("-p", "--template", dest="template", type="str",
97 | help="name of template in utils/template (default: graph.html)", default="graph.html")
98 | opt_parser.add_option("-k", "--keepother", action="store_true", default=False,
99 | help="include -OTHER tag in output for tags below threshold")
100 |
101 | opts, args = opt_parser.parse_args()
102 |
103 | threshold = opts.threshold
104 | exclude = set(opts.exclude.lower().split(","))
105 | reciprocal = opts.reciprocal
106 | keepother = opts.keepother
107 | output = opts.output
108 |
109 | profiler = CotagsProfiler({
110 | "threshold": threshold,
111 | "exclude": exclude,
112 | "reciprocal": reciprocal,
113 | "keepother": keepother,
114 | "graph": "undirected",
115 | "field": "hashtag"})
116 |
117 | profiler.gettweets(opts, args)
118 |
119 | data = profiler.report()
120 |
121 | profile = data["profile"]
122 | nodes = data["nodes"]
123 |
124 | if output == "csv":
125 | print(d3output.nodeslinkcsv(nodes))
126 | elif output == "json":
127 | values = d3output.nodeslinktrees(profile, nodes)
128 | print({"profile": profile, "values": values})
129 | elif output == "html":
130 | print(d3output.embed(opts.template, d3output.nodeslinktrees(profile, nodes)))
131 |
132 |
133 |
--------------------------------------------------------------------------------
/templates/wordcloud.html:
--------------------------------------------------------------------------------
1 |
2 |