├── README.md
└── apache_parser.py


/README.md:
--------------------------------------------------------------------------------
 1 | # apache_parser.py
 2 | 
 3 | This is a Python script and command-line tool,
 4 | with no dependencies, that allows parsing data
 5 | from Apache log files.
 6 | 
 7 | At this time it supports generating supports for:
 8 | 
 9 | * **uri** - pageviews for each uri
10 | * **time** - datetime with highest request/second
11 | * **status_code** - hits for each http status code
12 | * **referral** - uris of referring sites
13 | * **agent** - hits for each user agent
14 | * **subscriptions** - the number of feed subscribers per uri.
15 |     This is done by parsing user agents for their subscriber count.
16 | 
17 | ## Usage
18 | 
19 | Here are some example uses:
20 | 
21 |     python parser.py access.log subscriptions
22 |     python parser.py access.log uri --quantity 5
23 |     python parser.py access.log agent --cutoff 100
24 | 
25 | There is help available at the command-line as well.
26 | 
27 |     python parser.py --help
28 | 
29 | 
30 | ## User agents successfully parsed for feed subscribers
31 | 
32 | These are the feeds that have been tested against
33 | the feed subscription system:
34 | 
35 |     Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 3 subscribers; feed-id=7675226481817637975)
36 |     Netvibes (http://www.netvibes.com/; 5 subscribers; feedId: 5404723)
37 |     Bloglines/3.1 (http://www.bloglines.com; 1 subscriber)
38 |     NewsGatorOnline/2.0 (http://www.newsgator.com; 1 subscribers)
39 |     Zhuaxia.com 1 Subscribers
40 |     AideRSS/1.0 (aiderss.com); 2 subscribers
41 |     xianguo-rssbot/0.1 (http://www.xianguo.com/; 1 subscribers)
42 |     Fastladder FeedFetcher/0.01 (http://fastladder.com/; 1 subscriber)
43 |     HanRSS/1.1 (http://www.hanrss.com; 1 subscriber)
44 |     livedoor FeedFetcher/0.01 (http://reader.livedoor.com/; 1 subscriber)
45 | 
46 | 
47 | ## Credits
48 | 
49 | This script draws inspiration and code from:
50 | 
51 | * http://effbot.org/zone/wide-finder.htm
52 | * http://www.python.org/dev/peps/pep-0265/
53 | 


--------------------------------------------------------------------------------
/apache_parser.py:
--------------------------------------------------------------------------------
  1 | import re, operator
  2 | from optparse import OptionParser
  3 | from operator import itemgetter
  4 | 
  5 | def restrict(lst, cutoff, count):
  6 |     'Restrict the list by minimum value or count.'
  7 |     if cutoff:
  8 |         lst = (x for x in lst if x[1] > cutoff)
  9 |     if count:
 10 |         lst = lst[:count]
 11 |     return lst
 12 | 
 13 | def parse(filename):
 14 |     'Return tuple of dictionaries containing file data.'
 15 |     def make_entry(x):
 16 |         return { 
 17 |             'server_ip':x.group('ip'),
 18 |             'uri':x.group('uri'),
 19 |             'time':x.group('time'),
 20 |             'status_code':x.group('status_code'),
 21 |             'referral':x.group('referral'),
 22 |             'agent':x.group('agent'),
 23 |             }
 24 |     log_re = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<status_code>\d+) \d+ "(?P<referral>.*?)" "(?P<agent>.*?)"'
 25 |     search = re.compile(log_re).search
 26 |     matches = (search(line) for line in file(filename))
 27 |     return (make_entry(x) for x in matches if x)
 28 | 
 29 | def count_value(lst, key):
 30 |     d = {}
 31 |     for obj in lst:
 32 |         val = obj[key]
 33 |         if val in d:
 34 |             d[val] = d[val] + 1
 35 |         else:
 36 |             d[val] = 1
 37 |     return d.iteritems()
 38 | 
 39 | def print_results(lst):
 40 |     for item in lst:
 41 |         print "%50s %10s" % (item[0], item[1])
 42 | 
 43 | def generic_report_for_key(key, filename, cutoff, quantity):
 44 |     'Handles creating generic reports.'
 45 |     entries = parse(filename)
 46 |     lst = count_value(entries, key)
 47 |     lst = sorted(lst, key=itemgetter(1), reverse=True)
 48 |     lst =  restrict(lst, cutoff, quantity)
 49 |     print_results(lst)
 50 | 
 51 | def subscriptions(filename, cutoff, quantity):
 52 |     'Creates a custom report for determining number of subscribers per feed.'
 53 |     entries = parse(filename)
 54 |     entries = (x for x in entries if 'ubscriber' in x['agent'])
 55 | 
 56 |     feeds = {}
 57 |     for obj in entries:
 58 |         uri = obj['uri']
 59 |         agent = obj['agent']
 60 |         if uri in feeds:
 61 |             feeds[uri].append(agent)
 62 |         else:
 63 |             feeds[uri] = [agent]
 64 | 
 65 |     feed_re = '(?P<name>.*?) \(.*?; (?P<count>\d+) subscribers?(; .*?)?\)'
 66 |     feed_re2 = '(?P<name>.*?);? (?P<count>\d+) (S|s)ubscribers?'
 67 |     search = re.compile(feed_re).search
 68 |     search2 = re.compile(feed_re2).search
 69 | 
 70 |     # remove duplicate subscriptions to avoid
 71 |     # double counting
 72 |     results = []
 73 |     for key, readers in feeds.iteritems():
 74 |         sources = {}
 75 |         for reader in readers:
 76 |             m = search(reader)
 77 |             if not m:
 78 |                 m = search2(reader)
 79 |             if m:
 80 |                 name = m.group('name')
 81 |                 count = int(m.group('count'))
 82 |                 if name in sources:
 83 |                     if sources[name] < count:
 84 |                         sources[name] = count
 85 |                 else:
 86 |                     sources[name] = count
 87 |         
 88 |         # sum subscription totals
 89 |         vals = ( val for key,val in sources.iteritems())
 90 |         sum = reduce(operator.add, vals, 0)
 91 | 
 92 |         # there can be false positives with weird user agents,
 93 |         # but they won't have any subscribers listed, so we
 94 |         # filter them out here
 95 |         if sum > 0:
 96 |             results.append((key, sum))
 97 | 
 98 |     results = sorted(results, key=itemgetter(1), reverse=True)
 99 |     results = restrict(results, cutoff, quantity)
100 |     print_results(results)
101 |     
102 | def main():
103 |     p = OptionParser("usage: parser.py file uri|time|status_code|agent|referral|subscriptions")
104 |     p.add_option('-c','--cutoff',dest='cutoff',
105 |                  help="CUTOFF for minimum hits",
106 |                  metavar="CUTOFF")
107 |     p.add_option('-q','--quantity',dest='quantity',
108 |                  help="QUANTITY of results to return",
109 |                  metavar="QUANTITY")
110 |     (options, args) = p.parse_args()
111 |     if len(args) < 1:
112 |         p.error("must specify a file to parse")
113 |     if len(args) < 2:
114 |         p.error("must specify report type to generate")
115 | 
116 |     filename = args[0]
117 |     report_type = args[1]
118 |     cutoff = int(options.cutoff) if options.cutoff else None
119 |     qty = int(options.quantity) if options.quantity else None
120 |     
121 |     if report_type == 'subscriptions':
122 |         subscriptions(filename, cutoff=cutoff, quantity=qty)
123 |     elif report_type in ['uri','time','status_code','agent','referral']:
124 |         generic_report_for_key(report_type, filename, cutoff, qty)
125 |     else:
126 |         p.error("specified an invalid report type")
127 | 
128 | if __name__ == '__main__':
129 |     main()
130 | 


--------------------------------------------------------------------------------