├── watchtower ├── management │ ├── __init__.py │ ├── commands │ │ ├── __init__.py │ │ └── collect.py │ └── conf.py ├── migrations │ ├── __init__.py │ └── 0001_initial.py ├── tests.py ├── __init__.py ├── apps.py ├── admin.py ├── router.py ├── db │ ├── redis │ │ └── __init__.py │ ├── __init__.py │ ├── orm │ │ └── __init__.py │ └── influx │ │ └── __init__.py ├── conf.py ├── serializer.py ├── models.py └── middleware.py ├── requirements.txt ├── setup.cfg ├── doc └── img │ ├── overview.png │ └── queries.png ├── MANIFEST.in ├── .gitignore ├── setup.py ├── LICENSE └── README.md /watchtower/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /watchtower/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /watchtower/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | redis 2 | geoip2 3 | django-user-agents -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /watchtower/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /doc/img/overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/synw/django-watchtower/HEAD/doc/img/overview.png -------------------------------------------------------------------------------- /doc/img/queries.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/synw/django-watchtower/HEAD/doc/img/queries.png -------------------------------------------------------------------------------- /watchtower/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.2" 2 | default_app_config = 'watchtower.apps.WatchtowerConfig' -------------------------------------------------------------------------------- /watchtower/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class WatchtowerConfig(AppConfig): 5 | name = 'watchtower' 6 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include watchtower/migrations * 2 | recursive-include watchtower/db * 3 | recursive-include watchtower/management * -------------------------------------------------------------------------------- /watchtower/management/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | class bcolors: 4 | HEADER = '\033[95m' 5 | OKBLUE = '\033[94m' 6 | OKGREEN = '\033[92m' 7 | WARNING = '\033[93m' 8 | FAIL = '\033[91m' 9 | ENDC = '\033[0m' 10 | BOLD = '\033[1m' 11 | UNDERLINE = '\033[4m' -------------------------------------------------------------------------------- /watchtower/admin.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from django.contrib import admin 4 | 5 | from .models import Hit 6 | 7 | 8 | @admin.register(Hit) 9 | class HitAdmin(admin.ModelAdmin): 10 | list_display = ( 11 | 'created', 12 | 'path', 13 | 'method', 14 | 'ip', 15 | 'username', 16 | 'device', 17 | 'city', 18 | ) 19 | list_filter = ( 20 | 'created', 21 | 'edited', 22 | 'authenticated', 23 | 'staff', 24 | 'superuser', 25 | 'is_pc', 26 | 'is_bot', 27 | 'is_tablet', 28 | 'is_mobile', 29 | 'is_touch', 30 | ) -------------------------------------------------------------------------------- /watchtower/router.py: -------------------------------------------------------------------------------- 1 | class HitsRouter(object): 2 | 3 | def db_for_read(self, model, **hints): 4 | if model._meta.app_label == 'watchtower': 5 | return 'hits' 6 | return None 7 | 8 | def db_for_write(self, model, **hints): 9 | if model._meta.app_label == 'watchtower': 10 | return 'hits' 11 | return None 12 | 13 | def allow_relation(self, obj1, obj2, **hints): 14 | if obj1._meta.app_label == 'watchtower' or \ 15 | obj2._meta.app_label == 'watchtower': 16 | return True 17 | return None 18 | 19 | def allow_migrate(self, db, app_label, model_name=None, **hints): 20 | if app_label == 'watchtower': 21 | return db == 'hits' 22 | return None 23 | -------------------------------------------------------------------------------- /watchtower/db/redis/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import json 4 | import time 5 | import redis 6 | from watchtower.conf import SITE_SLUG, DBS 7 | from watchtower.serializer import decodeHitRow, decodeEventRow 8 | 9 | 10 | def getHits(r): 11 | prefix = SITE_SLUG + "_hit*" 12 | hits = [] 13 | for key in r.scan_iter(prefix): 14 | val = r.get(key) 15 | r.delete(key) 16 | hit = decodeHitRow(val) 17 | hits.append(hit) 18 | return hits 19 | 20 | 21 | def getEvents(r): 22 | global SITE_SLUG 23 | prefix = SITE_SLUG + "_event*" 24 | events = [] 25 | for key in r.scan_iter(prefix): 26 | val = r.get(key) 27 | r.delete(key) 28 | event = decodeEventRow(val) 29 | events.append(event) 30 | return events 31 | -------------------------------------------------------------------------------- /watchtower/management/commands/collect.py: -------------------------------------------------------------------------------- 1 | import time 2 | import redis 3 | from django.core.management.base import BaseCommand 4 | from watchtower.db import dispatch 5 | from watchtower.db.redis import getHits, getEvents 6 | from watchtower.conf import FREQUENCY, VERBOSITY 7 | 8 | 9 | class Command(BaseCommand): 10 | help = 'Start Watchtower collector' 11 | 12 | def handle(self, *args, **options): 13 | verbosity = options["verbosity"] 14 | if verbosity is None: 15 | verbosity = VERBOSITY 16 | if verbosity > 0: 17 | print("Collecting data ...") 18 | r = redis.Redis(host='localhost', port=6379, db=0) 19 | while True: 20 | hits = getHits(r) 21 | events = getEvents(r) 22 | dispatch(hits, events, verbosity) 23 | time.sleep(FREQUENCY) 24 | -------------------------------------------------------------------------------- /watchtower/conf.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.exceptions import ImproperlyConfigured 3 | 4 | redis_conf = { 5 | "host": "localhost", 6 | "port": 6379, 7 | "db": 0, 8 | } 9 | 10 | REDIS = getattr(settings, 'WT_REDIS', redis_conf) 11 | 12 | SITE_SLUG = getattr(settings, 'SITE_SLUG') 13 | 14 | SEPARATOR = getattr(settings, 'WT_SEPARATOR', "#!#") 15 | 16 | FREQUENCY = getattr(settings, 'WT_FREQUENCY', 5) 17 | 18 | VERBOSITY = getattr(settings, 'WT_VERBOSITY', 0) 19 | 20 | STOP = getattr(settings, 'WT_STOP', False) 21 | 22 | EXCLUDE = getattr(settings, 'WT_EXCLUDE', ["/admin/jsi18n/", "/media/"]) 23 | 24 | COLLECTOR = getattr(settings, 'WT_COLLECTOR', True) 25 | 26 | DBS = getattr(settings, 'WT_DATABASES', None) 27 | if DBS is None: 28 | raise ImproperlyConfigured("Please configure a database for Watchtower") 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *node_modules/ 2 | settings.py 3 | *.sqlite3 4 | .project 5 | .pydevproject 6 | .settings 7 | 8 | __pycache__/ 9 | *.py[cod] 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | env/ 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *,cover 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | 60 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | version = __import__('watchtower').__version__ 4 | 5 | setup( 6 | name='django-watchtower', 7 | packages=find_packages(), 8 | include_package_data=True, 9 | version=version, 10 | description='Collect metrics and events from Django', 11 | author='synw', 12 | author_email='synwe@yahoo.com', 13 | url='https://github.com/synw/django-watchtower', 14 | download_url='https://github.com/synw/django-watchtower/releases/tag/' + version, 15 | keywords=['django', 'monitoring'], 16 | classifiers=[ 17 | 'Development Status :: 3 - Alpha', 18 | 'Framework :: Django :: 1.11', 19 | 'Intended Audience :: Developers', 20 | 'License :: OSI Approved :: MIT License', 21 | 'Programming Language :: Python :: 3.6', 22 | ], 23 | install_requires=[ 24 | "redis", 25 | "geoip2", 26 | "django-user-agents", 27 | ], 28 | zip_safe=False 29 | ) 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /watchtower/db/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import print_function 4 | import time 5 | from django.conf import settings 6 | from watchtower.db import orm 7 | from watchtower.conf import DBS 8 | 9 | 10 | def dispatch(hits, events=None, verbosity=0): 11 | global DBS 12 | for key in DBS: 13 | db = DBS[key] 14 | if "hits_db" in db: 15 | try: 16 | djdb = db["hits_db"] 17 | except: 18 | print("Database ", db, "not found") 19 | orm.write(djdb, hits, verbosity) 20 | print_summary(num_hits=len(hits)) 21 | 22 | 23 | def print_summary(num_hits=0, num_events=0, verbosity=0): 24 | if verbosity > 0: 25 | if num_hits > 0: 26 | s = "s" 27 | if num_hits == 1: 28 | s = "" 29 | t = time.strftime('%X') 30 | print(t, ":", "processed", num_hits, "hit" + s) 31 | if num_events > 0: 32 | s = "s" 33 | if num_events == 1: 34 | s = "" 35 | t = time.strftime('%X') 36 | print(t, ":", "processed", num_events, "event" + s) 37 | -------------------------------------------------------------------------------- /watchtower/db/orm/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import time 4 | import json 5 | from watchtower.models import Hit 6 | 7 | 8 | def convertBool(val): 9 | if val == "false": 10 | return False 11 | elif val == "true": 12 | return True 13 | else: 14 | return False 15 | 16 | 17 | def write(db, hits, verbosity=0): 18 | i = len(hits) 19 | hit_objs = [] 20 | for hit in hits: 21 | if verbosity > 0 and i > 0: 22 | if verbosity > 2: 23 | print(json.dumps(hit, indent=2)) 24 | hit_obj = Hit( 25 | path=hit["path"], 26 | method=hit["method"], 27 | ip=hit["ip"], 28 | user_agent=hit["user_agent"], 29 | authenticated=convertBool(hit["is_authenticated"]), 30 | staff=convertBool(hit["is_staff"]), 31 | superuser=convertBool(hit["is_superuser"]), 32 | username=hit["user"], 33 | referer=hit["referer"], 34 | view=hit["view"], 35 | module=hit["module"], 36 | status_code=int(hit["status_code"]), 37 | reason_phrase=hit["reason_phrase"], 38 | request_time=float(hit["request_time"]), 39 | doc_size=hit["doc_size"], 40 | num_queries=int(hit["num_queries"]), 41 | queries_time=float(hit["queries_time"]), 42 | os=hit["ua"]["os"], 43 | os_version=hit["ua"]["os_version"], 44 | is_pc=convertBool(hit["ua"]["is_pc"]), 45 | is_bot=convertBool(hit["ua"]["is_bot"]), 46 | is_tablet=convertBool(hit["ua"]["is_tablet"]), 47 | is_mobile=convertBool(hit["ua"]["is_mobile"]), 48 | is_touch=convertBool(hit["ua"]["is_touch"]), 49 | browser=hit["ua"]["browser"], 50 | device=hit["ua"]["device"], 51 | country=hit["geo"]["country_name"], 52 | latitude=float(hit["geo"]["latitude"]), 53 | longitude=float(hit["geo"]["longitude"]), 54 | region=hit["geo"]["region"], 55 | city=hit["geo"]["city"] 56 | ) 57 | hit_objs.append(hit_obj) 58 | if verbosity == 1: 59 | if i > 0: 60 | print(i, "hits") 61 | elif verbosity > 1: 62 | print(i, "hits") 63 | Hit.objects.using(db).bulk_create(hit_objs) 64 | return i 65 | -------------------------------------------------------------------------------- /watchtower/serializer.py: -------------------------------------------------------------------------------- 1 | import json 2 | from watchtower.conf import SITE_SLUG, SEPARATOR 3 | G = None 4 | try: 5 | from django.contrib.gis.geoip2 import GeoIP2 6 | G = GeoIP2() 7 | except ImportError: 8 | pass 9 | 10 | 11 | def pack(data): 12 | global G 13 | sep = "#!#" 14 | h = ( 15 | SITE_SLUG, 16 | data["path"], 17 | data["method"], 18 | data["ip"], 19 | data["user_agent"], 20 | str(data["is_authenticated"]).lower(), 21 | str(data["is_staff"]).lower(), 22 | str(data["is_superuser"]).lower(), 23 | data["user"], 24 | data["referer"], 25 | data["view"], 26 | data["module"], 27 | str(data["status_code"]), 28 | data["reason_phrase"], 29 | str(data["request_time"]), 30 | str(data["doc_size"]), 31 | str(data["num_queries"]), 32 | str(data["queries_time"]), 33 | json.dumps(data["ua"]), 34 | ) 35 | hit = str.join(sep, h) 36 | return hit 37 | 38 | 39 | def getGeoData(ip): 40 | geo = { 41 | "latitude": 0, 42 | "country_name": "", 43 | "longitude": 0, 44 | "postal_code": "", 45 | "dma_code": "", 46 | "city": "", 47 | "country_code": "", 48 | "region": "" 49 | } 50 | if ip.startswith("127.") is False and ip.startswith("192.") is False and G is not None: 51 | geo = G.city(ip) 52 | return geo 53 | 54 | 55 | def decodeEventRow(row): 56 | row = row.decode("utf-8") 57 | event = {} 58 | for el in row.split(SEPARATOR): 59 | t = el.split(":;") 60 | k = t[0] 61 | v = t[1] 62 | event[k] = v 63 | return event 64 | 65 | 66 | def decodeHitRow(row): 67 | vals = row.decode().split(SEPARATOR) 68 | data = {} 69 | data["site"] = vals[0] 70 | data["path"] = vals[1] 71 | data["method"] = vals[2] 72 | data["ip"] = vals[3] 73 | data["user_agent"] = vals[4] 74 | data["is_authenticated"] = vals[5] 75 | data["is_staff"] = vals[6] 76 | data["is_superuser"] = vals[7] 77 | data["user"] = vals[8] 78 | data["referer"] = vals[9] 79 | data["view"] = vals[10] 80 | data["module"] = vals[11] 81 | data["status_code"] = int(vals[12]) 82 | data["reason_phrase"] = vals[13] 83 | data["request_time"] = int(vals[14]) 84 | data["doc_size"] = int(vals[15]) 85 | data["num_queries"] = int(vals[16]) 86 | data["queries_time"] = int(vals[17]) 87 | data["ua"] = json.loads(vals[18]) 88 | # geo data 89 | data["geo"] = getGeoData(data["ip"]) 90 | return data 91 | -------------------------------------------------------------------------------- /watchtower/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from django.db import models 4 | from django.utils.translation import ugettext_lazy as _ 5 | from django_extensions.db.fields import CreationDateTimeField, ModificationDateTimeField 6 | 7 | 8 | class Hit(models.Model): 9 | created = CreationDateTimeField(editable=False) 10 | edited = ModificationDateTimeField(editable=False) 11 | path = models.CharField( 12 | db_index=True, max_length=255, verbose_name=_("Path")) 13 | method = models.CharField(max_length=6, verbose_name=_("Method")) 14 | ip = models.CharField(max_length=12, verbose_name=_("Ip")) 15 | user_agent = models.CharField(max_length=255, verbose_name=_("User agent")) 16 | authenticated = models.BooleanField( 17 | verbose_name=_("Authenticated user"), default=False) 18 | staff = models.BooleanField(verbose_name=_("Staff user"), default=False) 19 | superuser = models.BooleanField( 20 | verbose_name=_("Superuser"), default=False) 21 | username = models.CharField( 22 | db_index=True, max_length=120, verbose_name=_("Username")) 23 | referer = models.CharField( 24 | max_length=255, blank=True, verbose_name=_("Referer")) 25 | view = models.CharField(max_length=120, blank=True, verbose_name=_("View")) 26 | module = models.CharField( 27 | max_length=120, blank=True, verbose_name=_("Module")) 28 | status_code = models.PositiveSmallIntegerField( 29 | verbose_name=_("Status code")) 30 | reason_phrase = models.CharField( 31 | max_length=120, blank=True, verbose_name=_("Reason phrase")) 32 | request_time = models.PositiveIntegerField(verbose_name=_("Request time")) 33 | doc_size = models.PositiveIntegerField(verbose_name=_("Document size")) 34 | num_queries = models.PositiveIntegerField( 35 | verbose_name=_("Number of queries")) 36 | queries_time = models.PositiveIntegerField(verbose_name=_("Queries time")) 37 | is_tablet = models.BooleanField(default=False, verbose_name=_("Tablet")) 38 | is_pc = models.BooleanField(default=False, verbose_name=_("PC")) 39 | is_bot = models.BooleanField(default=False, verbose_name=_("Bot")) 40 | os = models.CharField(max_length=120, verbose_name=_("Operating system")) 41 | os_version = models.CharField( 42 | max_length=120, verbose_name=_("Operating system version")) 43 | is_tablet = models.BooleanField(default=False, verbose_name=_("Tablet")) 44 | is_mobile = models.BooleanField(default=False, verbose_name=_("Mobile")) 45 | is_touch = models.BooleanField( 46 | default=False, verbose_name=_("Has touch capabilities")) 47 | browser = models.CharField(max_length=255, verbose_name=_("Browser")) 48 | device = models.CharField(max_length=255, verbose_name=_("Device")) 49 | country = models.CharField(max_length=120, verbose_name=_("Country")) 50 | city = models.CharField(max_length=120, verbose_name=_("City")) 51 | latitude = models.CharField(max_length=24, verbose_name=_("Latitude")) 52 | longitude = models.CharField(max_length=25, verbose_name=_("Longitude")) 53 | region = models.CharField(max_length=120, verbose_name=_("Region")) 54 | city = models.CharField(max_length=120, verbose_name=_("City")) 55 | 56 | class Meta: 57 | app_label = 'watchtower' 58 | verbose_name = _(u'Hit') 59 | verbose_name_plural = _(u'Hits') 60 | ordering = ['-created'] 61 | 62 | def __str__(self): 63 | return self.path + ' - ' + str(self.created) 64 | -------------------------------------------------------------------------------- /watchtower/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.4 on 2017-08-15 13:39 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django_extensions.db.fields 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | initial = True 12 | 13 | dependencies = [ 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name='Hit', 19 | fields=[ 20 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 21 | ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True)), 22 | ('edited', django_extensions.db.fields.ModificationDateTimeField(auto_now=True)), 23 | ('path', models.CharField(db_index=True, max_length=255, verbose_name='Path')), 24 | ('method', models.CharField(max_length=6, verbose_name='Method')), 25 | ('ip', models.CharField(max_length=12, verbose_name='Ip')), 26 | ('user_agent', models.CharField(max_length=255, verbose_name='User agent')), 27 | ('authenticated', models.BooleanField(default=False, verbose_name='Authenticated user')), 28 | ('staff', models.BooleanField(default=False, verbose_name='Staff user')), 29 | ('superuser', models.BooleanField(default=False, verbose_name='Superuser')), 30 | ('username', models.CharField(db_index=True, max_length=120, verbose_name='Username')), 31 | ('referer', models.CharField(blank=True, max_length=255, verbose_name='Referer')), 32 | ('view', models.CharField(blank=True, max_length=120, verbose_name='View')), 33 | ('module', models.CharField(blank=True, max_length=120, verbose_name='Module')), 34 | ('status_code', models.PositiveSmallIntegerField(verbose_name='Status code')), 35 | ('reason_phrase', models.CharField(blank=True, max_length=120, verbose_name='Reason phrase')), 36 | ('request_time', models.PositiveIntegerField(verbose_name='Request time')), 37 | ('doc_size', models.PositiveIntegerField(verbose_name='Document size')), 38 | ('num_queries', models.PositiveIntegerField(verbose_name='Number of queries')), 39 | ('queries_time', models.PositiveIntegerField(verbose_name='Queries time')), 40 | ('is_pc', models.BooleanField(default=False, verbose_name='PC')), 41 | ('is_bot', models.BooleanField(default=False, verbose_name='Bot')), 42 | ('os', models.CharField(max_length=120, verbose_name='Operating system')), 43 | ('os_version', models.CharField(max_length=120, verbose_name='Operating system version')), 44 | ('is_tablet', models.BooleanField(default=False, verbose_name='Tablet')), 45 | ('is_mobile', models.BooleanField(default=False, verbose_name='Mobile')), 46 | ('is_touch', models.BooleanField(default=False, verbose_name='Has touch capabilities')), 47 | ('browser', models.CharField(max_length=255, verbose_name='Browser')), 48 | ('device', models.CharField(max_length=255, verbose_name='Device')), 49 | ('country', models.CharField(max_length=120, verbose_name='Country')), 50 | ('latitude', models.CharField(max_length=24, verbose_name='Latitude')), 51 | ('longitude', models.CharField(max_length=25, verbose_name='Longitude')), 52 | ('region', models.CharField(max_length=120, verbose_name='Region')), 53 | ('city', models.CharField(max_length=120, verbose_name='City')), 54 | ], 55 | options={ 56 | 'verbose_name': 'Hit', 57 | 'verbose_name_plural': 'Hits', 58 | 'ordering': ['-created'], 59 | }, 60 | ), 61 | ] 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Django Watchtower 2 | 3 | Collect hits metrics from Django. 4 | 5 | How it works: numbers taken out from Django are stored in Redis and a collector saves them in some 6 | database(s) 7 | 8 | **Metrics**: each hit is saved with fields ip, request time, query time, user_agent, geographical 9 | information and [more](#collected-data) 10 | 11 | ## Install 12 | 13 | ```bash 14 | pip install django-watchtower 15 | ``` 16 | 17 | Add to installed apps: 18 | 19 | ```python 20 | "django_user_agents", 21 | "watchtower", 22 | ``` 23 | 24 | Add the middlewares: 25 | 26 | ```python 27 | MIDDLEWARE_CLASSES = ( 28 | # ... other middlewares 29 | 'django_user_agents.middleware.UserAgentMiddleware', 30 | 'watchtower.middleware.HitsMiddleware', 31 | ) 32 | ``` 33 | 34 | Set the Django databases: 35 | 36 | ```python 37 | DATABASES = { 38 | 'default': # ..., 39 | 'hits': { 40 | 'ENGINE': 'django.db.backends.sqlite3', 41 | 'NAME': os.path.join(BASE_DIR, 'hits.sqlite3'), 42 | } 43 | } 44 | 45 | DATABASE_ROUTERS = ['watchtower.router.HitsRouter'] 46 | ``` 47 | 48 | Add to settings.py: 49 | ```python 50 | 51 | # required 52 | SITE_SLUG = "mysite" 53 | 54 | # set the databases to use: required 55 | WT_DATABASES = { 56 | # required: at least one database 57 | "default": { 58 | "type": "django", 59 | "hits_db": "hits" # name of a DATABASE in settings 60 | }, 61 | 62 | # defaults: 63 | WT_REDIS = { 64 | "addr": "localhost:6379", 65 | "db": 0 66 | } 67 | ``` 68 | 69 | Install the GeoIp tools (optional): [get the geoip database](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data) 70 | 71 | Unzip and add to settings.py: 72 | 73 | ```python 74 | GEOIP_PATH = "/my/geo/folder" 75 | ``` 76 | 77 | Make the migrations: 78 | 79 | ``` 80 | python3 manage.py migrate watchtower --database=hits 81 | ``` 82 | 83 | ### Additional settings 84 | 85 | Exclude certain paths from hits recording: 86 | 87 | ```python 88 | WT_EXCLUDE = ["/path/not/recorded/"] 89 | # default: 90 | # ["/admin/jsi18n/", "/media/"] 91 | ``` 92 | 93 | Note: this will exclude all paths that start with the provided values 94 | 95 | Change the default collector save interval: 96 | 97 | ```python 98 | WT_FREQUENCY = 30 99 | ``` 100 | 101 | # Run the collector 102 | 103 | ```python 104 | python3 manage.py collect 105 | ``` 106 | 107 | Note: it is possible to save the data directly into the database not using Redis and the collector with the setting: 108 | 109 | ```python 110 | WT_COLLECTOR = False 111 | ``` 112 | 113 | Do not use this setting in production: it will not work when `DEBUG` is `False` 114 | 115 | # Collected data 116 | 117 | ```javascript 118 | { 119 | "site": "mysite", 120 | "user": "admin", 121 | "request_time": 35, 122 | "status_code": 200, 123 | "doc_size": 3912, 124 | "ip": "127.0.0.1", 125 | "user_agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0", 126 | "method": "GET", 127 | "view": "AddPostView", 128 | "module": "qcf.views", 129 | "is_superuser": true, 130 | "is_authenticated": true, 131 | "reason_phrase": "OK", 132 | "ua": { 133 | "os_version": "", 134 | "is_pc": true, 135 | "browser_version": "55.0", 136 | "is_mobile": false, 137 | "os": "Ubuntu", 138 | "is_tablet": false, 139 | "is_bot": false, 140 | "device": "Other", 141 | "is_touch": false, 142 | "browser": "Firefox" 143 | }, 144 | "geo": { 145 | "latitude": 0, 146 | "postal_code": "", 147 | "country_code": "", 148 | "region": "", 149 | "dma_code": "", 150 | "country_name": "", 151 | "longitude": 0, 152 | "city": "" 153 | }, 154 | "is_staff": true, 155 | "referer": "", 156 | "path": "/", 157 | "queries_time": 2, 158 | "num_queries": 1 159 | } 160 | ``` 161 | -------------------------------------------------------------------------------- /watchtower/db/influx/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import json 4 | from threading import Thread 5 | from influxdb import InfluxDBClient 6 | from watchtower.conf import INFLUX, SITE_SLUG 7 | 8 | 9 | if INFLUX is not None: 10 | CLI = InfluxDBClient( 11 | INFLUX["host"], 12 | INFLUX["port"], 13 | INFLUX["user"], 14 | INFLUX["password"], 15 | INFLUX["hits_db"], 16 | timeout=5, 17 | ssl=False 18 | ) 19 | ECLI = InfluxDBClient( 20 | INFLUX["host"], 21 | INFLUX["port"], 22 | INFLUX["user"], 23 | INFLUX["password"], 24 | INFLUX["events_db"], 25 | timeout=5, 26 | ssl=False 27 | ) 28 | 29 | 30 | def write_events(points): 31 | global ECLI 32 | try: 33 | ECLI.write_points(points) 34 | except Exception as err: 35 | raise err 36 | 37 | 38 | def write_hits(points): 39 | global CLI 40 | try: 41 | CLI.write_points(points) 42 | except Exception as err: 43 | raise err 44 | 45 | 46 | def process_events(events): 47 | points = [] 48 | for event in events: 49 | tags = { 50 | "service": "django", 51 | "domain": SITE_SLUG, 52 | "name": event["name"], 53 | } 54 | if "event_class" in event: 55 | tags["class"] = event["event_class"] 56 | if "content_type" in event: 57 | tags["content_type"] = event["content_type"] 58 | if "obj_pk" in event: 59 | tags["obj_pk"] = event["obj_pk"] 60 | if "user" in event: 61 | tags["user"] = event["user"] 62 | if "url" in event: 63 | tags["url"] = event["url"] 64 | if "admin_url" in event: 65 | tags["admin_url"] = event["admin_url"] 66 | if "notes" in event: 67 | tags["notes"] = event["notes"] 68 | if "bucket" in event: 69 | tags["bucket"] = event["bucket"] 70 | if "data" in event: 71 | tags["data"] = event["data"] 72 | if "scope" in event: 73 | tags["scope"] = event["scope"] 74 | data = { 75 | "measurement": "event", 76 | "tags": tags, 77 | "fields": { 78 | "num": 1, 79 | } 80 | } 81 | points.append(data) 82 | write_events(points) 83 | 84 | 85 | def process_hits(hits): 86 | points = [] 87 | for hit in hits: 88 | data = { 89 | "measurement": "hits", 90 | "tags": { 91 | "service": "django", 92 | "domain": SITE_SLUG, 93 | "user": hit["user"], 94 | "path": hit["path"], 95 | "referer": hit["referer"], 96 | "user_agent": hit["user_agent"], 97 | "method": hit["method"], 98 | "authenticated": hit["is_authenticated"], 99 | "staff": hit["is_staff"], 100 | "superuser": hit["is_superuser"], 101 | "status_code": hit["status_code"], 102 | "view": hit["view"], 103 | "module": hit["module"], 104 | "ip": hit["ip"], 105 | "os": hit["ua"]["os"], 106 | "os_version": hit["ua"]["os_version"], 107 | "is_pc": hit["ua"]["is_pc"], 108 | "is_bot": hit["ua"]["is_bot"], 109 | "is_tablet": hit["ua"]["is_tablet"], 110 | "is_mobile": hit["ua"]["is_mobile"], 111 | "is_touch": hit["ua"]["is_touch"], 112 | "browser": hit["ua"]["browser"], 113 | "device": hit["ua"]["device"], 114 | "country": hit["geo"]["country_name"], 115 | "latitude": float(hit["geo"]["latitude"]), 116 | "longitude": float(hit["geo"]["longitude"]), 117 | "region": hit["geo"]["region"], 118 | "city": hit["geo"]["city"] 119 | }, 120 | "fields": { 121 | "num": 1, 122 | "request_time": hit["request_time"], 123 | "doc_size": hit["doc_size"], 124 | "num_queries": hit["num_queries"], 125 | "queries_time": hit["queries_time"], 126 | } 127 | } 128 | points.append(data) 129 | write_hits(points) 130 | -------------------------------------------------------------------------------- /watchtower/middleware.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import inspect 4 | import time 5 | import redis 6 | from threading import Thread 7 | from django.db import connection 8 | from watchtower.db import dispatch 9 | from watchtower import serializer, conf as CONF 10 | try: 11 | from django.utils.deprecation import MiddlewareMixin 12 | except ImportError: 13 | MiddlewareMixin = object 14 | 15 | R = redis.StrictRedis( 16 | host=CONF.REDIS["host"], port=CONF.REDIS["port"], db=CONF.REDIS["db"]) 17 | 18 | HITNUM = int(time.time()) 19 | 20 | 21 | class HitsMiddleware(MiddlewareMixin): 22 | 23 | def process_view(self, request, view_func, *args, **kwargs): 24 | global CONF 25 | """ 26 | Credits: https://github.com/bitlabstudio/django-influxdb-metrics/blob/master/influxdb_metrics/middleware.py 27 | """ 28 | if CONF.STOP: 29 | return 30 | view = view_func 31 | if not inspect.isfunction(view_func): 32 | view = view.__class__ 33 | try: 34 | request._view_module = view.__module__ 35 | request._view_name = view.__name__ 36 | request._start_time = time.time() 37 | except AttributeError: 38 | pass 39 | 40 | def process_response(self, request, response): 41 | global CONF 42 | global HITNUM 43 | if CONF.STOP: 44 | return response 45 | data = {} 46 | data["path"] = request.path_info 47 | for expath in CONF.EXCLUDE: 48 | if data["path"].startswith(expath): 49 | return response 50 | doc_size = 0 51 | try: 52 | doc_size = len(response.content) 53 | except Exception: 54 | pass 55 | total_time = 0 56 | for query in connection.queries: 57 | query_time = query.get('time') 58 | if query_time is None: 59 | query_time = 0 60 | total_time += float(query_time) 61 | total_time = int(total_time * 1000) 62 | num_queries = len(connection.queries) 63 | data["method"] = request.method 64 | data['ip'] = request.META['REMOTE_ADDR'] 65 | data['user_agent'] = "Unknown" 66 | if "HTTP_USER_AGENT" in request.META: 67 | data["user_agent"] = request.META['HTTP_USER_AGENT'] 68 | data['referer'] = '' 69 | if 'HTTP_REFERER' in data: 70 | data['referer'] = request.META['HTTP_REFERER'] 71 | data['user'] = 'Anonymous' 72 | is_authenticated = "false" 73 | if request.user.is_authenticated: 74 | is_authenticated = "true" 75 | data['user'] = request.user.username 76 | is_staff = "false" 77 | is_superuser = "false" 78 | if request.user.is_authenticated: 79 | is_authenticated = "true" 80 | if request.user.is_staff: 81 | is_staff = "true" 82 | if request.user.is_superuser: 83 | is_superuser = "true" 84 | request_time = 0 85 | if hasattr(request, '_start_time'): 86 | request_time = int((time.time() - request._start_time) * 1000) 87 | data["is_superuser"] = is_superuser 88 | data["is_staff"] = is_staff 89 | data["is_authenticated"] = is_authenticated 90 | data["ajax"] = request.is_ajax() 91 | data["request_time"] = request_time 92 | try: 93 | data["view"] = request._view_name 94 | except Exception: 95 | data["view"] = "" 96 | try: 97 | data["module"] = request._view_module 98 | except Exception: 99 | data["module"] = "" 100 | data["status_code"] = response.status_code 101 | data["reason_phrase"] = response.reason_phrase 102 | data["doc_size"] = doc_size 103 | data["queries_time"] = total_time 104 | data["num_queries"] = num_queries 105 | data["content_length"] = doc_size 106 | ua = { 107 | "is_mobile": request.user_agent.is_mobile, 108 | "is_pc": request.user_agent.is_pc, 109 | "is_tablet": request.user_agent.is_tablet, 110 | "is_bot": request.user_agent.is_bot, 111 | "is_touch": request.user_agent.is_touch_capable, 112 | "browser": request.user_agent.browser.family, 113 | "browser_version": request.user_agent.browser.version_string, 114 | "os": request.user_agent.os.family, 115 | "os_version": request.user_agent.os.version_string, 116 | "device": request.user_agent.device.family, 117 | } 118 | data["ua"] = ua 119 | name = CONF.SITE_SLUG + "_hit" + str(HITNUM) 120 | data["geo"] = serializer.getGeoData(data['ip']) 121 | if CONF.COLLECTOR is True: 122 | hit = serializer.pack(data) 123 | R.set(name, hit) 124 | else: 125 | thread = Thread(target=dispatch, args=([data],)) 126 | thread.start() 127 | HITNUM += 1 128 | return response 129 | --------------------------------------------------------------------------------