├── tests ├── __init__.py ├── test.conf ├── README ├── base.py ├── sim_clients.py ├── test.py ├── fill_database.py ├── test_concurrency.py ├── test_get_new_hosts.py ├── test_purge_methods.py ├── test_stats.py └── test_models.py ├── packaging └── debian │ ├── compat │ ├── source │ └── format │ ├── patches │ ├── series │ └── 01-edit-setup-script.patch │ ├── denyhosts-server.manpages │ ├── denyhosts-server.links │ ├── denyhosts-server.dirs │ ├── docs │ ├── changelog │ ├── watch │ ├── denyhosts-server.install │ ├── denyhosts-server.logrotate │ ├── rules │ ├── denyhosts-server.service │ ├── etc │ ├── nginx │ │ └── sites-available │ │ │ ├── denyhosts-server-xmlrpc.conf │ │ │ └── denyhosts-server-stats.conf │ └── denyhosts-server.conf │ ├── denyhosts-server.postrm │ ├── control │ ├── denyhosts-server.upstart │ ├── denyhosts-server.postinst │ ├── copyright │ └── denyhosts-server.init ├── static └── graph │ └── README ├── setup.cfg ├── coverage.sh ├── .gitignore ├── denyhosts_server ├── __init__.py ├── models.py ├── utils.py ├── debug_views.py ├── config.py ├── views.py ├── database.py ├── controllers.py ├── main.py └── stats.py ├── denyhosts-server.service.example ├── scripts └── denyhosts-server ├── denyhosts-server.1 ├── setup.py ├── denyhosts-server.init.example ├── changelog.txt ├── denyhosts-server.conf.example ├── template └── stats.html ├── README.md └── LICENSE.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packaging/debian/compat: -------------------------------------------------------------------------------- 1 | 7 2 | -------------------------------------------------------------------------------- /packaging/debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /packaging/debian/patches/series: -------------------------------------------------------------------------------- 1 | 01-edit-setup-script.patch 2 | -------------------------------------------------------------------------------- /packaging/debian/denyhosts-server.manpages: -------------------------------------------------------------------------------- 1 | denyhosts-server.1 2 | -------------------------------------------------------------------------------- /static/graph/README: -------------------------------------------------------------------------------- 1 | This is a cache directory that will contain dynamically generate graphs 2 | -------------------------------------------------------------------------------- /packaging/debian/denyhosts-server.links: -------------------------------------------------------------------------------- 1 | /var/lib/denyhosts-server/static /var/www/sites/denyhosts-server/static 2 | -------------------------------------------------------------------------------- /packaging/debian/denyhosts-server.dirs: -------------------------------------------------------------------------------- 1 | /var/lib/denyhosts-server/static/graph 2 | /var/www/sites/denyhosts-server 3 | /var/log/denyhosts-server 4 | -------------------------------------------------------------------------------- /packaging/debian/docs: -------------------------------------------------------------------------------- 1 | LICENSE.md 2 | README.md 3 | changelog.txt 4 | denyhosts-server.conf.example 5 | denyhosts-server.init.example 6 | denyhosts-server.service.example 7 | -------------------------------------------------------------------------------- /packaging/debian/changelog: -------------------------------------------------------------------------------- 1 | denyhosts-server (2.0.0-1) trusty; urgency=medium 2 | 3 | * Initial Release. 4 | 5 | -- Sergey Dryabzhinsky Sun, 10 Jan 2016 01:17:46 +0300 6 | -------------------------------------------------------------------------------- /packaging/debian/watch: -------------------------------------------------------------------------------- 1 | version=3 2 | opts=filenamemangle=s/.+\/v?(\d\S*)\.tar\.gz/denyhosts_sync-$1\.tar\.gz/ \ 3 | https://github.com/janpascal/denyhosts_sync/tags .*/v?(\d\S*)\.tar\.gz debian uupdate 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [minify_js] 2 | sources = static/js/bootstrap.js static/js/jquery.js 3 | output = static/js/%s.min.js 4 | 5 | [minify_css] 6 | sources = static/css/bootstrap.css 7 | output = static/css/%s.min.css 8 | -------------------------------------------------------------------------------- /packaging/debian/denyhosts-server.install: -------------------------------------------------------------------------------- 1 | static/css/*.min.* /var/lib/denyhosts-server/static/css 2 | static/js/*.min.* /var/lib/denyhosts-server/static/js 3 | template /var/lib/denyhosts-server/ 4 | debian/etc/* etc/ 5 | -------------------------------------------------------------------------------- /coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | python-coverage run /usr/bin/trial \ 3 | tests/test_get_new_hosts.py \ 4 | tests/test_models.py \ 5 | tests/test_purge_methods.py \ 6 | tests/test_stats.py 7 | python-coverage html 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.sqlite 3 | .*.swp 4 | log.txt 5 | *.log 6 | _trial_temp/ 7 | build/ 8 | dist/ 9 | *.egg-info/ 10 | MANIFEST 11 | .coverage 12 | htmlcov 13 | static/graph/*.svg 14 | static/graph/*.png 15 | static/css/*.min.css 16 | static/js/*.min.js 17 | -------------------------------------------------------------------------------- /tests/test.conf: -------------------------------------------------------------------------------- 1 | # Configuration file for unit tests. 2 | 3 | [database] 4 | type: sqlite3 5 | database: unittest.sqlite 6 | 7 | [maintenance] 8 | 9 | [sync] 10 | 11 | [logging] 12 | logfile: unittest.log 13 | loglevel: DEBUG 14 | 15 | [stats] 16 | static_dir: static 17 | graph_dir: graph 18 | template_dir: template 19 | -------------------------------------------------------------------------------- /tests/README: -------------------------------------------------------------------------------- 1 | Run these tests from the project root directory using 2 | 3 | $ trial tests 4 | 5 | Trial is the Twisted test runner. On Debian systems, it is part of the 6 | python-twisted-core package. 7 | 8 | This directory also contains some scripts that are not unit test and cannot be 9 | run using trial: test.py, fill_database.py and sim_clients.py. The latter two 10 | scripts are used for performance testing. 11 | -------------------------------------------------------------------------------- /packaging/debian/denyhosts-server.logrotate: -------------------------------------------------------------------------------- 1 | /var/log/denyhosts-server/*.log { 2 | create 0640 denyhosts adm 3 | missingok 4 | daily 5 | rotate 32 6 | compress 7 | delaycompress 8 | notifempty 9 | postrotate 10 | if [ -x /usr/sbin/invoke-rc.d ]; then \ 11 | invoke-rc.d denyhosts-server reload > /dev/null; \ 12 | else \ 13 | /etc/init.d/denyhosts-server reload > /dev/null; \ 14 | fi 15 | endscript 16 | } 17 | -------------------------------------------------------------------------------- /denyhosts_server/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | denyhosts_sync_server is an open source implementation of the U{DenyHosts} 3 | synchronisation server. It is based on the 4 | U{Twisted } framework and uses 5 | U{Twistar } as an ORM layer. 6 | 7 | @author: Jan-Pascal van Best U{janpascal@vanbest.org} 8 | """ 9 | version_info = (2, 0, 0) 10 | version = '.'.join(map(str, version_info)) 11 | -------------------------------------------------------------------------------- /denyhosts-server.service.example: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=DenyHosts synchronisation service 3 | After=syslog.target 4 | 5 | [Service] 6 | Type=simple 7 | User=denyhosts-server 8 | Group=denyhosts-server 9 | EnvironmentFile=-/etc/default/denyhosts-server 10 | ExecStart=/usr/local/bin/denyhosts-server -c /etc/denyhosts-server.conf 11 | ExecReload=/bin/kill -HUP $MAINPID 12 | Restart=on-failure 13 | StandardOutput=syslog 14 | StandardError=syslog 15 | 16 | [Install] 17 | WantedBy=multi-user.target 18 | -------------------------------------------------------------------------------- /packaging/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | # 4 | 5 | DESTDIR := $(CURDIR)/debian/denyhosts-server 6 | 7 | %: 8 | dh --with python2 $@ 9 | 10 | override_dh_auto_install: 11 | python setup.py minify_css minify_js install --no-compile -O0 --force \ 12 | --root=$(DESTDIR) \ 13 | --install-scripts=usr/sbin 14 | 15 | override_dh_clean: 16 | rm -fv static/*/*.min.* 17 | rm -fr *.egg-info 18 | dh_clean 19 | 20 | override_dh_installchangelogs: 21 | dh_installchangelogs changelog.txt 22 | -------------------------------------------------------------------------------- /packaging/debian/denyhosts-server.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=DenyHosts synchronisation service 3 | After=syslog.target 4 | 5 | [Service] 6 | Type=simple 7 | User=denyhosts-server 8 | Group=denyhosts 9 | EnvironmentFile=-/etc/default/denyhosts-server 10 | WorkingDirectory=/var/lib/denyhosts-server 11 | ExecStart=/usr/sbin/denyhosts-server -c /etc/denyhosts-server.conf 12 | ExecReload=/bin/kill -HUP $MAINPID 13 | Restart=on-failure 14 | StandardOutput=syslog 15 | StandardError=syslog 16 | 17 | [Install] 18 | WantedBy=multi-user.target 19 | -------------------------------------------------------------------------------- /packaging/debian/etc/nginx/sites-available/denyhosts-server-xmlrpc.conf: -------------------------------------------------------------------------------- 1 | ### 2 | # Do copy this file into .conf 3 | # It can be overwritten by pkg 4 | # 5 | 6 | server { 7 | listen 80; 8 | 9 | root /var/www/sites/denyhosts-server; 10 | 11 | server_name xmlrpc.denyhosts-server; 12 | 13 | location / { 14 | include proxy_params; 15 | proxy_pass http://127.0.0.1:9911; 16 | } 17 | 18 | error_page 500 502 503 504 /50x.html; 19 | location = /50x.html { 20 | root /usr/share/nginx/html; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /packaging/debian/denyhosts-server.postrm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # postrm script for denyhosts 3 | 4 | set -e 5 | 6 | case "$1" in 7 | purge) 8 | rm -fr /var/log/denyhosts-server 9 | rm -fr /var/lib/denyhosts-server 10 | deluser denyhosts-server || true 11 | deluser --group --only-if-empty denyhosts || true 12 | service nginx reload || true 13 | rm -rf /var/www/sites/denyhosts-server 14 | ;; 15 | 16 | remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) 17 | ;; 18 | 19 | *) 20 | echo "postrm called with unknown argument \`$1'" >&2 21 | exit 1 22 | 23 | esac 24 | 25 | #DEBHELPER# 26 | 27 | exit 0 28 | -------------------------------------------------------------------------------- /packaging/debian/etc/nginx/sites-available/denyhosts-server-stats.conf: -------------------------------------------------------------------------------- 1 | ### 2 | # Do copy this file into .conf 3 | # It can be overwritten by pkg 4 | # 5 | 6 | server { 7 | listen 80; 8 | 9 | root /var/www/sites/denyhosts-server; 10 | index stats.html; 11 | 12 | server_name denyhosts-server; 13 | 14 | set $backend http://127.0.0.1:8811; 15 | 16 | location = / { 17 | include proxy_params; 18 | proxy_pass $backend; 19 | } 20 | 21 | location / { 22 | try_files $uri $uri/ =404; 23 | } 24 | 25 | location /static/graph { 26 | root /var/lib/denyhosts-server; 27 | try_files $uri $uri/ =404; 28 | } 29 | 30 | error_page 500 502 503 504 /50x.html; 31 | location = /50x.html { 32 | root /usr/share/nginx/html; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /tests/base.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import inspect 3 | import os 4 | import os.path 5 | 6 | from denyhosts_server import config 7 | from denyhosts_server import models 8 | from denyhosts_server import main 9 | from denyhosts_server import database 10 | 11 | from twisted.trial import unittest 12 | from twisted.enterprise import adbapi 13 | from twisted.internet.defer import inlineCallbacks, returnValue 14 | 15 | from twistar.registry import Registry 16 | 17 | class TestBase(unittest.TestCase): 18 | @inlineCallbacks 19 | def setUp(self): 20 | configfile = os.path.join( 21 | os.path.dirname(inspect.getsourcefile(TestBase)), 22 | "test.conf" 23 | ) 24 | 25 | config.read_config(configfile) 26 | 27 | Registry.DBPOOL = adbapi.ConnectionPool(config.dbtype, **config.dbparams) 28 | Registry.register(models.Cracker, models.Report, models.Legacy) 29 | 30 | yield database.clean_database() 31 | main.configure_logging() 32 | -------------------------------------------------------------------------------- /scripts/denyhosts-server: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # denyhosts sync server 4 | # Copyright (C) 2015 Jan-Pascal van Best 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Affero General Public License as published 8 | # by the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Affero General Public License for more details. 15 | 16 | # You should have received a copy of the GNU Affero General Public License 17 | # along with this program. If not, see . 18 | 19 | 20 | import denyhosts_server.main 21 | 22 | if __name__ == '__main__': 23 | denyhosts_server.main.run_main() 24 | 25 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 26 | -------------------------------------------------------------------------------- /packaging/debian/control: -------------------------------------------------------------------------------- 1 | Source: denyhosts-server 2 | Section: net 3 | Priority: optional 4 | Maintainer: Jan-Pascal van Best 5 | Uploaders: Sergey Dryabzhinsky 6 | Build-Depends: debhelper (>= 7.0.50~), python (>= 2.6.6-3~), dh-python | debhelper (>= 8), python-minify | python2.7-pip-minify | python2.6-pip-minify 7 | Standards-Version: 3.9.3 8 | Homepage: https://github.com/janpascal/denyhosts_sync.git 9 | 10 | Package: denyhosts-server 11 | Architecture: all 12 | Depends: lsb-base (>= 3.1-13), ${python:Depends}, ${misc:Depends}, 13 | python-twisted, python-ipaddr, python-jinja2, python-numpy, python-geoip, 14 | python-matplotlib, python-twistar | python2.7-pip-twistar | python2.6-pip-twistar 15 | Recommends: nginx | nginx-light | nginx-hosting | nginx-full 16 | Description: denyhosts-server is a server for denyhosts clients 17 | denyhosts-server is a server that allows denyhosts clients to share 18 | blocked IP addresses. It is intended to be a drop-in replacement for 19 | the service at xmlrpc.denyhosts.net that up to now has been provided 20 | by the original author of denyhosts. 21 | -------------------------------------------------------------------------------- /packaging/debian/denyhosts-server.upstart: -------------------------------------------------------------------------------- 1 | author "Sergey Dryabzhinsky " 2 | description "Handles starting of DenyHosts Sync on server reboot, or in the event of an error" 3 | 4 | env NAME=denyhosts-server 5 | env NAME2="DenyHosts sync server" 6 | env DAEMON="/usr/sbin/denyhosts-server" 7 | env CONFIG="/etc/denyhosts-server.conf" 8 | env PIDFILE="/var/run/denyhosts-server.pid" 9 | env DAEMONUSER=denyhosts-server 10 | env DAEMONGROUP=denyhosts 11 | env WORKDIR=/var/lib/denyhosts-server 12 | 13 | start on runlevel [2345] 14 | stop on runlevel [016] 15 | 16 | expect daemon 17 | 18 | respawn 19 | respawn limit 10 5 20 | 21 | exec start-stop-daemon --start --chuid $DAEMONUSER:$DAEMONGROUP \ 22 | --chdir $WORKDIR \ 23 | --make-pidfile --pidfile $PIDFILE --background \ 24 | --startas $DAEMON -- -c $CONFIG 25 | 26 | pre-start script 27 | if [ -f $PIDFILE ]; then 28 | pid=$(cat $PIDFILE) 29 | if kill -0 "$pid" > /dev/null; then 30 | start-stop-daemon --stop --quiet --retry 5 --pidfile $PIDFILE 31 | echo "Restarting $NAME2" | logger -t "$NAME" 32 | else 33 | echo "$NAME2 started" | logger -t "$NAME" 34 | fi 35 | rm -f $PIDFILE 36 | fi 37 | end script 38 | 39 | pre-stop script 40 | echo "$NAME2 stopped" | logger -t "$NAME" 41 | end script 42 | -------------------------------------------------------------------------------- /denyhosts_server/models.py: -------------------------------------------------------------------------------- 1 | # denyhosts sync server 2 | # Copyright (C) 2015 Jan-Pascal van Best 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published 6 | # by the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import logging 18 | 19 | from twistar.dbobject import DBObject 20 | 21 | class Cracker(DBObject): 22 | HASMANY=['reports'] 23 | 24 | def __str__(self): 25 | return "Cracker({},{},{},{},{},{})".format(self.id,self.ip_address,self.first_time,self.latest_time,self.resiliency,self.total_reports,self.current_reports) 26 | 27 | class Report(DBObject): 28 | BELONGSTO=['cracker'] 29 | 30 | def __str__(self): 31 | return "Report({},{},{},{})".format(self.id,self.ip_address,self.first_report_time,self.latest_report_time) 32 | 33 | class Legacy(DBObject): 34 | TABLENAME="legacy" 35 | pass 36 | 37 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 38 | -------------------------------------------------------------------------------- /tests/sim_clients.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import xmlrpclib 3 | import time 4 | import threading 5 | import random 6 | 7 | def random_ip_address(): 8 | return ".".join(map(str, (random.randint(0, 255) for _ in range(4)))) 9 | 10 | server = 'http://localhost:9911' 11 | 12 | def run(server, count): 13 | # Assuming every client will supply 1 new host every 8minutes 14 | # and ask for newly recognised crackers 15 | s = xmlrpclib.ServerProxy(server) 16 | for i in xrange(count): 17 | ip = random_ip_address() 18 | try: 19 | s.add_hosts([ip]) 20 | s.get_new_hosts(time.time()-480, 3, [ip], 3600) 21 | except Exception, e: 22 | print("Got exception {}".format(e)) 23 | 24 | def run_sim(server, num_threads, count): 25 | start_time = time.time() 26 | 27 | threads = [] 28 | print("Creating threads...") 29 | for i in xrange(num_threads): 30 | thread = threading.Thread(target=run, args=(server, count)) 31 | threads.append(thread) 32 | 33 | print("Starting threads...") 34 | for i in xrange(num_threads): 35 | threads[i].start() 36 | 37 | #print("Waiting for threads...") 38 | for i in xrange(num_threads): 39 | threads[i].join() 40 | 41 | end_time = time.time() 42 | print("Simulation {} clients {} times took {} seconds".format(count, num_threads, end_time - start_time)) 43 | print("Average time per client : {} seconds".format( (end_time - start_time) / count / num_threads)) 44 | print("Average requests per seconds: {}".format( 45 | count * num_threads / (end_time - start_time))) 46 | 47 | 48 | run_sim(server, 60, 100) 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /packaging/debian/denyhosts-server.postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | CNF=/etc/denyhosts-server.conf 6 | WD=/var/lib/denyhosts-server 7 | # denyhosts-server user 8 | DSU=denyhosts-server 9 | DSG=denyhosts 10 | DAEMON=/usr/sbin/denyhosts-server 11 | 12 | case "$1" in 13 | configure) 14 | if [ -z "$2" ]; then 15 | echo "" 16 | echo "Add user $DSU if not exists." 17 | 18 | addgroup --system $DSG || true 19 | adduser --system --no-create-home --home $WD \ 20 | --ingroup $DSG --disabled-password --disabled-login --gecos /dev/null $DSU || true 21 | usermod -aG www-data $DSU || true 22 | # For init/upgrade db 23 | usermod -s /bin/sh $DSU || true 24 | 25 | chown -R $DSU:$DSG /var/log/denyhosts-server 26 | chown -R $DSU:www-data /var/lib/denyhosts-server 27 | chown -R $DSU:www-data /var/www/sites/denyhosts-server 28 | 29 | echo "" 30 | service denyhosts-server stop || true 31 | echo "Init denyhosts-server database." 32 | su -c "cd $WD; $DAEMON -c $CNF --recreate-database --force" - $DSU 33 | echo "" 34 | else 35 | echo "" 36 | echo "Upgrade denyhosts-server database." 37 | service denyhosts-server stop || true 38 | su -c "cd $WD; $DAEMON -c $CNF --evolve-database --force" - $DSU 39 | echo "" 40 | fi 41 | service nginx reload || true 42 | ;; 43 | 44 | abort-upgrade|abort-remove|abort-deconfigure) 45 | ;; 46 | 47 | *) 48 | echo "postinst called with unknown argument '$1'" >&2 49 | exit 1 50 | ;; 51 | esac 52 | 53 | #DEBHELPER# 54 | 55 | exit 0 56 | -------------------------------------------------------------------------------- /denyhosts-server.1: -------------------------------------------------------------------------------- 1 | .TH denyhosts-server 1 "Fri Juli 24 23:51:12 CEST 2015" 2 | .SH NAME 3 | .B denyhosts-server 4 | - service to share blocked addresses from denyhosts. 5 | .SH SYNOPSIS 6 | .B denyhosts-server [-h] [-c CONFIG] [--recreate-database] 7 | .B [--evolve-database] [--purge-legacy-addresses] 8 | .B [--purge-reported-addresses] [--purge-ip IP_ADDRESS] [-f] 9 | .SH DESCRIPTION 10 | .PP 11 | denyhosts-server is a server that allows denyhosts clients to share blocked IP 12 | addresses. It is intended to be a drop-in replacement for the service at 13 | xmlrpc.denyhosts.net that up to now has been provided by the original author 14 | of denyhosts. 15 | 16 | .SH "OPTIONS 17 | .TP 18 | .B \-h 19 | Print help information and exit. 20 | .TP 21 | .B \-c CONFIG 22 | Read configuration from CONFIG 23 | .TP 24 | .B \-\-recreate-database 25 | If necessary, remove old database tables, then create new tables and exit. 26 | .TP 27 | .B \-\-evolve-database 28 | Update database schema to the latest version and exit. 29 | .TP 30 | .B \-\-purge-legacy-addresses 31 | Purge all ip addresses previously downloaded from the legacy server and exit. 32 | .TP 33 | .B \-\-purge-reported-addresses 34 | Purge all ip addresses previously reported by denyhosts clients and exit. 35 | .TP 36 | .B \-\-purge-ip IP_ADDRESS 37 | Purge given IP address from the database and exit. The address is purged 38 | from the legacy table and from the reports by denyhosts clients. 39 | .TP 40 | .B \-f, \-\-force 41 | When the \-\-recreate-database, \-\-evolve-database, \-\-purge-legacy-addresses, 42 | \-\-purge-reported-addresses or \-\-purge-ip are given, execute the given 43 | operation directly without asking questions. 44 | 45 | .SH AUTHOR 46 | Jan-Pascal van Best 47 | .SH SEE ALSO 48 | .B denyhosts 49 | (1) 50 | -------------------------------------------------------------------------------- /tests/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import xmlrpclib 3 | import time 4 | 5 | 6 | server = 'http://localhost:9911' 7 | print("Connecting to server {}".format(server)) 8 | s = xmlrpclib.ServerProxy(server) 9 | 10 | #print s.add_hosts(["127.0.0.3"]) 11 | #print s.add_hosts(["192.168.1.22"]) 12 | print("Adding one host") 13 | print s.add_hosts(["69.192.72.154"]) 14 | print s.add_hosts(["69.192.72.155"]) 15 | print s.add_hosts(["69.192.72.156"]) 16 | print s.add_hosts(["69.192.72.157"]) 17 | 18 | # Concurrency testing 19 | for i in range(0, 100): 20 | print("Running test {}".format(i)) 21 | s.debug.test() 22 | print("Running maintenance") 23 | s.debug.maintenance() 24 | time.sleep(5) 25 | 26 | s.debug.test() 27 | s.debug.maintenance() 28 | 29 | #print s.add_hosts(["test4.example.org"]) 30 | 31 | #print("New crackers, resilience=3600") 32 | #print s.get_new_hosts(time.time()-3600, 3, ["old.example.net"], 3600) 33 | #print("New crackers, resilience=60") 34 | #print s.get_new_hosts(time.time()-3600, 3, ["old.example.net"], 60) 35 | #print("New crackers, resilience=60, min_reporters=2") 36 | #print s.get_new_hosts(time.time()-3600, 2, ["old.example.net"], 60) 37 | #print("New crackers, resilience=60, min_reporters=1") 38 | #print s.get_new_hosts(time.time()-3600, 1, ["old.example.net"], 60) 39 | print("Illegal arguments: ") 40 | try: 41 | print s.get_new_hosts("12312iasda", 1, ["old.example.net"], 60) 42 | except Exception, e: 43 | print(e) 44 | print("New crackers, resilience=60, min_reporters=1") 45 | print s.get_new_hosts(time.time()-3600, 1, ["69.192.72.154"], 60) 46 | #print("All hosts:") 47 | #print s.list_all_hosts() 48 | # #print s.dump_database() 49 | # 50 | #print("Cracker info for 69.192.72.154:") 51 | #try: 52 | # print s.get_cracker_info("69.192.72.154") 53 | #except Exception, e: 54 | # print e 55 | # 56 | -------------------------------------------------------------------------------- /packaging/debian/copyright: -------------------------------------------------------------------------------- 1 | Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: DenyHosts-Server 3 | Upstream-Contact: Jan-Pascal van Best 4 | Source: https://github.com/janpascal/denyhosts_sync 5 | 6 | Files: * 7 | Copyright: 2005-2006, Phil Schwartz 8 | 2014-2015, Jesse Smith 9 | 2012, 2014-2015, Matt Ruffalo 10 | License: GPL-2+ 11 | 2015, Jan-Pascal van Best janpascal@vanbest.org 12 | License: GPL-3 13 | 14 | Files: debian/* 15 | Copyright: 2006, Marco Bertorello 16 | 2011, Kyle Willmon 17 | License: GPL-2+ 18 | 2015, Sergey Dryabzhinsky 19 | License: GPL-3 20 | 21 | License: GPL-2+ 22 | This program is free software; you can redistribute it and/or modify 23 | it under the terms of the GNU General Public License as published by 24 | the Free Software Foundation; either version 2 of the License. 25 | . 26 | This program is distributed in the hope that it will be useful, 27 | but WITHOUT ANY WARRANTY; without even the implied warranty of 28 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 29 | GNU General Public License for more details. 30 | . 31 | You should have received a copy of the GNU General Public License 32 | along with this package; if not, write to the Free Software 33 | Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 34 | 02110-1301, USA. 35 | . 36 | On Debian systems, the complete text of the GNU General Public 37 | License, version 2, can be found in '/usr/share/common-licenses/GPL-2'. 38 | . 39 | On Debian systems, the complete text of the GNU General Public 40 | License, version 3, can be found in '/usr/share/common-licenses/GPL-3'. 41 | -------------------------------------------------------------------------------- /denyhosts_server/utils.py: -------------------------------------------------------------------------------- 1 | # denyhosts sync server 2 | # Copyright (C) 2015 Jan-Pascal van Best 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published 6 | # by the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import logging 18 | 19 | from twisted.internet.defer import inlineCallbacks, returnValue 20 | from twisted.internet import reactor, task 21 | 22 | _hosts_busy = set() 23 | 24 | @inlineCallbacks 25 | def wait_and_lock_host(host): 26 | try: 27 | while host in _hosts_busy: 28 | logging.debug("waiting to update host {}, {} blocked now".format(host, len(_hosts_busy))) 29 | yield task.deferLater(reactor, 0.01, lambda _:0, 0) 30 | _hosts_busy.add(host) 31 | except: 32 | logging.debug("Exception in locking {}".format(host), exc_info=True) 33 | 34 | returnValue(0) 35 | 36 | def unlock_host(host): 37 | try: 38 | _hosts_busy.remove(host) 39 | #logging.debug("host {} unlocked, {} blocked now".format(host, len(_hosts_busy))) 40 | except: 41 | logging.debug("Exception in unlocking {}".format(host), exc_info=True) 42 | 43 | def none_waiting(): 44 | return len(_hosts_busy) == 0 45 | 46 | def count_waiting(): 47 | return len(_hosts_busy) 48 | 49 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 50 | -------------------------------------------------------------------------------- /tests/fill_database.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import xmlrpclib 3 | import time 4 | import threading 5 | import random 6 | 7 | server = 'http://localhost:9911' 8 | 9 | def run(server, count, known_crackers): 10 | #print("Connecting to server {}".format(server)) 11 | s = xmlrpclib.ServerProxy(server) 12 | start_time = time.time() 13 | s.debug.test_bulk_insert(count, known_crackers, start_time - random.random()*7*24*3600) 14 | #print("Inserting {} hosts took {} seconds".format(count, time.time() - start_time)) 15 | 16 | def run_insert_test(server, num_threads, count, known_crackers): 17 | start_time = time.time() 18 | 19 | threads = [] 20 | #print("Creating threads...") 21 | for i in xrange(num_threads): 22 | thread = threading.Thread(target=run, args=(server, count, known_crackers)) 23 | threads.append(thread) 24 | 25 | #print("Starting threads...") 26 | for i in xrange(num_threads): 27 | threads[i].start() 28 | 29 | #print("Waiting for threads...") 30 | for i in xrange(num_threads): 31 | threads[i].join() 32 | 33 | print("Inserting {} hosts {} times took {} seconds".format(count, num_threads, time.time() - start_time)) 34 | print("Average time per insert: {} seconds".format( (time.time() - start_time) / count / num_threads)) 35 | 36 | 37 | s = xmlrpclib.ServerProxy(server) 38 | for num_threads in xrange(20, 59): 39 | print("Inserting {} hosts {} times, please wait...".format(100, num_threads)) 40 | s.debug.clear_bulk_cracker_list() 41 | run_insert_test(server, num_threads, 100, True) 42 | time.sleep(3) 43 | 44 | print("==============================================") 45 | 46 | #s = xmlrpclib.ServerProxy(server) 47 | #for i in xrange(100): 48 | # s.clear_bulk_cracker_list() 49 | # print("Adding 20*5000=100,000 hosts...") 50 | # run_insert_test(server, 20, 5000, False) 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /packaging/debian/patches/01-edit-setup-script.patch: -------------------------------------------------------------------------------- 1 | --- a/setup.py 2 | +++ b/setup.py 3 | @@ -11,24 +11,24 @@ 4 | description='DenyHosts Synchronisation Server', 5 | author='Jan-Pascal van Best', 6 | author_email='janpascal@vanbest.org', 7 | - url='http://www.github.com/janpascal/denyhosts_sync_server', 8 | + url='https://github.com/janpascal/denyhosts_sync', 9 | packages=['denyhosts_server'], 10 | - install_requires=["Twisted", "twistar", "ipaddr", "jinja2", "numpy", "matplotlib", "GeoIP", "minify"], 11 | +# install_requires=["Twisted", "twistar", "ipaddr", "jinja2", "numpy", "matplotlib", "GeoIP", "minify"], 12 | scripts=['scripts/denyhosts-server'], 13 | data_files=[ 14 | - ('static/js', glob('static/js/*.min.js')), 15 | - ('static/css', glob('static/css/*.min.css')), 16 | - ('static/graph', glob('static/graph/README')), 17 | - ('template', glob('template/*')), 18 | - ('docs', [ 19 | - 'README.md', 20 | - 'LICENSE.md', 21 | - 'changelog.txt', 22 | - 'denyhosts-server.conf.example', 23 | - 'denyhosts-server.service.example', 24 | - 'denyhosts-server.init.example' 25 | - ] 26 | - ), 27 | +# ('static/js', glob('static/js/*.min.js')), 28 | +# ('static/css', glob('static/css/*.min.css')), 29 | +# ('static/graph', glob('static/graph/README')), 30 | +# ('template', glob('template/*')), 31 | +# ('docs', [ 32 | +# 'README.md', 33 | +# 'LICENSE.md', 34 | +# 'changelog.txt', 35 | +# 'denyhosts-server.conf.example', 36 | +# 'denyhosts-server.service.example', 37 | +# 'denyhosts-server.init.example' 38 | +# ] 39 | +# ), 40 | ], 41 | license=""" 42 | Copyright (C) 2015 Jan-Pascal van Best 43 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | from glob import glob 5 | from denyhosts_server import version 6 | 7 | etcpath = "/etc" 8 | 9 | setup(name='denyhosts-server', 10 | version=version, 11 | description='DenyHosts Synchronisation Server', 12 | author='Jan-Pascal van Best', 13 | author_email='janpascal@vanbest.org', 14 | url='https://github.com/janpascal/denyhosts_sync', 15 | packages=['denyhosts_server'], 16 | install_requires=["Twisted", "twistar", "ipaddr", "jinja2", "numpy", "matplotlib", "GeoIP", "minify"], 17 | scripts=['scripts/denyhosts-server'], 18 | data_files=[ 19 | ('static/js', glob('static/js/*.min.js')), 20 | ('static/css', glob('static/css/*.min.css')), 21 | ('static/graph', glob('static/graph/README')), 22 | ('template', glob('template/*')), 23 | ('docs', [ 24 | 'README.md', 25 | 'LICENSE.md', 26 | 'changelog.txt', 27 | 'denyhosts-server.conf.example', 28 | 'denyhosts-server.service.example', 29 | 'denyhosts-server.init.example' 30 | ] 31 | ), 32 | ], 33 | license=""" 34 | Copyright (C) 2015 Jan-Pascal van Best 35 | 36 | This program is free software: you can redistribute it and/or modify 37 | it under the terms of the GNU Affero General Public License as published 38 | by the Free Software Foundation, either version 3 of the License, or 39 | (at your option) any later version. 40 | 41 | This program is distributed in the hope that it will be useful, 42 | but WITHOUT ANY WARRANTY; without even the implied warranty of 43 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 44 | GNU Affero General Public License for more details. 45 | 46 | You should have received a copy of the GNU Affero General Public License 47 | along with this program. If not, see . 48 | """ 49 | ) 50 | -------------------------------------------------------------------------------- /denyhosts-server.init.example: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | ### BEGIN INIT INFO 4 | # Provides: denyhosts-server 5 | # Required-Start: $syslog $network $remote_fs 6 | # Required-Stop: $syslog $network $remote_fs 7 | # Should-Start: mysql 8 | # Should-Stop: mysql 9 | # Default-Start: 2 3 4 5 10 | # Default-Stop: 0 1 6 11 | # Short-Description: DenyHosts synchronisation service 12 | # Description: Service that allows DenyHosts clients to share blocked IP 13 | # addresses. 14 | ### END INIT INFO 15 | 16 | PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin 17 | DAEMON=/usr/local/bin/denyhosts-server 18 | DESC="DenyHosts sync server" 19 | NAME=`basename $DAEMON` 20 | PIDFILE=/var/run/$NAME.pid 21 | DAEMONUSER=denyhosts-server 22 | CONFIG_FILE=/etc/denyhosts-server.conf 23 | QUIET=--quiet 24 | 25 | test -x $DAEMON || exit 0 26 | 27 | # Include defaults if available 28 | if [ -f /etc/default/denyhosts-server ] ; then 29 | . /etc/default/denyhosts-server 30 | fi 31 | DAEMON_ARGS="-c $CONFIG_FILE" 32 | 33 | #set -e 34 | 35 | . /lib/lsb/init-functions 36 | 37 | case "$1" in 38 | start) 39 | log_begin_msg "Starting $DESC: $NAME" 40 | start-stop-daemon --start $QUIET --chuid $DAEMONUSER \ 41 | --make-pidfile --pidfile $PIDFILE --background \ 42 | --startas /bin/bash -- -c "exec $DAEMON $DAEMON_ARGS >> /var/log/denyhosts-server/console.log 2>&1" 43 | log_end_msg $? 44 | ;; 45 | stop) 46 | log_begin_msg "Stopping $DESC: $NAME" 47 | start-stop-daemon --stop $QUIET --chuid $DAEMONUSER \ 48 | --pidfile $PIDFILE \ 49 | --startas /bin/bash -- -c "exec $DAEMON $DAEMON_ARGS >> /var/log/denyhosts-server/console.log 2>&1" 50 | log_end_msg $? 51 | ;; 52 | restart) 53 | $0 stop 54 | $0 start 55 | ;; 56 | status) 57 | status_of_proc "$DAEMON" $NAME 58 | ;; 59 | reload|force-reload) 60 | log_begin_msg "Reloading $DESC: $NAME" 61 | start-stop-daemon --stop --signal HUP --pidfile $PIDFILE 62 | log_end_msg $? 63 | ;; 64 | *) 65 | N=/etc/init.d/$NAME 66 | echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2 67 | exit 1 68 | ;; 69 | esac 70 | 71 | exit 0 72 | -------------------------------------------------------------------------------- /packaging/debian/denyhosts-server.init: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | ### BEGIN INIT INFO 4 | # Provides: denyhosts-server 5 | # Required-Start: $syslog $network $remote_fs 6 | # Required-Stop: $syslog $network $remote_fs 7 | # Should-Start: mysql 8 | # Should-Stop: mysql 9 | # Default-Start: 2 3 4 5 10 | # Default-Stop: 0 1 6 11 | # Short-Description: DenyHosts synchronisation service 12 | # Description: Service that allows DenyHosts clients to share blocked IP 13 | # addresses. 14 | ### END INIT INFO 15 | 16 | PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin 17 | DAEMON=/usr/sbin/denyhosts-server 18 | DESC="DenyHosts sync server" 19 | NAME=`basename $DAEMON` 20 | PIDFILE=/var/run/$NAME.pid 21 | DAEMONUSER=denyhosts-server 22 | DAEMONGROUP=denyhosts 23 | WORKDIR=/var/lib/denyhosts-server 24 | CONFIG_FILE=/etc/denyhosts-server.conf 25 | QUIET=--quiet 26 | 27 | test -x $DAEMON || exit 0 28 | 29 | # Include defaults if available 30 | if [ -f /etc/default/denyhosts-server ] ; then 31 | . /etc/default/denyhosts-server 32 | fi 33 | DAEMON_ARGS="-c $CONFIG_FILE" 34 | 35 | #set -e 36 | 37 | . /lib/lsb/init-functions 38 | 39 | case "$1" in 40 | start) 41 | log_begin_msg "Starting $DESC: $NAME" 42 | start-stop-daemon --start $QUIET --chuid $DAEMONUSER:$DAEMONGROUP \ 43 | --chdir $WORKDIR \ 44 | --make-pidfile --pidfile $PIDFILE --background \ 45 | --startas $DAEMON -- $DAEMON_ARGS 46 | log_end_msg $? 47 | ;; 48 | stop) 49 | log_begin_msg "Stopping $DESC: $NAME" 50 | start-stop-daemon --stop $QUIET --retry=TERM/30/KILL/5 --chuid $DAEMONUSER:$DAEMONGROUP \ 51 | --chdir $WORKDIR \ 52 | --pidfile $PIDFILE \ 53 | --startas $DAEMON -- $DAEMON_ARGS 54 | log_end_msg $? 55 | ;; 56 | restart) 57 | $0 stop 58 | $0 start 59 | ;; 60 | status) 61 | status_of_proc "$DAEMON" $NAME 62 | ;; 63 | reload|force-reload) 64 | log_begin_msg "Reloading $DESC: $NAME" 65 | start-stop-daemon --stop --signal HUP --pidfile $PIDFILE 66 | log_end_msg $? 67 | ;; 68 | *) 69 | N=/etc/init.d/$NAME 70 | echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2 71 | exit 1 72 | ;; 73 | esac 74 | 75 | exit 0 76 | -------------------------------------------------------------------------------- /tests/test_concurrency.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from denyhosts_server import models 4 | from denyhosts_server import controllers 5 | from denyhosts_server import views 6 | from denyhosts_server.models import Cracker, Report 7 | 8 | from twisted.internet import reactor,defer,task 9 | from twisted.internet.defer import inlineCallbacks, returnValue 10 | 11 | import base 12 | 13 | def random_ip_address(): 14 | return ".".join(map(str, (random.randint(0, 255) for _ in range(4)))) 15 | 16 | def sleep(seconds): 17 | d = defer.Deferred() 18 | reactor.callLater(seconds, d.callback, seconds) 19 | return d 20 | 21 | class MockRequest: 22 | def __init__(self, ip): 23 | self._ip = ip 24 | 25 | def getClientIP(self): 26 | return self._ip 27 | 28 | class ConcurrencyTest(base.TestBase): 29 | 30 | @inlineCallbacks 31 | def test_try_and_confuse_server(self): 32 | self.view = views.Server() 33 | request = MockRequest("127.0.0.1") 34 | for i in range(0, 50): 35 | print("count:{}".format(i)) 36 | 37 | self.count = 0 38 | def called(result): 39 | self.count += 1 40 | 41 | for t in range(5): 42 | task.deferLater(reactor, 0.01, self.view.xmlrpc_add_hosts, request, ["1.1.1.1", "2.2.2.2"]).addCallback(called) 43 | self.count -= 1 44 | for t in range(5): 45 | task.deferLater(reactor, 0.01, self.view.xmlrpc_add_hosts, request, ["1.1.1.7", "2.2.2.8"]).addCallback(called) 46 | self.count -= 1 47 | task.deferLater(reactor, 0.01, controllers.perform_maintenance).addCallback(called) 48 | self.count -= 1 49 | 50 | while self.count < 0: 51 | yield sleep(0.1) 52 | """" 53 | def run(self, count): 54 | for i in xrange(count): 55 | ip = random_ip_address() 56 | try: 57 | self.view.add_hosts([ip]) 58 | self.view.get_new_hosts(time.time()-480, 3, [ip], 3600) 59 | except Exception, e: 60 | print("Got exception {}".format(e)) 61 | 62 | @inlineCallbacks 63 | def test_simulate_clients(self): 64 | num_threads = 20 65 | num_runs = 100 66 | threads = [] 67 | 68 | print("Creating threads...") 69 | for i in xrange(num_threads): 70 | thread = threading.Thread(target=run, args=(num_runs)) 71 | threads.append(thread) 72 | 73 | print("Starting threads...") 74 | for i in xrange(num_threads): 75 | threads[i].start() 76 | 77 | #print("Waiting for threads...") 78 | for i in xrange(num_threads): 79 | threads[i].join() 80 | """ 81 | -------------------------------------------------------------------------------- /tests/test_get_new_hosts.py: -------------------------------------------------------------------------------- 1 | # denyhosts sync server 2 | # Copyright (C) 2015 Jan-Pascal van Best 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published 6 | # by the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import time 18 | 19 | from denyhosts_server import models 20 | from denyhosts_server import controllers 21 | from denyhosts_server import database 22 | from denyhosts_server.models import Cracker, Report 23 | 24 | from twisted.internet.defer import inlineCallbacks, returnValue 25 | 26 | import base 27 | import logging 28 | 29 | class GetNewHostsTest(base.TestBase): 30 | 31 | @inlineCallbacks 32 | def test_get_qualifying_crackers(self): 33 | now = time.time() 34 | c = yield Cracker(ip_address="192.168.1.1", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 35 | 36 | # First test: cracker without reports should not be reported 37 | hosts = yield controllers.get_qualifying_crackers(1, 0, now, 50, []) 38 | self.assertEqual(len(hosts), 0, "Cracker without reports should not be returned") 39 | 40 | client_ip = "1.1.1.1" 41 | yield controllers.add_report_to_cracker(c, client_ip, when=now) 42 | 43 | hosts = yield controllers.get_qualifying_crackers(1, -1, now-1, 50, []) 44 | self.assertEqual(len(hosts), 1, "When one report, get_new_hosts with resilience <0 should return one host") 45 | 46 | hosts = yield controllers.get_qualifying_crackers(2, -1, now-1, 50, []) 47 | self.assertEqual(len(hosts), 0, "When one report, get_new_hosts with resilience 0 and threshold 2 should return empty list") 48 | 49 | client_ip = "1.1.1.2" 50 | yield controllers.add_report_to_cracker(c, client_ip, when=now+3600) 51 | 52 | hosts = yield controllers.get_qualifying_crackers(2, 3500, now-1, 50, []) 53 | self.assertEqual(len(hosts), 1, "Two reports should result in a result") 54 | 55 | hosts = yield controllers.get_qualifying_crackers(2, 3500, now-1, 50, [c.ip_address]) 56 | self.assertEqual(len(hosts), 0, "Two reports, remove reported host from list") 57 | 58 | hosts = yield controllers.get_qualifying_crackers(3, 3500, now-1, 50, []) 59 | self.assertEqual(len(hosts), 0, "Two reports, asked for three") 60 | 61 | hosts = yield controllers.get_qualifying_crackers(2, 4000, now-1, 50, []) 62 | self.assertEqual(len(hosts), 0, "Two reports, not enough resiliency") 63 | 64 | logging.debug("Testing d2") 65 | client_ip = "1.1.1.3" 66 | yield controllers.add_report_to_cracker(c, client_ip, when=now+7200) 67 | 68 | hosts = yield controllers.get_qualifying_crackers(2, 3500, now+3601, 50, []) 69 | self.assertEqual(len(hosts), 0, "Condition (d2)") 70 | 71 | logging.debug("Testing d1") 72 | client_ip = "1.1.1.3" 73 | yield controllers.add_report_to_cracker(c, client_ip, when=now+7200+24*3600+1) 74 | 75 | hosts = yield controllers.get_qualifying_crackers(2, 24*3600+1, now+3601, 50, []) 76 | self.assertEqual(len(hosts), 1, "Condition (d1)") 77 | 78 | 79 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 80 | -------------------------------------------------------------------------------- /changelog.txt: -------------------------------------------------------------------------------- 1 | Release 2.0.0 (2015-10-01) 2 | - Renamed to denyhosts-server to avoid confusion with Debian dh_* commands 3 | - Add percentage to country graph 4 | 5 | Release 1.4.1 (2015-10-01) 6 | - Add status command to init script 7 | - Add Description header to initscript 8 | - Remove ancient IE compatibility cruft 9 | - Put un-minified js and css in package source, minify using 10 | python setup.py minify_js minify_css 11 | 12 | Release 1.4.0 (2015-09-19) 13 | - Add historical statistics that survive a database purge, using 14 | the 'history' table 15 | - Add graph showing the distribution of attacks over countries of origin 16 | - Several small fixes 17 | - Remove hostname column from stats when reverse dns lookup is disabled 18 | 19 | Release 1.3.0 (2015-09-08) 20 | - Improve performance, removed some blocking code from the statistics update 21 | - Better graphs, replaced pygal graphing library with matplotlib 22 | - Fix crash when statistics page was requested before statistics are ready 23 | - Show ip address instead of 'disabled' when hostname resolving is disabled 24 | - Stability fixes 25 | - Add trend lines to graphs 26 | - Add historical graph, showing all reports in the database 27 | - Calculate graphs for stats page in separate thread 28 | 29 | Release 1.2.0 (2015-07-31) 30 | - Fix Content-Type headers of SVG files 31 | - Fix size of SVG graphs for Internet Explorer 32 | - Fix error with generating stats when database is empty 33 | - Improve logging of exceptions 34 | - Provide more information in the README for installing the GeoIP library 35 | 36 | Release 1.1.1 (2015-07-24) 37 | - Bugfix release, include actual JavaScript and CSS files 38 | 39 | Release 1.1 (2015-07-24) 40 | - Moved XMLRPC location to the more standard http://your.host.name:9911/RPC2 41 | - Added server statistics page at http://your.host.name:9911 42 | When updating, please review the [stats] section of the configuration file. 43 | - Write periodical basic database state to log file 44 | - Improved error handling 45 | - Do not install the configuration file by default, provide an example file instead 46 | - Provide example systemd service file and init script 47 | 48 | Release 1.0 (2015-07-15) 49 | - Unit tests added 50 | - Fix and/or document dependencies on ipaddr and other Python libraries 51 | - Add database evolution to repair bug in database maintenance, which could leave crackers without reports 52 | - Add separate legacy_expiry_days config setting 53 | - Added --purge-legacy-addresses, --purge-reported-addresses and --purge-ip command line options 54 | - Change default setting not to use legacy sync server 55 | - Sending SIGHUP now causes dh_syncserver to re-read the configuration file 56 | - Updated and clarified licensing of Anne Bezemer's algorithm 57 | 58 | Release 0.9 (2015-07-06) 59 | - Improve report merging algorithm 60 | - Use better default for parameters for sync with legacy server 61 | 62 | Release 0.4 (2015-07-04) 63 | - Make maintenance job quicker and less memory intensive 64 | - Make debug xmlrpc functions a configuration option 65 | 66 | Release 0.3 (2015-07-02) 67 | - Added README.md 68 | - Improved setup script 69 | - Fix creating initial database 70 | - Fix default log file and add cp_max to default config file 71 | - Add database schema version check at daemon startup 72 | - Exit dh_syncserver from --recreate and --evolve-database in case of error 73 | - Check for supported database type in config file 74 | - Database optimisation 75 | - Clean up default config file 76 | - Support MySQLdb database 77 | - Added --recreate-database and --evolve-database command line options 78 | - Make log level configurable 79 | - Fix concurrency issues 80 | - Define defaults for config file options 81 | - Add automatic periodical legacy sync job, to fetch hosts from legacy sync server 82 | - Stability fixes 83 | - Rename config file to dh_syncserver.conf and install in /etc/ 84 | - Added setup.py script; move main.py to dh_syncserver script 85 | - Moved code to dh_syncserver namespace to make room for tests 86 | - Use hosts_added parameter of get_new_hosts to filter out hosts just sent by client 87 | - Make tcp port to listen on configurable 88 | - Max number number of crackers reported to denyhosts configurable (default 50) 89 | - Implemented maintenance job 90 | - Check IP addresses for local addresses, RFC1918, multicast, loopback, etc 91 | - Refuse timestamps from future; return Fault reply on illegal input 92 | - Check xmlrpc parameters for validity 93 | - Added copyright info 94 | - Mostly implemented synchronisation algorithm from Debian #622697 95 | -------------------------------------------------------------------------------- /denyhosts-server.conf.example: -------------------------------------------------------------------------------- 1 | # section database. All configuration items besides 'type' are passed as-is 2 | # to the database connect() function 3 | 4 | # Database settings. Depending on the database type, you can add several 5 | # parameters to connect to the database. 6 | 7 | # For sqlite3, just fill in the database filename as "database" 8 | # sqlite3 is not suitable for a high volume server 9 | 10 | # For PostgreSQL use the following parameters: 11 | # type - psycopg2 12 | # dbname – the database name (only in the dsn string) 13 | # database – the database name (only as keyword argument) 14 | # user – user name used to authenticate 15 | # password – password used to authenticate 16 | # host – database host address (defaults to UNIX socket if not provided) 17 | # port – connection port number (defaults to 5432 if not provided) 18 | 19 | 20 | # For mysql, use the following parameters: 21 | # host – name of host to connect to. Default: use the local host via a UNIX socket (where applicable) 22 | # user – user to authenticate as. Default: current effective user. 23 | # passwd – password to authenticate with. Default: no password. 24 | # db – database to use. Default: no default database. 25 | # port – TCP port of MySQL server. Default: standard port (3306). 26 | # unix_socket – location of UNIX socket. Default: use default location or TCP for remote hosts. 27 | # connect_timeout – Abort if connect is not completed within given number of seconds. Default: no timeout (?) 28 | 29 | [database] 30 | # Type of database. Choice of sqlite3, MySQLdb, psycopg2 (PostgreSQL) 31 | # Default: sqlite3 32 | #type: sqlite3 33 | 34 | # Database name. Default: /var/lib/denyhosts-server/denyhosts.sqlite 35 | #database: /var/lib/denyhosts-server/denyhosts.sqlite 36 | 37 | # Maximum size of database connection pool. Default: 5 38 | # For high volume servers, set this to 100 or so. 39 | #cp_max: 5 40 | 41 | [sync] 42 | # Maximum number of cracker IP addresses reported back to 43 | # denyhosts clients per sync. Default: 50 44 | #max_reported_crackers: 50 45 | 46 | # TCP addr to listen on. Default: empty (all) 47 | #listen_host: 48 | 49 | # TCP port to listen on. Default: 9911 50 | #listen_port: 9911 51 | 52 | # Enable debugging methods. See debug_views.py for details. 53 | # Default: no 54 | #enable_debug_methods: no 55 | 56 | # Legacy server to use as a source of bad hosts, to bootstrap 57 | # the database. Leave empty if you don't want to use a legacy server. 58 | # Set legacy_server to http://xmlrpc.denyhosts.net:9911 in order to 59 | # use the legacy server maintained by the original DenyHosts author 60 | # Default: No legacy server configured. 61 | #legacy_server: 62 | 63 | # How often (in seconds) to download hosts from legacy server. 64 | # Default: 300 seconds (5 minutes) 65 | #legacy_frequency: 300 66 | 67 | # Threshold value for legacy server. Default: 10 68 | #legacy_threshold = 10 69 | 70 | # Resiliency value for legacy server (in seconds) 71 | # Default: 10800 (three hours) 72 | #legacy_resiliency = 10800 73 | 74 | [maintenance] 75 | # Maintenance interval in seconds (3600 = one hour; 86400 = one day) 76 | # Default: 3600 77 | #interval_seconds: 3600 78 | 79 | # Number of days before reports are expired. Default: 30 80 | #expiry_days: 30 81 | 82 | # Number of days before hosts retrieved from legacy server are expired. Default: 30 83 | #legacy_expiry_days: 30 84 | 85 | [logging] 86 | # Location of the log file. Default: /var/log/denyhosts-server/denyhosts-server.log 87 | #logfile: /var/log/denyhosts-server/denyhosts-server.log 88 | 89 | # Log level. One of CRITICAL, ERROR, WARNING, INFO of DEBUG 90 | # Default: INFO. Set to WARNING for high-volume server 91 | #loglevel: INFO 92 | 93 | [stats] 94 | # How often (in seconds) to update the statistics HTML page 95 | # and graphs. Default: 600 (10 minutes) 96 | #update_frequency: 600 97 | 98 | # Location of static files. Place the css directory containing 99 | # bootstrap.min.css and the js directory containing bootstrap.min.js 100 | # here. Default: static/ under the project root 101 | #static_dir: static 102 | 103 | # Location of graph files. The server will cache the generated statistic 104 | # graph images here. This directory should be writable by the server. 105 | # Default: static/graph 106 | #graph_dir: static/graph 107 | 108 | # Location of template files. 109 | # Default: template 110 | #template_dir: template 111 | 112 | # Whether to resolve hostnames for the IP addresses in the statistics 113 | # Default: yes 114 | #resolve_hostnames: yes 115 | 116 | # TCP addr to serve statistics. Can be the same a the listen_host in the 117 | # [sync] section. Default: empty (all) 118 | #listen_host: 119 | 120 | # TCP port to serve statistics. Can be the same a the listen_port in the 121 | # [sync] section. Default: 9911 122 | #listen_port: 9911 123 | -------------------------------------------------------------------------------- /packaging/debian/etc/denyhosts-server.conf: -------------------------------------------------------------------------------- 1 | # section database. All configuration items besides 'type' are passed as-is 2 | # to the database connect() function 3 | 4 | # Database settings. Depending on the database type, you can add several 5 | # parameters to connect to the database. 6 | 7 | # For sqlite3, just fill in the database filename as "database" 8 | # sqlite3 is not suitable for a high volume server 9 | 10 | # For PostgreSQL use the following parameters: 11 | # type - psycopg2 12 | # dbname – the database name (only in the dsn string) 13 | # database – the database name (only as keyword argument) 14 | # user – user name used to authenticate 15 | # password – password used to authenticate 16 | # host – database host address (defaults to UNIX socket if not provided) 17 | # port – connection port number (defaults to 5432 if not provided) 18 | 19 | 20 | # For mysql, use the following parameters: 21 | # host – name of host to connect to. Default: use the local host via a UNIX socket (where applicable) 22 | # user – user to authenticate as. Default: current effective user. 23 | # passwd – password to authenticate with. Default: no password. 24 | # db – database to use. Default: no default database. 25 | # port – TCP port of MySQL server. Default: standard port (3306). 26 | # unix_socket – location of UNIX socket. Default: use default location or TCP for remote hosts. 27 | # connect_timeout – Abort if connect is not completed within given number of seconds. Default: no timeout (?) 28 | 29 | [database] 30 | # Type of database. Choice of sqlite3, MySQLdb, psycopg2 (PostgreSQL) 31 | # Default: sqlite3 32 | type: sqlite3 33 | 34 | # Database name. Default: /var/lib/denyhosts-server/denyhosts.sqlite 35 | database: /var/lib/denyhosts-server/denyhosts.sqlite 36 | 37 | # Maximum size of database connection pool. Default: 5 38 | # For high volume servers, set this to 100 or so. 39 | cp_max: 16 40 | 41 | [sync] 42 | # Maximum number of cracker IP addresses reported back to 43 | # denyhosts clients per sync. Default: 50 44 | #max_reported_crackers: 50 45 | 46 | # TCP addr to listen on. Default: 0.0.0.0 47 | #listen_host: 127.0.0.1 48 | 49 | # TCP port to listen on. Default: 9911 50 | #listen_port: 9911 51 | 52 | # Enable debugging methods. See debug_views.py for details. 53 | # Default: no 54 | #enable_debug_methods: no 55 | 56 | # Legacy server to use as a source of bad hosts, to bootstrap 57 | # the database. Leave empty if you don't want to use a legacy server. 58 | # Set legacy_server to http://xmlrpc.denyhosts.net:9911 in order to 59 | # use the legacy server maintained by the original DenyHosts author 60 | # Default: No legacy server configured. 61 | #legacy_server: 62 | 63 | # How often (in seconds) to download hosts from legacy server. 64 | # Default: 300 seconds (5 minutes) 65 | #legacy_frequency: 300 66 | 67 | # Threshold value for legacy server. Default: 10 68 | #legacy_threshold = 10 69 | 70 | # Resiliency value for legacy server (in seconds) 71 | # Default: 10800 (three hours) 72 | #legacy_resiliency = 10800 73 | 74 | [maintenance] 75 | # Maintenance interval in seconds (3600 = one hour; 86400 = one day) 76 | # Default: 3600 77 | #interval_seconds: 3600 78 | 79 | # Number of days before reports are expired. Default: 30 80 | expiry_days: 366 81 | 82 | # Number of days before hosts retrieved from legacy server are expired. Default: 30 83 | #legacy_expiry_days: 30 84 | 85 | [logging] 86 | # Location of the log file. Default: /var/log/denyhosts-server/denyhosts-server.log 87 | logfile: /var/log/denyhosts-server/denyhosts-server.log 88 | 89 | # Log level. One of CRITICAL, ERROR, WARNING, INFO of DEBUG 90 | # Default: INFO. Set to WARNING for high-volume server 91 | #loglevel: INFO 92 | 93 | [stats] 94 | # How often (in seconds) to update the statistics HTML page 95 | # and graphs. Default: 600 (10 minutes) 96 | #update_frequency: 600 97 | 98 | # Location of static files. Place the css directory containing 99 | # bootstrap.min.css and the js directory containing bootstrap.min.js 100 | # here. Default: static/ under the project root 101 | #static_dir: static 102 | 103 | # Location of graph files. The server will cache the generated statistic 104 | # graph images here. This directory should be writable by the server. 105 | # Default: static/graph 106 | #graph_dir: static/graph 107 | 108 | # Location of template files. 109 | # Default: template 110 | #template_dir: template 111 | 112 | # Whether to resolve hostnames for the IP addresses in the statistics 113 | # Default: yes 114 | resolve_hostnames: no 115 | 116 | # TCP addr to serve statistics. Can be the same a the listen_host in the 117 | # [sync] section. Default: 0.0.0.0 118 | #listen_host: 127.0.0.1 119 | 120 | # TCP port to serve statistics. Can be the same a the listen_port in the 121 | # [sync] section. Default: 9911 122 | #listen_port: 8811 123 | -------------------------------------------------------------------------------- /tests/test_purge_methods.py: -------------------------------------------------------------------------------- 1 | # denyhosts sync server 2 | # Copyright (C) 2015 Jan-Pascal van Best 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published 6 | # by the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import time 18 | 19 | from denyhosts_server import models 20 | from denyhosts_server import controllers 21 | from denyhosts_server import database 22 | from denyhosts_server.models import Cracker, Report, Legacy 23 | 24 | from twisted.internet.defer import inlineCallbacks, returnValue 25 | 26 | import base 27 | 28 | class PurgingTest(base.TestBase): 29 | 30 | @inlineCallbacks 31 | def test_purge_reports(self): 32 | now = time.time() 33 | c = yield Cracker(ip_address="192.168.1.1", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 34 | c2 = yield Cracker(ip_address="192.168.1.2", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 35 | c3 = yield Cracker(ip_address="192.168.1.3", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 36 | 37 | yield controllers.add_report_to_cracker(c, "127.0.0.1", when=now) 38 | yield controllers.add_report_to_cracker(c, "127.0.0.2", when=now) 39 | yield controllers.add_report_to_cracker(c, "127.0.0.9", when=now) 40 | yield controllers.add_report_to_cracker(c2, "127.0.0.3", when=now) 41 | 42 | yield controllers.purge_legacy_addresses() 43 | 44 | crackers = yield Cracker.all() 45 | self.assertEqual(len(crackers), 3, "Should still have three crackers after purging legacy") 46 | 47 | reports = yield Report.all() 48 | self.assertEqual(len(reports), 4, "Should still have four reports after purging legacy") 49 | 50 | yield controllers.purge_reported_addresses() 51 | 52 | crackers = yield Cracker.all() 53 | self.assertEqual(len(crackers), 0, "Should have no crackers after purging reports") 54 | 55 | reports = yield Report.all() 56 | self.assertEqual(len(reports), 0, "Should have no reports after purging reports") 57 | 58 | @inlineCallbacks 59 | def test_purge_legacy(self): 60 | now = time.time() 61 | legacy = yield Legacy(ip_address="192.168.211.1", retrieved_time=now).save() 62 | legacy = yield Legacy(ip_address="192.168.211.2", retrieved_time=now).save() 63 | 64 | legacy = yield Legacy.all() 65 | self.assertEqual(len(legacy), 2, "Should have two legacy reports") 66 | 67 | yield controllers.purge_reported_addresses() 68 | 69 | legacy = yield Legacy.all() 70 | self.assertEqual(len(legacy), 2, "Should still have two legacy reports after purging client reports") 71 | 72 | yield controllers.purge_legacy_addresses() 73 | 74 | legacy = yield Legacy.all() 75 | self.assertEqual(len(legacy), 0, "Should no legacy reports after purging legacy") 76 | 77 | rows = yield database.run_query('SELECT `value` FROM info WHERE `key`="last_legacy_sync"') 78 | self.assertEqual(rows[0][0], '0', "Purging legacy should reset last legacy sync time") 79 | 80 | @inlineCallbacks 81 | def test_purge_ip(self): 82 | now = time.time() 83 | c = yield Cracker(ip_address="192.168.211.1", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 84 | c2 = yield Cracker(ip_address="192.168.211.2", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 85 | c3 = yield Cracker(ip_address="192.168.211.3", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 86 | 87 | yield controllers.add_report_to_cracker(c, "127.0.0.1", when=now) 88 | yield controllers.add_report_to_cracker(c, "127.0.0.2", when=now) 89 | yield controllers.add_report_to_cracker(c, "127.0.0.9", when=now) 90 | yield controllers.add_report_to_cracker(c2, "127.0.0.3", when=now) 91 | 92 | legacy = yield Legacy(ip_address="192.168.211.1", retrieved_time=now).save() 93 | legacy = yield Legacy(ip_address="192.168.211.2", retrieved_time=now).save() 94 | 95 | yield controllers.purge_ip("192.168.211.1") 96 | 97 | crackers = yield Cracker.find(orderby='ip_address ASC') 98 | self.assertEqual(len(crackers), 2, "Should still have two crackers after purging one") 99 | self.assertEquals(crackers[0].ip_address, "192.168.211.2", "Should remove the right cracker") 100 | self.assertEquals(crackers[1].ip_address, "192.168.211.3", "Should remove the right cracker") 101 | 102 | reports = yield Report.all() 103 | self.assertEqual(len(reports), 1, "Should still have one report left after purging cracker with three reports") 104 | self.assertEquals(reports[0].ip_address, "127.0.0.3", "Should remove the right report") 105 | 106 | legacy = yield Legacy.all() 107 | self.assertEqual(len(legacy), 1, "Should still have one legacy reports after purging one") 108 | self.assertEquals(legacy[0].ip_address, "192.168.211.2", "Should remove the right legacy host") 109 | 110 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 111 | -------------------------------------------------------------------------------- /denyhosts_server/debug_views.py: -------------------------------------------------------------------------------- 1 | # denyhosts sync server 2 | # Copyright (C) 2015 Jan-Pascal van Best 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published 6 | # by the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import time 18 | import logging 19 | import random 20 | 21 | from twisted.web import xmlrpc 22 | from twisted.web.xmlrpc import withRequest 23 | from twisted.internet.defer import inlineCallbacks, returnValue 24 | from twisted.internet import reactor 25 | import ipaddr 26 | 27 | import models 28 | from models import Cracker, Report 29 | import config 30 | import controllers 31 | import utils 32 | 33 | class DebugServer(xmlrpc.XMLRPC): 34 | """ 35 | Debug xmlrpc methods 36 | """ 37 | 38 | def __init__(self, server): 39 | self.server = server 40 | 41 | @inlineCallbacks 42 | def xmlrpc_list_all_hosts(self): 43 | crackers = yield Cracker.all() 44 | returnValue([c.ip_address for c in crackers]) 45 | 46 | # Concurrency test. Remove before public installation! 47 | @withRequest 48 | def xmlrpc_test(self, request): 49 | reactor.callLater(1, self.server.xmlrpc_add_hosts, request, ["1.1.1.1", "2.2.2.2"]) 50 | reactor.callLater(1, self.server.xmlrpc_add_hosts, request, ["1.1.1.1", "2.2.2.2"]) 51 | reactor.callLater(1, self.server.xmlrpc_add_hosts, request, ["1.1.1.1", "2.2.2.2"]) 52 | reactor.callLater(1, self.server.xmlrpc_add_hosts, request, ["1.1.1.1", "2.2.2.2"]) 53 | reactor.callLater(1, self.server.xmlrpc_add_hosts, request, ["1.1.1.1", "2.2.2.2"]) 54 | reactor.callLater(1, self.server.xmlrpc_add_hosts, request, ["1.1.1.7", "2.2.2.8"]) 55 | reactor.callLater(1, self.server.xmlrpc_add_hosts, request, ["1.1.1.7", "2.2.2.8"]) 56 | reactor.callLater(1, self.server.xmlrpc_add_hosts, request, ["1.1.1.7", "2.2.2.8"]) 57 | reactor.callLater(1, self.server.xmlrpc_add_hosts, request, ["1.1.1.7", "2.2.2.8"]) 58 | reactor.callLater(1, self.server.xmlrpc_add_hosts, request, ["1.1.1.7", "2.2.2.8"]) 59 | return 0 60 | 61 | # For concurrency testing. Remove before public installation! 62 | def xmlrpc_maintenance(self): 63 | return controllers.perform_maintenance() 64 | 65 | def random_ip_address(self): 66 | while True: 67 | ip = ".".join(map(str, (random.randint(0, 255) for _ in range(4)))) 68 | if self.server.is_valid_ip_address(ip): 69 | return ip 70 | 71 | _crackers = [] 72 | @inlineCallbacks 73 | def xmlrpc_test_bulk_insert(self, count, same_crackers = False, when=None): 74 | if same_crackers and len(self._crackers) < count: 75 | logging.debug("Filling static crackers from {} to {}".format(len(self._crackers), count)) 76 | for i in xrange(len(self._crackers), count): 77 | self._crackers.append(self.random_ip_address()) 78 | 79 | if when is None: 80 | when = time.time() 81 | 82 | for i in xrange(count): 83 | reporter = self.random_ip_address() 84 | if same_crackers: 85 | cracker_ip = self._crackers[i] 86 | else: 87 | cracker_ip = self.random_ip_address() 88 | 89 | logging.debug("Adding report for {} from {}".format(cracker_ip, reporter)) 90 | 91 | yield utils.wait_and_lock_host(cracker_ip) 92 | 93 | cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1) 94 | if cracker is None: 95 | cracker = Cracker(ip_address=cracker_ip, first_time=when, latest_time=when, total_reports=0, current_reports=0) 96 | yield cracker.save() 97 | yield controllers.add_report_to_cracker(cracker, reporter, when=when) 98 | 99 | utils.unlock_host(cracker_ip) 100 | logging.debug("Done adding report for {} from {}".format(cracker_ip,reporter)) 101 | total = yield Cracker.count() 102 | total_reports = yield Report.count() 103 | returnValue((total,total_reports)) 104 | 105 | def xmlrpc_clear_bulk_cracker_list(self): 106 | self._crackers = [] 107 | return 0 108 | 109 | @inlineCallbacks 110 | def xmlrpc_get_cracker_info(self, ip): 111 | if not self.server.is_valid_ip_address(ip): 112 | logging.warning("Illegal host ip address {}".format(ip)) 113 | raise xmlrpc.Fault(101, "Illegal IP address \"{}\".".format(ip)) 114 | #logging.info("Getting info for cracker {}".format(ip_address)) 115 | cracker = yield controllers.get_cracker(ip) 116 | if cracker is None: 117 | raise xmlrpc.Fault(104, "Cracker {} unknown".format(ip)) 118 | returnValue([]) 119 | 120 | #logging.info("found cracker: {}".format(cracker)) 121 | reports = yield cracker.reports.get() 122 | #logging.info("found reports: {}".format(reports)) 123 | cracker_cols=['ip_address','first_time', 'latest_time', 'resiliency', 'total_reports', 'current_reports'] 124 | report_cols=['ip_address','first_report_time', 'latest_report_time'] 125 | returnValue( [cracker.toHash(cracker_cols), [r.toHash(report_cols) for r in reports]] ) 126 | 127 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 128 | -------------------------------------------------------------------------------- /denyhosts_server/config.py: -------------------------------------------------------------------------------- 1 | # denyhosts sync server 2 | # Copyright (C) 2015 Jan-Pascal van Best 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published 6 | # by the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import ConfigParser 18 | import inspect 19 | import logging 20 | import os 21 | import os.path 22 | import sys 23 | import sqlite3 24 | 25 | def _get(config, section, option, default=None): 26 | try: 27 | result = config.get(section, option) 28 | except ConfigParser.NoOptionError: 29 | result = default 30 | return result 31 | 32 | def _getint(config, section, option, default=None): 33 | try: 34 | result = config.getint(section, option) 35 | except ConfigParser.NoOptionError: 36 | result = default 37 | return result 38 | 39 | def _getboolean(config, section, option, default=None): 40 | try: 41 | result = config.getboolean(section, option) 42 | except ConfigParser.NoOptionError: 43 | result = default 44 | return result 45 | 46 | def _getfloat(config, section, option, default=None): 47 | try: 48 | result = config.getfloat(section, option) 49 | except ConfigParser.NoOptionError: 50 | result = default 51 | return result 52 | 53 | def read_config(filename): 54 | global dbtype, dbparams 55 | global maintenance_interval, expiry_days, legacy_expiry_days 56 | global max_reported_crackers 57 | global logfile 58 | global loglevel 59 | global xmlrpc_listen_host 60 | global xmlrpc_listen_port 61 | global legacy_server 62 | global legacy_frequency 63 | global legacy_threshold, legacy_resiliency 64 | global enable_debug_methods 65 | global stats_frequency 66 | global stats_resolve_hostnames 67 | global stats_listen_host 68 | global stats_listen_port 69 | global static_dir, graph_dir, template_dir 70 | 71 | _config = ConfigParser.SafeConfigParser() 72 | _config.readfp(open(filename,'r')) 73 | 74 | dbtype = _get(_config, "database", "type", "sqlite3") 75 | if dbtype not in ["sqlite3","MySQLdb"]: 76 | print("Database type {} not supported, exiting".format(dbtype)) 77 | sys.exit() 78 | 79 | dbparams = { 80 | key: value 81 | for (key,value) in _config.items("database") 82 | if key != "type" 83 | } 84 | if dbtype=="sqlite3": 85 | dbparams["check_same_thread"] = False 86 | dbparams["detect_types"] = sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 87 | dbparams["cp_max"] = 1 88 | if "database" not in dbparams: 89 | dbparams["database"] = "/var/lib/denyhosts-server/denyhosts.sqlite" 90 | elif dbtype=="MySQLdb": 91 | dbparams["cp_reconnect"] = True 92 | 93 | if "cp_max" in dbparams: 94 | dbparams["cp_max"] = int(dbparams["cp_max"]) 95 | if "cp_min" in dbparams: 96 | dbparams["cp_min"] = int(dbparams["cp_min"]) 97 | if "port" in dbparams: 98 | dbparams["port"] = int(dbparams["port"]) 99 | if "connect_timeout" in dbparams: 100 | dbparams["connect_timeout"] = float(dbparams["connect_timeout"]) 101 | if "timeout" in dbparams: 102 | dbparams["timeout"] = float(dbparams["timeout"]) 103 | 104 | maintenance_interval = _getint(_config, "maintenance", "interval_seconds", 3600) 105 | expiry_days = _getfloat(_config, "maintenance", "expiry_days", 30) 106 | legacy_expiry_days = _getfloat(_config, "maintenance", "legacy_expiry_days", 30) 107 | 108 | max_reported_crackers = _getint(_config, "sync", "max_reported_crackers", 50) 109 | xmlrpc_listen_host = _get(_config, "sync", "listen_host", "") 110 | xmlrpc_listen_port = _getint(_config, "sync", "listen_port", 9911) 111 | enable_debug_methods = _getboolean(_config, "sync", "enable_debug_methods", False) 112 | legacy_server = _get(_config, "sync", "legacy_server", None) 113 | legacy_frequency = _getint(_config, "sync", "legacy_frequency", 300) 114 | legacy_threshold = _getint(_config, "sync", "legacy_threshold", 10) 115 | legacy_resiliency = _getint(_config, "sync", "legacy_resiliency", 10800) 116 | 117 | logfile = _get(_config, "logging", "logfile", "/var/log/denyhosts-server/denyhosts-server.log") 118 | loglevel = _get(_config, "logging", "loglevel", "INFO") 119 | try: 120 | loglevel = int(loglevel) 121 | except ValueError: 122 | try: 123 | loglevel = logging.__dict__[loglevel] 124 | except KeyError: 125 | print("Illegal log level {}".format(loglevel)) 126 | loglevel = logging.INFO 127 | 128 | stats_frequency = _getint(_config, "stats", "update_frequency", 600) 129 | # By default - static files placed in process working directory 130 | package_dir = os.getcwd() 131 | if not os.path.exists( os.path.join( package_dir, "static" ) ): 132 | # If not - use developer or inplace working path 133 | package_dir = os.path.dirname(os.path.dirname(inspect.getsourcefile(read_config))) 134 | static_dir = _get(_config, "stats", "static_dir", 135 | os.path.join( 136 | package_dir, 137 | "static")) 138 | graph_dir = _get(_config, "stats", "graph_dir", os.path.join(static_dir, "graph")) 139 | template_dir = _get(_config, "stats", "template_dir", os.path.join(package_dir, "template")) 140 | stats_resolve_hostnames = _getboolean(_config, "stats", "resolve_hostnames", True) 141 | stats_listen_host = _get(_config, "stats", "listen_host", "") 142 | stats_listen_port = _getint(_config, "stats", "listen_port", 9911) 143 | -------------------------------------------------------------------------------- /template/stats.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | DenyHosts 8 | 9 | 10 | 11 | 12 |

DenyHosts synchronisation server statistics

13 | 14 | 15 | 16 | 17 |
18 |
19 |

Summary

20 |
21 |
22 |
23 |
24 |
25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 |
Number of clients contributing data{{ num_clients }}
Number of reported hosts{{ num_hosts }}
Total number of reports{{ num_reports }}
Reports in last 24 hours{{ daily_reports }}
New hosts in last 24 hours{{ daily_new_hosts }}
Synchronisation server version{{ server_version }}
45 |
46 |
47 | 74 |
75 |
76 | 77 |
78 |
79 | 80 |
81 |
82 | 83 |
84 |
85 | 86 |
87 |
88 | 89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 | 97 | {% macro hostlist(title, hostlist) -%} 98 |
99 |
100 |

{{ title }}

101 |
102 |
103 | 104 | 105 | 106 | {% if has_hostnames %} 107 | 108 | {% endif %} 109 | 110 | 111 | 112 | 113 | 114 | {% for host in hostlist %} 115 | 116 | 117 | {% if has_hostnames %} 118 | 119 | {% endif %} 120 | 121 | 122 | 123 | 124 | 125 | {% endfor %} 126 |
IP addressHost nameCountryLatest reportFirst reportReport count
{{ host.ip_address }}{{ host.hostname }}{{ host.country }}{{ host.latest_time|datetime }}{{ host.first_time|datetime }}{{ host.total_reports }}
127 |
128 |
129 | {%- endmacro %} 130 | 131 | {{ hostlist("Recent reports (time in UTC)", recent_hosts) }} 132 | 133 | {{ hostlist("Most reported hosts (time in UTC)", most_reported_hosts) }} 134 | 135 |
136 | Page last updated at {{ last_updated|datetime }} 137 |
138 | 139 | 140 | -------------------------------------------------------------------------------- /tests/test_stats.py: -------------------------------------------------------------------------------- 1 | # denyhosts sync server 2 | # Copyright (C) 2015 Jan-Pascal van Best 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published 6 | # by the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import datetime 18 | import inspect 19 | import os 20 | import os.path 21 | import time 22 | import traceback 23 | 24 | from denyhosts_server.models import Cracker, Report, Legacy 25 | from denyhosts_server import config 26 | from denyhosts_server import controllers 27 | from denyhosts_server import stats 28 | 29 | from twisted.internet.defer import inlineCallbacks, returnValue 30 | 31 | from twistar.registry import Registry 32 | 33 | import base 34 | 35 | class StatsTest(base.TestBase): 36 | 37 | @inlineCallbacks 38 | def test_fixup(self): 39 | now = time.time() 40 | c1 = yield Cracker(ip_address="194.109.6.92", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 41 | c2 = yield Cracker(ip_address="192.30.252.128", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 42 | 43 | hosts = [c1, c2] 44 | config.stats_resolve_hostnames = True 45 | stats.fixup_crackers(hosts) 46 | 47 | self.assertEqual(c1.hostname, "www.xs4all.nl", "Reverse DNS of www.xs4all.nl") 48 | self.assertEqual(c1.country, "Netherlands", "Testing geoip of www.xs4all.nl") 49 | 50 | self.assertEqual(c2.hostname, "github.com", "Reverse DNS of github.com") 51 | self.assertEqual(c2.country, "United States", "Testing geoip of github.com") 52 | 53 | def stats_settings(self): 54 | tests_dir = os.path.dirname(inspect.getsourcefile(self.__class__)) 55 | package_dir = os.path.dirname(tests_dir) 56 | config.static_dir = os.path.join(package_dir, "static") 57 | config.template_dir = os.path.join(package_dir, "template") 58 | config.graph_dir = os.path.join(os.getcwd(), "graph") 59 | try: 60 | os.mkdir(config.graph_dir) 61 | except OSError: 62 | pass 63 | config.stats_resolve_hostnames = False 64 | 65 | 66 | @inlineCallbacks 67 | def prepare_stats(self): 68 | self.stats_settings() 69 | 70 | now = time.time() 71 | c1 = yield Cracker(ip_address="192.168.1.1", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 72 | c2 = yield Cracker(ip_address="192.168.1.2", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 73 | 74 | yield controllers.add_report_to_cracker(c1, "127.0.0.1", when=now-25*3600) 75 | yield controllers.add_report_to_cracker(c1, "127.0.0.2", when=now) 76 | yield controllers.add_report_to_cracker(c1, "127.0.0.3", when=now) 77 | yield controllers.add_report_to_cracker(c2, "127.0.0.2", when=now) 78 | yield controllers.add_report_to_cracker(c2, "127.0.0.3", when=now+1) 79 | 80 | yield Registry.DBPOOL.runInteraction(stats.fixup_history_txn) 81 | yesterday = datetime.date.today() - datetime.timedelta(days=1) 82 | yield Registry.DBPOOL.runInteraction(stats.update_country_history_txn, yesterday, include_history=True) 83 | yield stats.update_stats_cache() 84 | 85 | @inlineCallbacks 86 | def test_empty_state(self): 87 | self.stats_settings() 88 | 89 | yield stats.update_stats_cache() 90 | self.assertIsNotNone(stats._cache, "Stats for empty database should not be None") 91 | 92 | cached = stats._cache["stats"] 93 | self.assertEqual(cached["num_hosts"], 0, "Number of hosts in empty database") 94 | self.assertEqual(cached["num_reports"], 0, "Number of reports in empty database") 95 | self.assertEqual(cached["num_clients"], 0, "Number of clients in empty database") 96 | 97 | html = yield stats.render_stats() 98 | #print(html) 99 | self.assertTrue("Number of clients" in html, "HTML should contain number of clients") 100 | self.assertTrue("../static/graphs/hourly.svg" in html, "HTML should contain path to hourly graph") 101 | 102 | @inlineCallbacks 103 | def test_stats_cache(self): 104 | yield self.prepare_stats() 105 | 106 | cached = stats._cache["stats"] 107 | print(cached) 108 | self.assertEqual(cached["num_hosts"], 2, "Number of hosts in database") 109 | self.assertEqual(cached["num_reports"], 5, "Number of reports in database") 110 | self.assertEqual(cached["num_clients"], 3, "Number of clients in database") 111 | 112 | self.assertEquals(cached["recent_hosts"][0].ip_address, "192.168.1.2", "Most recent host") 113 | self.assertEquals(len(cached["recent_hosts"]), 2, "Reported both hosts in most recent list") 114 | 115 | self.assertEquals(cached["recent_hosts"][0].ip_address, "192.168.1.2", "Most recent host") 116 | self.assertEquals(len(cached["recent_hosts"]), 2, "Reported both hosts in most recent list") 117 | 118 | self.assertEquals(cached["most_reported_hosts"][0].ip_address, "192.168.1.1", "Most reported host") 119 | self.assertEquals(len(cached["most_reported_hosts"]), 2, "Reported both hosts in most reported list") 120 | 121 | self.assertTrue(os.access(os.path.join(config.graph_dir, "hourly.svg"), os.R_OK), "Creation of hourly graph") 122 | self.assertTrue(os.access(os.path.join(config.graph_dir, "monthly.svg"), os.R_OK), "Creation of monthly graph") 123 | self.assertTrue(os.access(os.path.join(config.graph_dir, "contrib.svg"), os.R_OK), "Creation of contributors graph") 124 | 125 | @inlineCallbacks 126 | def test_stats_render(self): 127 | yield self.prepare_stats() 128 | 129 | html = yield stats.render_stats() 130 | #print(html) 131 | self.assertTrue("Number of clients" in html, "HTML should contain number of clients") 132 | self.assertTrue("../static/graphs/hourly.svg" in html, "HTML should contain path to hourly graph") 133 | self.assertFalse("127.0.0.1" in html, "HTML should not contain reported ip addresses") 134 | self.assertEqual(html.count("192.168.1.1"), 2, "HTML should contain ip address of hosts in tables") 135 | 136 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 137 | -------------------------------------------------------------------------------- /denyhosts_server/views.py: -------------------------------------------------------------------------------- 1 | # denyhosts sync server 2 | # Copyright (C) 2015 Jan-Pascal van Best 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published 6 | # by the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import time 18 | import logging 19 | import random 20 | 21 | from twisted.web import server, xmlrpc, error 22 | from twisted.web.resource import Resource 23 | from twisted.web.xmlrpc import withRequest 24 | from twisted.internet.defer import inlineCallbacks, returnValue 25 | from twisted.internet import reactor 26 | from twisted.python import log 27 | import ipaddr 28 | 29 | import models 30 | from models import Cracker, Report 31 | import config 32 | import controllers 33 | import utils 34 | import stats 35 | 36 | class Server(xmlrpc.XMLRPC): 37 | """ 38 | An example object to be published. 39 | """ 40 | 41 | @staticmethod 42 | def is_valid_ip_address(ip_address): 43 | try: 44 | ip = ipaddr.IPAddress(ip_address) 45 | except: 46 | return False 47 | if (ip.is_reserved or ip.is_private or ip.is_loopback or 48 | ip.is_unspecified or ip.is_multicast or 49 | ip.is_link_local): 50 | return False 51 | return True 52 | 53 | @withRequest 54 | @inlineCallbacks 55 | def xmlrpc_add_hosts(self, request, hosts): 56 | try: 57 | x_real_ip = request.received_headers.get("X-Real-IP") 58 | remote_ip = x_real_ip or request.getClientIP() 59 | 60 | logging.info("add_hosts({}) from {}".format(hosts, remote_ip)) 61 | for cracker_ip in hosts: 62 | if not self.is_valid_ip_address(cracker_ip): 63 | logging.warning("Illegal host ip address {} from {}".format(cracker_ip, remote_ip)) 64 | raise xmlrpc.Fault(101, "Illegal IP address \"{}\".".format(cracker_ip)) 65 | 66 | logging.debug("Adding report for {} from {}".format(cracker_ip, remote_ip)) 67 | yield utils.wait_and_lock_host(cracker_ip) 68 | try: 69 | cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1) 70 | if cracker is None: 71 | now = time.time() 72 | cracker = Cracker(ip_address=cracker_ip, first_time=now, 73 | latest_time=now, resiliency=0, total_reports=0, current_reports=0) 74 | yield cracker.save() 75 | yield controllers.add_report_to_cracker(cracker, remote_ip) 76 | finally: 77 | utils.unlock_host(cracker_ip) 78 | logging.debug("Done adding report for {} from {}".format(cracker_ip,remote_ip)) 79 | except xmlrpc.Fault, e: 80 | raise e 81 | except Exception, e: 82 | log.err(_why="Exception in add_hosts") 83 | raise xmlrpc.Fault(104, "Error adding hosts: {}".format(str(e))) 84 | 85 | returnValue(0) 86 | 87 | @withRequest 88 | @inlineCallbacks 89 | def xmlrpc_get_new_hosts(self, request, timestamp, threshold, hosts_added, resiliency): 90 | try: 91 | x_real_ip = request.received_headers.get("X-Real-IP") 92 | remote_ip = x_real_ip or request.getClientIP() 93 | 94 | logging.debug("get_new_hosts({},{},{},{}) from {}".format(timestamp, threshold, 95 | hosts_added, resiliency, remote_ip)) 96 | try: 97 | timestamp = long(timestamp) 98 | threshold = int(threshold) 99 | resiliency = long(resiliency) 100 | except: 101 | logging.warning("Illegal arguments to get_new_hosts from client {}".format(remote_ip)) 102 | raise xmlrpc.Fault(102, "Illegal parameters.") 103 | 104 | now = time.time() 105 | # refuse timestamps from the future 106 | if timestamp > now: 107 | logging.warning("Illegal timestamp to get_new_hosts from client {}".format(remote_ip)) 108 | raise xmlrpc.Fault(103, "Illegal timestamp.") 109 | 110 | for host in hosts_added: 111 | if not self.is_valid_ip_address(host): 112 | logging.warning("Illegal host ip address {}".format(host)) 113 | raise xmlrpc.Fault(101, "Illegal IP address \"{}\".".format(host)) 114 | 115 | # TODO: maybe refuse timestamp from far past because it will 116 | # cause much work? OTOH, denyhosts will use timestamp=0 for 117 | # the first run! 118 | # TODO: check if client IP is a known cracker 119 | 120 | result = {} 121 | result['timestamp'] = str(long(time.time())) 122 | result['hosts'] = yield controllers.get_qualifying_crackers( 123 | threshold, resiliency, timestamp, 124 | config.max_reported_crackers, set(hosts_added)) 125 | logging.debug("returning: {}".format(result)) 126 | except xmlrpc.Fault, e: 127 | raise e 128 | except Exception, e: 129 | log.err(_why="Exception in xmlrpc_get_new_hosts") 130 | raise xmlrpc.Fault(105, "Error in get_new_hosts: {}".format(str(e))) 131 | returnValue( result) 132 | 133 | class WebResource(Resource): 134 | #isLeaf = True 135 | 136 | def getChild(self, name, request): 137 | if name == '': 138 | return self 139 | return Resource.getChild(self, name, request) 140 | 141 | def render_GET(self, request): 142 | logging.debug("GET({})".format(request)) 143 | request.setHeader("Content-Type", "text/html; charset=utf-8") 144 | def done(result): 145 | if result is None: 146 | request.write("

An error has occurred

") 147 | else: 148 | request.write(result.encode('utf-8')) 149 | request.finish() 150 | def fail(err): 151 | request.processingFailed(err) 152 | stats.render_stats().addCallbacks(done, fail) 153 | return server.NOT_DONE_YET 154 | 155 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 156 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # README for denyhosts-server 2 | 3 | `denyhosts-server` is a server that allows `denyhosts` clients to share blocked IP 4 | addresses. It is intended to be a drop-in replacement for the service at 5 | `xmlrpc.denyhosts.net` that up to now has been provided by the original author 6 | of `denyhosts`. 7 | 8 | ## Features 9 | - Drop-in replacement for legacy `denyhosts` sync server 10 | - Supports sqlite and mysql databases 11 | - Can run as non-privileged user 12 | - Supports database evolution for easy upgrades 13 | - Robust design which supports hundreds of connections per second 14 | - Supports bootstrapping from legacy server 15 | - Synchronisation algorithm that has safeguards against database poisoning 16 | - Fully configurable 17 | - Dynamically generated statistics web page 18 | 19 | ## Prerequisites 20 | - MySQL database is preferred for large sites. For testing purposes sqlite is 21 | also supported 22 | - Python 2.7 with setuptools 23 | - The other Python libraries are installed automatically by the setup.py script. 24 | The GeoIP library needs the libgeoip development headers. On a Debian system, 25 | install them by running `apt-get install libgeoip-dev`. To install the 26 | free GeoIP database, run `apt-get install geoip-database`. 27 | Note: To install the Python GeoIP library on FreeBSD, edit your 28 | `~/.pydistutils.cfg` to contain the following: 29 | ``` 30 | [build_ext] 31 | include_dirs=/usr/local/include 32 | library_dirs=/usr/local/lib 33 | ``` 34 | - `denyhosts-server` is developed and tested on a Debian GNU/Linux system. It should 35 | work on any Linux system with Python. Microsoft Windows is not a supported 36 | platform, although it should work without major modifications. 37 | - On most installations the sqlite3 Python library comes with Python 2.7. If 38 | not, you need to install it manually, possibly with using pip: 39 | `pip install pysqlite` or, on Debian/Ubuntu, `apt-get install python-pysqlite2`. 40 | - If you use a MySQL database, you need to install the appropriate Python 41 | library. possibly by running `pip install MySQL-python`. On Debian/Ubuntu, 42 | use `apt-get install python-mysqldb`. 43 | 44 | ## Installation 45 | Run the following command: `sudo ./setup.py minify_js minify_css install`. This will download the 46 | needed Python libraries, minify the used JavaScript and CSS libraries, install the Python scripts 47 | onto your system (usually in `/usr/local/lib/python2.7/dist-packages`), install the default configuration 48 | file in `/etc/denyhosts-server.conf` and the Python script 49 | `/usr/local/bin/denyhosts-server`. 50 | 51 | ## Configuration 52 | Create the database and a database user with full rights to it. Copy the 53 | `denyhosts-server.conf.example` file to `/etc/denyhosts-server.conf` and edit it. 54 | Fill in the database parameters, the location of the log file (which should be 55 | writable by the system user that will be running denyhosts-server) and 56 | other settings you wish to change. `graph_dir` in the `stats` sections is 57 | another location that should be writable by `denyhosts-server`. 58 | 59 | Prepare the database for first use with the command `denyhosts-server 60 | --recreate-database`. This will create the tables needed by denyhosts-server. 61 | 62 | ## Running denyhosts-server 63 | Simply run `denyhosts-server`. Unless there are unexpected errors, this will give no 64 | output and the server will just keep running. Configure your DenyHosts clients 65 | to use the new synchronisation server by setting 66 | ``` 67 | SYNC_SERVER=http://your.host.name:9911 68 | ``` 69 | Once the server is running, you can watch the statistics page at 70 | `http://your.host.name:9911` 71 | 72 | These URLs look the same, but the xmlrpc library from the `DenyHosts` 73 | client will actually connect to `http://your.host.name:9911/RPC2`. The port 74 | numbers are configurable. The statistics page can be on the same port as 75 | the RPC server, or on another. 76 | 77 | ## Signals 78 | When `denyhosts-server` receives the `SIGHUP` signal, it will re-read the 79 | configuration file. Changes to the database configuration are ignored. 80 | 81 | ## Updates 82 | Installing the new version of `denyhosts-server` with `./setup.py install`. 83 | Edit the configuration file at `/etc/denyhosts-server.conf` to configure any new 84 | feature added since the last release. Check `changelog.txt` for new 85 | configuration items. 86 | 87 | Stop denyhosts-server, update the database tables by running `denyhosts-server --evolve-database` and 88 | restart denyhosts-server. 89 | 90 | ## Maintenance 91 | Old reports will be automatically purged by the configurable maintenance job. 92 | See the configuration file for configuration options. To clean all reports by 93 | clients, use the `--purge-reported-addresses` command line option. To clean all 94 | reports retrieved from the legacy sync server, use the 95 | `--purge-legacy-addresses` command line option. To purge a specific IP address 96 | from both the reported and the legacy host lists, use the `--purge-ip` command 97 | line option. 98 | 99 | Note: Use these options with care. Do not use them while `denyhosts-server` is 100 | running, since this may cause database inconsistencies. Use the `--force` 101 | command line options to skip the safety prompt when using the purge options. 102 | 103 | ## Links 104 | - [`denyhosts-server` project site](https://github.com/janpascal/denyhosts_sync) 105 | - [`denyhosts` project site](https://github.com/denyhosts/denyhosts) 106 | - [Information on synchronisation algorithm](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=622697) 107 | - [Original, seemingly abandoned `DenyHosts` project](http://www.denyhosts.net) 108 | 109 | ## Copyright and license 110 | 111 | ### denyhosts-server 112 | Copyright (C) 2015 Jan-Pascal van Best 113 | 114 | This program is free software: you can redistribute it and/or modify 115 | it under the terms of the GNU Affero General Public License as published 116 | by the Free Software Foundation, either version 3 of the License, or 117 | (at your option) any later version. 118 | 119 | This program is distributed in the hope that it will be useful, 120 | but WITHOUT ANY WARRANTY; without even the implied warranty of 121 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 122 | GNU Affero General Public License for more details. 123 | 124 | You should have received a copy of the GNU Affero General Public License 125 | along with this program. If not, see . 126 | 127 | ### Synchronisation algorithm 128 | The synchronisation algorithm implemented in denyhosts-server is based 129 | on an article by Anne Bezemer, published as Debian bug#622697 and 130 | archived at [Debian bug#622697](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=622697) 131 | The article is Copyright (C) 2011 J.A. Bezemer 132 | and licensed "either GPL >=3 or AGPL >=3, at your option". 133 | -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- 1 | # denyhosts sync server 2 | # Copyright (C) 2015 Jan-Pascal van Best 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published 6 | # by the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import time 18 | 19 | from denyhosts_server import models 20 | from denyhosts_server import controllers 21 | from denyhosts_server.models import Cracker, Report 22 | 23 | from twisted.internet.defer import inlineCallbacks, returnValue 24 | 25 | import base 26 | 27 | class ModelsTest(base.TestBase): 28 | 29 | @inlineCallbacks 30 | def test_add_cracker(self): 31 | cracker_ip = "127.0.0.1" 32 | now = time.time() 33 | cracker = Cracker(ip_address=cracker_ip, first_time=now, latest_time=now, total_reports=0, current_reports=0, resiliency=0) 34 | cracker = yield cracker.save() 35 | 36 | c2 = yield controllers.get_cracker(cracker_ip) 37 | yield self.assertIsNotNone(c2) 38 | self.assertEqual(c2.ip_address, cracker.ip_address, "Save and re-fetch cracker from database") 39 | self.assertEqual(c2.id, cracker.id, "Save and re-fetch cracker from database") 40 | self.assertEqual(c2.first_time, cracker.first_time, "Save and re-fetch cracker from database") 41 | self.assertEqual(c2.latest_time, cracker.latest_time, "Save and re-fetch cracker from database") 42 | self.assertEqual(c2.total_reports, cracker.total_reports, "Save and re-fetch cracker from database") 43 | self.assertEqual(c2.current_reports, cracker.current_reports, "Save and re-fetch cracker from database") 44 | self.assertEqual(c2.id, cracker.id, "Save and re-fetch cracker from database") 45 | 46 | @inlineCallbacks 47 | def test_add_report(self): 48 | now = time.time() 49 | yield Cracker(ip_address="192.168.1.1", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 50 | c = yield controllers.get_cracker("192.168.1.1") 51 | yield self.assertIsNotNone(c) 52 | yield controllers.add_report_to_cracker(c, "127.0.0.1") 53 | 54 | r = yield Report.find(where=["cracker_id=? and ip_address=?",c.id,"127.0.0.1"], limit=1) 55 | returnValue(self.assertIsNotNone(r, "Added report is in database")) 56 | 57 | @inlineCallbacks 58 | def test_add_multiple_reports(self): 59 | now = time.time() 60 | 61 | yield Cracker(ip_address="192.168.1.1", first_time=now, latest_time=now, total_reports=0, current_reports=0).save() 62 | c = yield controllers.get_cracker("192.168.1.1") 63 | yield self.assertIsNotNone(c) 64 | yield controllers.add_report_to_cracker(c, "127.0.0.1", now) 65 | 66 | reports = yield Report.find(where=["cracker_id=? and ip_address=?",c.id,"127.0.0.1"]) 67 | self.assertEqual(len(reports), 1, "First added report is in database") 68 | 69 | # Add second report shortly after first 70 | yield controllers.add_report_to_cracker(c, "127.0.0.1", now+1) 71 | reports = yield Report.find(where=["cracker_id=? and ip_address=?",c.id,"127.0.0.1"]) 72 | self.assertEqual(len(reports), 1, "Second added report should be ignored") 73 | 74 | # Add second report after 24 hours 75 | yield controllers.add_report_to_cracker(c, "127.0.0.1", now+24*3600+1) 76 | reports = yield Report.find(where=["cracker_id=? and ip_address=?",c.id,"127.0.0.1"]) 77 | self.assertEqual(len(reports), 2, "Second report should be added after 24 hours ") 78 | 79 | # Add third report shortly after second, should be ignored 80 | yield controllers.add_report_to_cracker(c, "127.0.0.1", now+24*3600+10) 81 | reports = yield Report.find(where=["cracker_id=? and ip_address=?",c.id,"127.0.0.1"]) 82 | self.assertEqual(len(reports), 2, "Third report shortly after second should be ignored") 83 | 84 | # Add third report after again 24 hours 85 | yield controllers.add_report_to_cracker(c, "127.0.0.1", now+2*24*3600+20) 86 | reports = yield Report.find(where=["cracker_id=? and ip_address=?",c.id,"127.0.0.1"]) 87 | self.assertEqual(len(reports), 3, "Third report after again 24 hours, should be added") 88 | 89 | # Add fourth report 90 | time_added = now + 2*24*3600 + 30 91 | yield controllers.add_report_to_cracker(c, "127.0.0.1", time_added) 92 | reports = yield Report.find( 93 | where=["cracker_id=? and ip_address=?",c.id,"127.0.0.1"], 94 | orderby='latest_report_time asc') 95 | self.assertEqual(len(reports), 3, "Fourth report, should be merged") 96 | self.assertEqual(reports[-1].latest_report_time, time_added, "Latest report time should be updated") 97 | 98 | self.assertEquals(c.current_reports, 1, "Only one unique reporter should be counted") 99 | self.assertEquals(c.total_reports, 6, "Cracker reported six timed in total") 100 | 101 | # Perform maintenance, expire original report 102 | yield controllers.perform_maintenance(limit = now+1) 103 | reports = yield Report.find(where=["cracker_id=? and ip_address=?",c.id,"127.0.0.1"]) 104 | self.assertEqual(len(reports), 2, "Maintenance should remove oldest report") 105 | yield c.refresh() 106 | self.assertEqual(c.current_reports, 1, "Maintenance should still leave one unique reporter") 107 | 108 | # Perform maintenance, expire second report 109 | yield controllers.perform_maintenance(limit = now+24*3600+11) 110 | reports = yield Report.find(where=["cracker_id=? and ip_address=?",c.id,"127.0.0.1"]) 111 | self.assertEqual(len(reports), 1, "Maintenance should remove one more report") 112 | yield c.refresh() 113 | self.assertEqual(c.current_reports, 1, "Maintenance should still leave one unique reporter") 114 | 115 | # Perform maintenance again, expire last report and cracker 116 | yield controllers.perform_maintenance(limit = now+2*24*3600+31) 117 | reports = yield Report.find(where=["cracker_id=? and ip_address=?",c.id,"127.0.0.1"]) 118 | self.assertEqual(len(reports), 0, "Maintenance should remove last report") 119 | cracker = yield controllers.get_cracker("192.168.1.1") 120 | self.assertIsNone(cracker, "Maintenance should remove cracker") 121 | 122 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 123 | -------------------------------------------------------------------------------- /denyhosts_server/database.py: -------------------------------------------------------------------------------- 1 | # denyhosts sync server 2 | # Copyright (C) 2015 Jan-Pascal van Best 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published 6 | # by the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import logging 18 | import datetime 19 | 20 | from twistar.registry import Registry 21 | from twisted.internet.defer import inlineCallbacks, returnValue 22 | 23 | import GeoIP 24 | 25 | import config 26 | import stats 27 | 28 | def _remove_tables(txn): 29 | print("Removing all data from database and removing tables") 30 | txn.execute("DROP TABLE IF EXISTS info") 31 | txn.execute("DROP TABLE IF EXISTS crackers") 32 | txn.execute("DROP TABLE IF EXISTS reports") 33 | txn.execute("DROP TABLE IF EXISTS legacy") 34 | txn.execute("DROP TABLE IF EXISTS history") 35 | txn.execute("DROP TABLE IF EXISTS country_history") 36 | 37 | def _evolve_database_initial(txn, dbtype): 38 | if dbtype=="sqlite3": 39 | autoincrement="AUTOINCREMENT" 40 | elif dbtype=="MySQLdb": 41 | autoincrement="AUTO_INCREMENT" 42 | 43 | txn.execute("""CREATE TABLE crackers ( 44 | id INTEGER PRIMARY KEY {}, 45 | ip_address CHAR(15), 46 | first_time INTEGER, 47 | latest_time INTEGER, 48 | total_reports INTEGER, 49 | current_reports INTEGER 50 | )""".format(autoincrement)) 51 | txn.execute("CREATE UNIQUE INDEX cracker_ip_address ON crackers (ip_address)") 52 | 53 | txn.execute("""CREATE TABLE reports( 54 | id INTEGER PRIMARY KEY {}, 55 | cracker_id INTEGER, 56 | ip_address CHAR(15), 57 | first_report_time INTEGER, 58 | latest_report_time INTEGER 59 | )""".format(autoincrement)) 60 | txn.execute("CREATE INDEX report_first_time ON reports (first_report_time)") 61 | txn.execute("CREATE UNIQUE INDEX report_cracker_ip ON reports (cracker_id, ip_address)") 62 | txn.execute("CREATE INDEX report_cracker_first ON reports (cracker_id, first_report_time)") 63 | 64 | txn.execute("""CREATE TABLE legacy( 65 | id INTEGER PRIMARY KEY {}, 66 | ip_address CHAR(15), 67 | retrieved_time INTEGER 68 | )""".format(autoincrement)) 69 | txn.execute("CREATE UNIQUE INDEX legacy_ip ON legacy (ip_address)") 70 | txn.execute("CREATE INDEX legacy_retrieved ON legacy (retrieved_time)") 71 | 72 | def _evolve_database_v1(txn, dbtype): 73 | txn.execute("""CREATE TABLE info ( 74 | `key` CHAR(32) PRIMARY KEY, 75 | `value` VARCHAR(255) 76 | )""") 77 | if dbtype=="sqlite3": 78 | txn.execute('INSERT INTO info VALUES ("schema_version", ?)', (str(_schema_version),)) 79 | elif dbtype=="MySQLdb": 80 | txn.execute('INSERT INTO info VALUES ("schema_version", %s)', (str(_schema_version),)) 81 | txn.execute('INSERT INTO info VALUES ("last_legacy_sync", 0)') 82 | 83 | def _evolve_database_v2(txn, dbtype): 84 | txn.execute("ALTER TABLE crackers ADD resiliency INTEGER") 85 | txn.execute("CREATE INDEX cracker_qual ON crackers (current_reports, resiliency, latest_time, first_time)") 86 | txn.execute("CREATE INDEX cracker_first ON crackers (first_time)") 87 | txn.execute("UPDATE crackers SET resiliency=latest_time-first_time") 88 | 89 | def _evolve_database_v3(txn, dbtype): 90 | if dbtype=="sqlite3": 91 | txn.execute("DROP INDEX cracker_qual") 92 | elif dbtype=="MySQLdb": 93 | txn.execute("ALTER TABLE crackers DROP INDEX cracker_qual") 94 | txn.execute("CREATE INDEX cracker_qual ON crackers (latest_time, current_reports, resiliency, first_time)") 95 | 96 | def _evolve_database_v4(txn, dbtype): 97 | txn.execute("CREATE INDEX report_latest ON reports (latest_report_time)") 98 | 99 | def _evolve_database_v5(txn, dbtype): 100 | if dbtype=="sqlite3": 101 | txn.execute("DROP INDEX report_cracker_ip") 102 | elif dbtype=="MySQLdb": 103 | txn.execute("ALTER TABLE reports DROP INDEX report_cracker_ip") 104 | txn.execute("CREATE INDEX report_cracker_ip ON reports (cracker_id, ip_address, latest_report_time)") 105 | 106 | def _evolve_database_v6(txn, dbtype): 107 | # Remove crackers without reports from database. This may have occured 108 | # because of a bug in controllers.perform_maintenance() 109 | txn.execute(""" 110 | DELETE FROM crackers 111 | WHERE id NOT IN 112 | ( SELECT cracker_id FROM reports ) 113 | """) 114 | 115 | def _evolve_database_v7(txn, dbtype): 116 | txn.execute("""CREATE TABLE history ( 117 | `date` DATE PRIMARY KEY, 118 | num_reports INTEGER, 119 | num_contributors INTEGER, 120 | num_reported_hosts INTEGER 121 | )""") 122 | 123 | stats.update_recent_history_txn(txn) 124 | 125 | def _evolve_database_v8(txn, dbtype): 126 | txn.execute("""CREATE TABLE country_history ( 127 | country_code CHAR(5) PRIMARY KEY, 128 | country VARCHAR(50), 129 | num_reports INTEGER 130 | )""") 131 | txn.execute("CREATE INDEX country_history_count ON country_history(num_reports)") 132 | txn.execute('INSERT INTO `info` VALUES ("last_country_history_update", "1900-01-01")') 133 | 134 | print("Calculating per-country totals...") 135 | stats.update_country_history_txn(txn, None, include_history=True) 136 | 137 | print("Fixing up historical data...") 138 | stats.fixup_history_txn(txn) 139 | 140 | _evolutions = { 141 | 1: _evolve_database_v1, 142 | 2: _evolve_database_v2, 143 | 3: _evolve_database_v3, 144 | 4: _evolve_database_v4, 145 | 5: _evolve_database_v5, 146 | 6: _evolve_database_v6, 147 | 7: _evolve_database_v7, 148 | 8: _evolve_database_v8 149 | } 150 | 151 | _schema_version = len(_evolutions) 152 | 153 | def _evolve_database(txn): 154 | print("Evolving database") 155 | dbtype = config.dbtype 156 | 157 | try: 158 | txn.execute('SELECT `value` FROM info WHERE `key`="schema_version"') 159 | result = txn.fetchone() 160 | if result is not None: 161 | current_version = int(result[0]) 162 | else: 163 | print("No schema version in database") 164 | _evolve_database_initial(txn, dbtype) 165 | current_version = 0 166 | except: 167 | print("No schema version in database") 168 | _evolve_database_initial(txn, dbtype) 169 | current_version = 0 170 | 171 | if current_version > _schema_version: 172 | print("Illegal database schema {}".format(current_version)) 173 | return 174 | 175 | print("Current database schema is version {}".format(current_version)) 176 | 177 | while current_version < _schema_version: 178 | current_version += 1 179 | print("Evolving database to version {}...".format(current_version)) 180 | _evolutions[current_version](txn, dbtype) 181 | 182 | if dbtype=="sqlite3": 183 | txn.execute('UPDATE info SET `value`=? WHERE `key`="schema_version"', (str(current_version),)) 184 | elif dbtype=="MySQLdb": 185 | txn.execute('UPDATE info SET `value`=%s WHERE `key`="schema_version"', (str(current_version),)) 186 | 187 | print("Updated database schema, current version is {}".format(_schema_version)) 188 | 189 | def evolve_database(): 190 | return Registry.DBPOOL.runInteraction(_evolve_database) 191 | 192 | @inlineCallbacks 193 | def clean_database(): 194 | yield Registry.DBPOOL.runInteraction(_remove_tables) 195 | yield Registry.DBPOOL.runInteraction(_evolve_database) 196 | 197 | @inlineCallbacks 198 | def check_database_version(): 199 | try: 200 | rows = yield Registry.DBPOOL.runQuery('SELECT `value` FROM `info` WHERE `key`="schema_version"') 201 | if rows is not None: 202 | current_version = int(rows[0][0]) 203 | else: 204 | print("No schema version in database") 205 | current_version = 0 206 | except: 207 | current_version = 0 208 | 209 | if current_version != _schema_version: 210 | logging.debug("Wrong database schema {}, expecting {}, exiting".format(current_version, _schema_version)) 211 | print("Wrong database schema {}, expecting {}, exiting".format(current_version, _schema_version)) 212 | from twisted.internet import reactor 213 | reactor.stop() 214 | else: 215 | logging.info("Database schema is up to date (version {})".format(current_version)) 216 | returnValue(current_version) 217 | 218 | # FIXME Not the proper way. What if there's a question mark somewhere 219 | # else in the query? 220 | def translate_query(query): 221 | if config.dbtype == "MySQLdb": 222 | return query.replace('?', '%s') 223 | elif config.dbtype == "sqlite3": 224 | return query 225 | else: 226 | print("unsupported database {}".format(config.dbtype)) 227 | return query 228 | 229 | def run_query(query, *args): 230 | return Registry.DBPOOL.runQuery(translate_query(query), args) 231 | 232 | def run_operation(query, *args): 233 | return Registry.DBPOOL.runOperation(translate_query(query), args) 234 | 235 | def run_truncate_query(table): 236 | if config.dbtype == "MySQLdb": 237 | query = "TRUNCATE TABLE `{}`".format(table) 238 | elif config.dbtype == "sqlite3": 239 | query = "DELETE FROM `{}`".format(table) 240 | else: 241 | print("unsupported database {}".format(config.dbtype)) 242 | return Registry.DBPOOL.runQuery(query) 243 | -------------------------------------------------------------------------------- /denyhosts_server/controllers.py: -------------------------------------------------------------------------------- 1 | # denyhosts sync server 2 | # Copyright (C) 2015 Jan-Pascal van Best 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published 6 | # by the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import logging 18 | import time 19 | import xmlrpclib 20 | 21 | from twisted.internet.defer import inlineCallbacks, returnValue 22 | from twisted.internet.threads import deferToThread 23 | 24 | import config 25 | import database 26 | import models 27 | from models import Cracker, Report, Legacy 28 | import utils 29 | 30 | def get_cracker(ip_address): 31 | return Cracker.find(where=["ip_address=?",ip_address], limit=1) 32 | 33 | # Note: lock cracker IP first! 34 | # Report merging algorithm by Anne Bezemer, see 35 | # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=622697 36 | @inlineCallbacks 37 | def add_report_to_cracker(cracker, client_ip, when=None): 38 | if when is None: 39 | when = time.time() 40 | 41 | reports = yield Report.find( 42 | where=["cracker_id=? AND ip_address=?", cracker.id, client_ip], 43 | orderby='latest_report_time ASC' 44 | ) 45 | if len(reports) == 0: 46 | report = Report(ip_address=client_ip, first_report_time=when, latest_report_time=when) 47 | yield report.save() 48 | cracker.current_reports += 1 49 | yield report.cracker.set(cracker) 50 | elif len(reports) == 1: 51 | report = reports[0] 52 | # Add second report after 24 hours 53 | if when > report.latest_report_time + 24*3600: 54 | report = Report(ip_address=client_ip, first_report_time=when, latest_report_time=when) 55 | yield report.save() 56 | yield report.cracker.set(cracker) 57 | elif len(reports) == 2: 58 | latest_report = reports[1] 59 | # Add third report after again 24 hours 60 | if when > latest_report.latest_report_time + 24*3600: 61 | report = Report(ip_address=client_ip, first_report_time=when, latest_report_time=when) 62 | yield report.save() 63 | yield report.cracker.set(cracker) 64 | else: 65 | latest_report = reports[-1] 66 | latest_report.latest_report_time = when 67 | yield latest_report.save() 68 | 69 | cracker.total_reports += 1 70 | cracker.latest_time = when 71 | cracker.resiliency = when - cracker.first_time 72 | 73 | yield cracker.save() 74 | 75 | @inlineCallbacks 76 | def get_qualifying_crackers(min_reports, min_resilience, previous_timestamp, 77 | max_crackers, latest_added_hosts): 78 | # Thank to Anne Bezemer for the algorithm in this function. 79 | # See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=622697 80 | 81 | # This query takes care of conditions (a) and (b) 82 | # cracker_ids = yield database.runGetPossibleQualifyingCrackerQuery(min_reports, min_resilience, previous_timestamp) 83 | cracker_ids = yield database.run_query(""" 84 | SELECT DISTINCT c.id, c.ip_address 85 | FROM crackers c 86 | WHERE (c.current_reports >= ?) 87 | AND (c.resiliency >= ?) 88 | AND (c.latest_time >= ?) 89 | ORDER BY c.first_time DESC 90 | """, min_reports, min_resilience, previous_timestamp) 91 | 92 | if cracker_ids is None: 93 | returnValue([]) 94 | 95 | # Now look for conditions (c) and (d) 96 | result = [] 97 | for c in cracker_ids: 98 | cracker_id = c[0] 99 | if c[1] in latest_added_hosts: 100 | logging.debug("Skipping {}, just reported by client".format(c[1])) 101 | continue 102 | cracker = yield Cracker.find(cracker_id) 103 | if cracker is None: 104 | continue 105 | logging.debug("Examining cracker:") 106 | logging.debug(cracker) 107 | reports = yield cracker.reports.get(orderby="first_report_time ASC") 108 | #logging.debug("reports:") 109 | #for r in reports: 110 | # logging.debug(" "+str(r)) 111 | logging.debug("r[m-1].first, prev: {}, {}".format(reports[min_reports-1].first_report_time, previous_timestamp)) 112 | if (len(reports)>=min_reports and 113 | reports[min_reports-1].first_report_time >= previous_timestamp): 114 | # condition (c) satisfied 115 | logging.debug("c") 116 | result.append(cracker.ip_address) 117 | else: 118 | logging.debug("checking (d)...") 119 | satisfied = False 120 | for report in reports: 121 | #logging.debug(" "+str(report)) 122 | if (not satisfied and 123 | report.latest_report_time>=previous_timestamp and 124 | report.latest_report_time-cracker.first_time>=min_resilience): 125 | logging.debug(" d1") 126 | satisfied = True 127 | if (report.latest_report_time<=previous_timestamp and 128 | report.latest_report_time-cracker.first_time>=min_resilience): 129 | logging.debug(" d2 failed") 130 | satisfied = False 131 | break 132 | if satisfied: 133 | logging.debug("Appending {}".format(cracker.ip_address)) 134 | result.append(cracker.ip_address) 135 | else: 136 | logging.debug(" skipping") 137 | if len(result)>=max_crackers: 138 | break 139 | 140 | if len(result) < max_crackers: 141 | # Add results from legacy server 142 | extras = yield Legacy.find(where=["retrieved_time>?", previous_timestamp], 143 | orderby="retrieved_time DESC", limit=max_crackers-len(result)) 144 | result = result + [extra.ip_address for extra in extras] 145 | 146 | logging.debug("Returning {} hosts".format(len(result))) 147 | returnValue(result) 148 | 149 | # Periodical database maintenance 150 | # From algorithm by Anne Bezemer, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=622697 151 | # Expiry/maintenance every hour/day: 152 | # remove reports with .latestreporttime older than (for example) 1 month 153 | # and only update cracker.currentreports 154 | # remove reports that were reported by what we now "reliably" know to 155 | # be crackers themselves 156 | # remove crackers that have no reports left 157 | 158 | # TODO remove reports by identified crackers 159 | 160 | @inlineCallbacks 161 | def perform_maintenance(limit = None, legacy_limit = None): 162 | logging.info("Starting maintenance job...") 163 | 164 | if limit is None: 165 | now = time.time() 166 | limit = now - config.expiry_days * 24 * 3600 167 | 168 | if legacy_limit is None: 169 | now = time.time() 170 | legacy_limit = now - config.legacy_expiry_days * 24 * 3600 171 | 172 | reports_deleted = 0 173 | crackers_deleted = 0 174 | legacy_deleted = 0 175 | 176 | batch_size = 1000 177 | 178 | while True: 179 | old_reports = yield Report.find(where=["latest_report_time 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published 6 | # by the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import argparse 18 | import logging 19 | import signal 20 | import sys 21 | import ConfigParser 22 | 23 | from twisted.web import server, resource, static 24 | from twisted.enterprise import adbapi 25 | from twisted.internet import task, reactor 26 | from twisted.internet.defer import inlineCallbacks, returnValue 27 | from twisted.python import log 28 | 29 | from twistar.registry import Registry 30 | 31 | import views 32 | import debug_views 33 | import models 34 | import controllers 35 | import config 36 | import database 37 | import stats 38 | import utils 39 | 40 | import __init__ 41 | 42 | def stop_reactor(value): 43 | print(value) 44 | reactor.stop() 45 | 46 | def sighup_handler(signum, frame): 47 | global configfile 48 | global main_xmlrpc_handler 49 | 50 | logging.warning("Received SIGHUP, reloading configuration file...") 51 | debug_was_on = config.enable_debug_methods 52 | old_xmlrpc_listen_port = config.xmlrpc_listen_port 53 | old_stats_listen_port = config.stats_listen_port 54 | config.read_config(configfile) 55 | 56 | configure_logging() 57 | schedule_jobs() 58 | 59 | if debug_was_on and not config.enable_debug_methods: 60 | # Remove debug methods 61 | # Missing API in class XMLRPC 62 | del main_xmlrpc_handler.subHandlers["debug"] 63 | 64 | if config.enable_debug_methods and not debug_was_on: 65 | d = debug_views.DebugServer(main_xmlrpc_handler) 66 | main_xmlrpc_handler.putSubHandler('debug', d) 67 | 68 | stop_listening().addCallback(lambda _: start_listening()) 69 | 70 | @inlineCallbacks 71 | def shutdown(): 72 | global main_xmlrpc_handler 73 | global _xmlrpc_listener 74 | global _xmlrpc_site 75 | try: 76 | site = _xmlrpc_site 77 | logging.info("shutting down, first closing listening ports...") 78 | print("Shutting down, hold on a moment...") 79 | yield stop_listening() 80 | 81 | # This doesn't work, site.session is always empty 82 | logging.info("Ports closed, waiting for current sessions to close...") 83 | logging.debug("Clients still connected: {}".format(len(site.sessions))) 84 | while not len(site.sessions)==0: 85 | logging.debug("Waiting, {} sessions still active".format(len(site.sessions))) 86 | yield task.deferLater(reactor, 1, lambda _:0, 0) 87 | 88 | logging.info("No more sessions, waiting for locked hosts...") 89 | while not utils.none_waiting(): 90 | logging.info("Waiting to shut down, {} hosts still blocked".format(utils.count_waiting())) 91 | yield task.deferLater(reactor, 1, lambda _:0, 0) 92 | logging.debug("reactor.getDelayedCalls: {}".format([c.func for c in reactor.getDelayedCalls()])) 93 | 94 | logging.info("All hosts unlocked, waiting 3 more seconds...") 95 | yield task.deferLater(reactor, 1, lambda _:0, 0) 96 | logging.debug("Waiting 2 more seconds...") 97 | yield task.deferLater(reactor, 1, lambda _:0, 0) 98 | logging.debug("Waiting 1 more second...") 99 | yield task.deferLater(reactor, 1, lambda _:0, 0) 100 | logging.info("Continuing shutdown") 101 | except: 102 | logging.exception("Error in shutdown callback") 103 | 104 | _xmlrpc_listener = None 105 | _stats_listener = None 106 | _xmlrpc_site = None 107 | 108 | # Returns a callback. Wait on it before the port(s) are actually closed 109 | def stop_listening(): 110 | logging.debug("main.stop_listening()") 111 | global _xmlrpc_listener 112 | global _stats_listener 113 | global _xmlrpc_site 114 | 115 | # It's not easy to actually close a listening port. 116 | # You need to close both the port and the protocol, 117 | # and wait for them 118 | if _xmlrpc_listener is not None: 119 | deferred = _xmlrpc_listener.stopListening() 120 | deferred.addCallback(_xmlrpc_listener.loseConnection) 121 | else: 122 | deferred = Deferred() 123 | 124 | if _stats_listener is not None: 125 | deferred.addCallback(_stats_listener.stopListening) 126 | deferred.addCallback(_stats_listener.loseConnection) 127 | 128 | _xmlrpc_listener = None 129 | _stats_listener = None 130 | _xmlrpc_site = None 131 | 132 | return deferred 133 | 134 | def start_listening(): 135 | logging.debug("main.start_listening()") 136 | global _xmlrpc_listener 137 | global _xmlrpc_site 138 | global _stats_listener 139 | global main_xmlrpc_handler 140 | 141 | # Configure web resources 142 | main_xmlrpc_handler = views.Server() 143 | stats_resource = views.WebResource() 144 | web_static = static.File(config.static_dir) 145 | static.File.contentTypes['.svg'] = 'image/svg+xml' 146 | web_graphs = static.File(config.graph_dir) 147 | 148 | # Roots 149 | if config.stats_listen_port == config.xmlrpc_listen_port: 150 | xmlrpc_root = stats_resource 151 | else: 152 | xmlrpc_root = resource.Resource() 153 | stats_root = stats_resource 154 | 155 | # /RPC2 156 | xmlrpc_root.putChild('RPC2', main_xmlrpc_handler) 157 | 158 | # xmlrpc debug handler 159 | if config.enable_debug_methods: 160 | d = debug_views.DebugServer(main_xmlrpc_handler) 161 | main_xmlrpc_handler.putSubHandler('debug', d) 162 | 163 | # /static 164 | stats_root.putChild('static', web_static) 165 | # /static/graph 166 | web_static.putChild('graph', web_graphs) 167 | 168 | logging.info("Start listening on host:port {}:{}".format(config.xmlrpc_listen_host, config.xmlrpc_listen_port)) 169 | _xmlrpc_site = server.Site(xmlrpc_root) 170 | _xmlrpc_listener = reactor.listenTCP(config.xmlrpc_listen_port, _xmlrpc_site, interface=config.xmlrpc_listen_host) 171 | 172 | if config.stats_listen_port == config.xmlrpc_listen_port and config.xmlrpc_listen_host == config.stats_listen_host: 173 | _stats_listener = None 174 | else: 175 | logging.info("Start serving statistics on host:port {}:{}".format(config.stats_listen_host, config.stats_listen_port)) 176 | _stats_listener = reactor.listenTCP(config.stats_listen_port, server.Site(stats_root), interface=config.stats_listen_host) 177 | 178 | maintenance_job = None 179 | legacy_sync_job = None 180 | stats_job = None 181 | 182 | def schedule_jobs(): 183 | global maintenance_job, legacy_sync_job, stats_job 184 | 185 | # Reschedule maintenance job 186 | if maintenance_job is not None: 187 | maintenance_job.stop() 188 | maintenance_job = task.LoopingCall(controllers.perform_maintenance) 189 | maintenance_job.start(config.maintenance_interval, now=False) 190 | 191 | # Reschedule legacy sync job 192 | if legacy_sync_job is not None: 193 | legacy_sync_job.stop() 194 | legacy_sync_job = task.LoopingCall(controllers.download_from_legacy_server) 195 | legacy_sync_job.start(config.legacy_frequency, now=False) 196 | 197 | # Reschedule stats job 198 | if stats_job is not None: 199 | stats_job.stop() 200 | stats_job = task.LoopingCall(stats.update_stats_cache) 201 | stats_job.start(config.stats_frequency, now=True) 202 | 203 | def configure_logging(): 204 | # Remove all handlers associated with the root logger object. 205 | for handler in logging.root.handlers[:]: 206 | logging.root.removeHandler(handler) 207 | 208 | # Use basic configuration 209 | logging.basicConfig(filename=config.logfile, 210 | level=config.loglevel, 211 | format="%(asctime)s %(module)-8s %(levelname)-8s %(message)s", 212 | datefmt="%Y-%m-%d %H:%M:%S") 213 | 214 | # Collect Twisted log messages in Python logging system 215 | observer = log.PythonLoggingObserver() 216 | observer.start() 217 | 218 | def run_main(): 219 | global configfile 220 | global maintenance_job, legacy_sync_job 221 | global main_xmlrpc_handler, stats_resource, web_root, web_static 222 | 223 | parser = argparse.ArgumentParser(description="DenyHosts sync server") 224 | parser.add_argument("-c", "--config", default="/etc/denyhosts-server.conf", help="Configuration file") 225 | parser.add_argument("--recreate-database", action='store_true', help="Wipe and recreate the database") 226 | parser.add_argument("--evolve-database", action='store_true', help="Evolve the database to the latest schema version") 227 | parser.add_argument("--purge-legacy-addresses", action='store_true', 228 | help="Purge all hosts downloaded from the legacy server. DO NOT USE WHEN DENYHOSTS-SERVER IS RUNNING!") 229 | parser.add_argument("--purge-reported-addresses", action='store_true', 230 | help="Purge all hosts that have been reported by clients. DO NOT USE WHEN DENYHOSTS-SERVER IS RUNNING!") 231 | parser.add_argument("--purge-ip", action='store', 232 | help="Purge ip address from both legacy and reported host lists. DO NOT USE WHEN DENYHOSTS-SERVER IS RUNNING!") 233 | parser.add_argument("-f", "--force", action='store_true', 234 | help="Do not ask for confirmation, execute action immediately") 235 | args = parser.parse_args() 236 | 237 | configfile = args.config 238 | 239 | try: 240 | config.read_config(args.config) 241 | except ConfigParser.NoSectionError, e: 242 | print("Error in reading the configuration file from \"{}\": {}.".format(args.config, e)) 243 | print("Please review the configuration file. Look at the supplied denyhosts-server.conf.example for more information.") 244 | sys.exit() 245 | 246 | configure_logging() 247 | 248 | Registry.DBPOOL = adbapi.ConnectionPool(config.dbtype, **config.dbparams) 249 | Registry.register(models.Cracker, models.Report, models.Legacy) 250 | 251 | single_shot = False 252 | 253 | if not args.force and (args.recreate_database 254 | or args.evolve_database 255 | or args.purge_legacy_addresses 256 | or args.purge_reported_addresses 257 | or args.recreate_database 258 | or args.purge_ip is not None): 259 | print("WARNING: do not run this method when denyhosts-server is running.") 260 | reply = raw_input("Are you sure you want to continue (Y/N): ") 261 | if not reply.upper().startswith('Y'): 262 | sys.exit() 263 | 264 | if args.recreate_database: 265 | single_shot = True 266 | database.clean_database().addCallbacks(stop_reactor, stop_reactor) 267 | 268 | if args.evolve_database: 269 | single_shot = True 270 | database.evolve_database().addCallbacks(stop_reactor, stop_reactor) 271 | 272 | if args.purge_legacy_addresses: 273 | single_shot = True 274 | controllers.purge_legacy_addresses().addCallbacks(stop_reactor, stop_reactor) 275 | 276 | if args.purge_reported_addresses: 277 | single_shot = True 278 | controllers.purge_reported_addresses().addCallbacks(stop_reactor, stop_reactor) 279 | 280 | if args.purge_ip is not None: 281 | single_shot = True 282 | controllers.purge_ip(args.purge_ip).addCallbacks(stop_reactor, stop_reactor) 283 | 284 | if not single_shot: 285 | signal.signal(signal.SIGHUP, sighup_handler) 286 | reactor.addSystemEventTrigger("after", "startup", database.check_database_version) 287 | reactor.addSystemEventTrigger("before", "shutdown", shutdown) 288 | 289 | start_listening() 290 | 291 | # Set up maintenance and legacy sync jobs 292 | schedule_jobs() 293 | 294 | # Start reactor 295 | logging.info("Starting denyhosts-server version {}".format(__init__.version)) 296 | reactor.run() 297 | 298 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 299 | -------------------------------------------------------------------------------- /denyhosts_server/stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # denyhosts sync server 4 | # Copyright (C) 2015 Jan-Pascal van Best 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Affero General Public License as published 8 | # by the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Affero General Public License for more details. 15 | 16 | # You should have received a copy of the GNU Affero General Public License 17 | # along with this program. If not, see . 18 | 19 | 20 | import config 21 | import datetime 22 | import logging 23 | import os.path 24 | import socket 25 | import time 26 | 27 | from twisted.internet import reactor, threads, task 28 | from twisted.internet.defer import inlineCallbacks, returnValue 29 | from twisted.python import log 30 | from twistar.registry import Registry 31 | 32 | from jinja2 import Template, Environment, FileSystemLoader 33 | 34 | import GeoIP 35 | 36 | import matplotlib 37 | # Prevent errors from matplotlib instantiating a Tk window 38 | matplotlib.use('Agg') 39 | import matplotlib.pyplot as plt 40 | import matplotlib.dates as mdates 41 | import numpy 42 | 43 | import models 44 | import database 45 | import __init__ 46 | 47 | def format_datetime(value, format='medium'): 48 | dt = datetime.datetime.fromtimestamp(value) 49 | if format == 'full': 50 | format="EEEE, d. MMMM y 'at' HH:mm:ss" 51 | elif format == 'medium': 52 | format="%a %d-%m-%Y %H:%M:%S" 53 | return dt.strftime(format) 54 | 55 | def insert_zeroes(rows, max = None): 56 | result = [] 57 | index = 0 58 | if max is None: 59 | max = rows[-1][0] + 1 60 | 61 | for value in xrange(max): 62 | if index < len(rows) and rows[index][0] == value: 63 | result.append(rows[index]) 64 | index += 1 65 | else: 66 | result.append((value,0)) 67 | return result 68 | 69 | def humanize_number(number, pos): 70 | """Return a humanized string representation of a number.""" 71 | abbrevs = ( 72 | (1E15, 'P'), 73 | (1E12, 'T'), 74 | (1E9, 'G'), 75 | (1E6, 'M'), 76 | (1E3, 'k'), 77 | (1, '') 78 | ) 79 | if number < 1000: 80 | return str(number) 81 | for factor, suffix in abbrevs: 82 | if number >= factor: 83 | break 84 | return '%.*f%s' % (0, number / factor, suffix) 85 | 86 | # Functions containing blocking io, call from thread! 87 | def fixup_crackers(hosts): 88 | gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE) 89 | for host in hosts: 90 | try: 91 | host.country = gi.country_name_by_addr(host.ip_address) 92 | except Exception, e: 93 | logging.debug("Exception looking up country for {}: {}".format(host.ip_address, e)) 94 | host.country = '' 95 | try: 96 | if config.stats_resolve_hostnames: 97 | hostinfo = socket.gethostbyaddr(host.ip_address) 98 | host.hostname = hostinfo[0] 99 | else: 100 | host.hostname = host.ip_address 101 | except Exception, e: 102 | logging.debug("Exception looking up reverse DNS for {}: {}".format(host.ip_address, e)) 103 | host.hostname = "-" 104 | 105 | def make_daily_graph(txn): 106 | # Calculate start of daily period: yesterday on the beginning of the 107 | # current hour 108 | now = time.time() 109 | dt_now = datetime.datetime.fromtimestamp(now) 110 | start_hour = dt_now.hour 111 | dt_onthehour = dt_now.replace(minute=0, second=0, microsecond=0) 112 | dt_start = dt_onthehour - datetime.timedelta(days=1) 113 | yesterday = int(dt_start.strftime('%s')) 114 | 115 | txn.execute(database.translate_query(""" 116 | SELECT CAST((first_report_time-?)/3600 AS UNSIGNED INTEGER), count(*) 117 | FROM reports 118 | WHERE first_report_time > ? 119 | GROUP BY CAST((first_report_time-?)/3600 AS UNSIGNED INTEGER) 120 | ORDER BY first_report_time ASC 121 | """), (yesterday, yesterday, yesterday)) 122 | rows = txn.fetchall() 123 | if not rows: 124 | logging.debug("No data for past 24 hours") 125 | rows = [(0,0)] 126 | #logging.debug("Daily: {}".format(rows)) 127 | rows = insert_zeroes(rows, 24) 128 | #logging.debug("Daily: {}".format(rows)) 129 | 130 | x = [dt_start + datetime.timedelta(hours=row[0]) for row in rows] 131 | y = [row[1] for row in rows] 132 | 133 | # calc the trendline 134 | x_num = mdates.date2num(x) 135 | 136 | z = numpy.polyfit(x_num, y, 1) 137 | p = numpy.poly1d(z) 138 | 139 | xx = numpy.linspace(x_num.min(), x_num.max(), 100) 140 | dd = mdates.num2date(xx) 141 | 142 | fig = plt.figure() 143 | ax = fig.gca() 144 | ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M')) 145 | ax.xaxis.set_major_locator(mdates.HourLocator(interval=4)) 146 | ax.yaxis.set_major_formatter(matplotlib.ticker.FuncFormatter(humanize_number)) 147 | ax.set_title("Reports per hour") 148 | ax.plot(x,y, linestyle='solid', marker='o', markerfacecolor='blue') 149 | ax.plot(dd, p(xx), "b--") 150 | ax.set_ybound(lower=0) 151 | fig.autofmt_xdate() 152 | fig.savefig(os.path.join(config.graph_dir, 'hourly.svg')) 153 | fig.clf() 154 | plt.close(fig) 155 | 156 | def make_monthly_graph(txn): 157 | # Calculate start of monthly period: last month on the beginning of the 158 | # current day 159 | today = datetime.date.today() 160 | dt_start = today - datetime.timedelta(weeks=4) 161 | 162 | txn.execute(database.translate_query(""" 163 | SELECT date, num_reports 164 | FROM history 165 | WHERE date >= ? 166 | ORDER BY date ASC 167 | """), (dt_start,)) 168 | rows = txn.fetchall() 169 | if rows is None or len(rows)==0: 170 | return 171 | 172 | (x,y) = zip(*rows) 173 | 174 | # calc the trendline 175 | x_num = mdates.date2num(x) 176 | 177 | z = numpy.polyfit(x_num, y, 1) 178 | p = numpy.poly1d(z) 179 | 180 | xx = numpy.linspace(x_num.min(), x_num.max(), 100) 181 | dd = mdates.num2date(xx) 182 | 183 | fig = plt.figure() 184 | ax = fig.gca() 185 | ax.xaxis.set_major_formatter(mdates.DateFormatter('%d %b')) 186 | ax.xaxis.set_major_locator(mdates.DayLocator(interval=4)) 187 | ax.yaxis.set_major_formatter(matplotlib.ticker.FuncFormatter(humanize_number)) 188 | ax.set_title("Reports per day") 189 | ax.plot(x,y, linestyle='solid', marker='o', markerfacecolor='blue') 190 | ax.plot(dd, p(xx),"b--") 191 | ax.set_ybound(lower=0) 192 | fig.autofmt_xdate() 193 | fig.savefig(os.path.join(config.graph_dir, 'monthly.svg')) 194 | fig.clf() 195 | plt.close(fig) 196 | 197 | def make_history_graph(txn): 198 | # Graph since first record 199 | txn.execute(database.translate_query(""" 200 | SELECT date FROM history 201 | ORDER BY date ASC 202 | LIMIT 1 203 | """)) 204 | first_time = txn.fetchall() 205 | if first_time is not None and len(first_time)>0 and first_time[0][0] is not None: 206 | dt_first = first_time[0][0] 207 | else: 208 | dt_first= datetime.date.today() 209 | num_days = ( datetime.date.today() - dt_first ).days 210 | #logging.debug("First day in data set: {}".format(dt_first)) 211 | #logging.debug("Number of days in data set: {}".format(num_days)) 212 | if num_days == 0: 213 | return 214 | 215 | txn.execute(database.translate_query(""" 216 | SELECT date, num_reports 217 | FROM history 218 | ORDER BY date ASC 219 | """)) 220 | rows = txn.fetchall() 221 | if rows is None or len(rows)==0: 222 | return 223 | 224 | (x,y) = zip(*rows) 225 | 226 | # calc the trendline 227 | x_num = mdates.date2num(x) 228 | 229 | z = numpy.polyfit(x_num, y, 1) 230 | p = numpy.poly1d(z) 231 | 232 | xx = numpy.linspace(x_num.min(), x_num.max(), 100) 233 | dd = mdates.num2date(xx) 234 | 235 | fig = plt.figure() 236 | ax = fig.gca() 237 | 238 | locator = mdates.AutoDateLocator(interval_multiples=False) 239 | ax.xaxis.set_major_locator(locator) 240 | ax.xaxis.set_major_formatter(mdates.AutoDateFormatter(locator)) 241 | ax.yaxis.set_major_formatter(matplotlib.ticker.FuncFormatter(humanize_number)) 242 | ax.set_title("Reports per day") 243 | if (num_days<100): 244 | ax.plot(x,y, linestyle='solid', marker='o', markerfacecolor='blue') 245 | else: 246 | ax.plot(x,y, linestyle='solid', marker='') 247 | ax.plot(dd, p(xx),"b--") 248 | ax.set_ybound(lower=0) 249 | fig.autofmt_xdate() 250 | fig.savefig(os.path.join(config.graph_dir, 'history.svg')) 251 | fig.clf() 252 | plt.close(fig) 253 | 254 | def make_contrib_graph(txn): 255 | # Number of reporters over days 256 | txn.execute(database.translate_query(""" 257 | SELECT date FROM history 258 | ORDER BY date ASC 259 | LIMIT 1 260 | """)) 261 | first_time = txn.fetchall() 262 | if first_time is not None and len(first_time)>0 and first_time[0][0] is not None: 263 | dt_first = first_time[0][0] 264 | else: 265 | dt_first= datetime.date.today() 266 | num_days = ( datetime.date.today() - dt_first ).days 267 | if num_days == 0: 268 | return 269 | 270 | txn.execute(database.translate_query(""" 271 | SELECT date, num_contributors 272 | FROM history 273 | ORDER BY date ASC 274 | """)) 275 | rows = txn.fetchall() 276 | if rows is None or len(rows)==0: 277 | return 278 | 279 | (x,y) = zip(*rows) 280 | 281 | fig = plt.figure() 282 | ax = fig.gca() 283 | locator = mdates.AutoDateLocator(interval_multiples=False) 284 | ax.xaxis.set_major_locator(locator) 285 | ax.xaxis.set_major_formatter(mdates.AutoDateFormatter(locator)) 286 | ax.yaxis.set_major_formatter(matplotlib.ticker.FuncFormatter(humanize_number)) 287 | ax.set_title("Number of contributors") 288 | if (num_days<100): 289 | ax.plot(x,y, linestyle='solid', marker='o', markerfacecolor='blue') 290 | else: 291 | ax.plot(x,y, linestyle='solid', marker='') 292 | ax.set_ybound(lower=0) 293 | fig.autofmt_xdate() 294 | fig.savefig(os.path.join(config.graph_dir, 'contrib.svg')) 295 | fig.clf() 296 | plt.close(fig) 297 | 298 | def make_country_piegraph(txn): 299 | # Total reports per country 300 | limit = 10 # Fixme configurable 301 | txn.execute(database.translate_query(""" 302 | SELECT country, num_reports 303 | FROM country_history 304 | ORDER BY num_reports DESC 305 | LIMIT ? 306 | """),(limit,)) 307 | 308 | rows = txn.fetchall() 309 | if rows is None or len(rows)==0: 310 | return 311 | 312 | (labels,sizes) = zip(*rows) 313 | 314 | fig = plt.figure() 315 | ax = fig.add_subplot(111) 316 | ax.pie(sizes, labels=labels, 317 | autopct='%1.1f%%', shadow=True, startangle=90) 318 | # Set aspect ratio to be equal so that pie is drawn as a circle. 319 | ax.axis('equal') 320 | 321 | fig.savefig(os.path.join(config.graph_dir, 'country_pie.svg')) 322 | fig.clf() 323 | plt.close(fig) 324 | 325 | def make_country_bargraph(txn): 326 | # Total reports per country 327 | limit = 10 # Fixme configurable 328 | 329 | txn.execute(""" 330 | SELECT sum(num_reports) 331 | FROM country_history 332 | """) 333 | 334 | rows = txn.fetchall() 335 | if rows is None or len(rows)==0: 336 | return 337 | total_reports = rows[0][0] 338 | 339 | txn.execute(database.translate_query(""" 340 | SELECT country, num_reports 341 | FROM country_history 342 | ORDER BY num_reports DESC 343 | LIMIT ? 344 | """),(limit,)) 345 | 346 | rows = txn.fetchall() 347 | if rows is None or len(rows)==0: 348 | return 349 | 350 | (countries,counts) = zip(*reversed(rows)) 351 | max_count = max(counts) 352 | 353 | fig = plt.figure() 354 | ax = fig.add_subplot(111) 355 | 356 | y_pos = numpy.arange(len(countries)) 357 | bars = ax.barh(y_pos, counts, align='center', alpha=0.6) 358 | count = 0 359 | for bar in bars: 360 | height = bar.get_height() 361 | ax.text(max_count / 20., bar.get_y() + height / 2., 362 | "{}% {}".format(round(counts[count]*100./total_reports, 1), countries[count] ), 363 | ha='left', va='center') 364 | count += 1 365 | ax.set_yticks(y_pos) 366 | ax.set_yticklabels([]) 367 | ax.set_title('Number of attacks per country of origin (top {})'.format(limit)) 368 | ax.xaxis.set_major_formatter(matplotlib.ticker.FuncFormatter(humanize_number)) 369 | ax.set_ylim(ymin=-1) 370 | fig.tight_layout() 371 | 372 | fig.savefig(os.path.join(config.graph_dir, 'country_bar.svg')) 373 | fig.clf() 374 | plt.close(fig) 375 | 376 | _cache = None 377 | _stats_busy = False 378 | 379 | @inlineCallbacks 380 | def update_stats_cache(): 381 | global _stats_busy 382 | global _cache 383 | if _stats_busy: 384 | logging.debug("Already updating statistics cache, exiting") 385 | returnValue(None) 386 | _stats_busy = True 387 | 388 | logging.debug("Updating statistics cache...") 389 | 390 | # Fill history table for yesterday, when necessary 391 | yield update_recent_history() 392 | yield update_country_history() 393 | 394 | now = time.time() 395 | stats = {} 396 | stats["last_updated"] = now 397 | stats["has_hostnames"] = config.stats_resolve_hostnames 398 | # Note paths configured in main.py by the Resource objects 399 | stats["static_base"] = "../static" 400 | stats["graph_base"] = "../static/graph" 401 | stats["server_version"] = __init__.version 402 | try: 403 | #rows = yield database.run_query("SELECT num_hosts,num_reports, num_clients, new_hosts FROM stats ORDER BY time DESC LIMIT 1") 404 | stats["num_hosts"] = yield models.Cracker.count() 405 | stats["num_reports"] = yield models.Report.count() 406 | 407 | rows = yield database.run_query("SELECT count(DISTINCT ip_address) FROM reports") 408 | if len(rows)>0: 409 | stats["num_clients"] = rows[0][0] 410 | else: 411 | stats["num_clients"] = 0 412 | 413 | yesterday = now - 24*3600 414 | stats["daily_reports"] = yield models.Report.count(where=["first_report_time>?", yesterday]) 415 | stats["daily_new_hosts"] = yield models.Cracker.count(where=["first_time>?", yesterday]) 416 | 417 | recent_hosts = yield models.Cracker.find(orderby="latest_time DESC", limit=10) 418 | yield threads.deferToThread(fixup_crackers, recent_hosts) 419 | stats["recent_hosts"] = recent_hosts 420 | 421 | most_reported_hosts = yield models.Cracker.find(orderby="total_reports DESC", limit=10) 422 | yield threads.deferToThread(fixup_crackers, most_reported_hosts) 423 | stats["most_reported_hosts"] = most_reported_hosts 424 | 425 | logging.info("Stats: {} reports for {} hosts from {} reporters".format( 426 | stats["num_reports"], stats["num_hosts"], stats["num_clients"])) 427 | 428 | if stats["num_reports"] > 0: 429 | yield Registry.DBPOOL.runInteraction(make_daily_graph) 430 | yield Registry.DBPOOL.runInteraction(make_monthly_graph) 431 | yield Registry.DBPOOL.runInteraction(make_contrib_graph) 432 | yield Registry.DBPOOL.runInteraction(make_history_graph) 433 | yield Registry.DBPOOL.runInteraction(make_country_bargraph) 434 | 435 | if _cache is None: 436 | _cache = {} 437 | _cache["stats"] = stats 438 | _cache["time"] = time.time() 439 | logging.debug("Finished updating statistics cache...") 440 | except Exception, e: 441 | log.err(_why="Error updating statistics: {}".format(e)) 442 | logging.warning("Error updating statistics: {}".format(e)) 443 | 444 | _stats_busy = False 445 | 446 | @inlineCallbacks 447 | def render_stats(): 448 | global _cache 449 | logging.info("Rendering statistics page...") 450 | if _cache is None: 451 | while _cache is None: 452 | logging.debug("No statistics cached yet, waiting for cache generation to finish...") 453 | yield task.deferLater(reactor, 1, lambda _:0, 0) 454 | 455 | now = time.time() 456 | try: 457 | env = Environment(loader=FileSystemLoader(config.template_dir)) 458 | env.filters['datetime'] = format_datetime 459 | template = env.get_template('stats.html') 460 | html = template.render(_cache["stats"]) 461 | 462 | logging.info("Done rendering statistics page...") 463 | returnValue(html) 464 | except Exception, e: 465 | log.err(_why="Error rendering statistics page: {}".format(e)) 466 | logging.warning("Error creating statistics page: {}".format(e)) 467 | 468 | def update_history_txn(txn, date): 469 | try: 470 | logging.info("Updating history table for {}".format(date)) 471 | start = time.mktime(date.timetuple()) 472 | end = start + 24*60*60 473 | #logging.debug("Date start, end: {}, {}".format(start, end)) 474 | 475 | txn.execute(database.translate_query(""" 476 | SELECT COUNT(*), 477 | COUNT(DISTINCT cracker_id), 478 | COUNT(DISTINCT ip_address) 479 | FROM reports 480 | WHERE (first_report_time>=? AND first_report_time=? AND latest_report_time0 and rows[0][0] is not None: 513 | last_filled_date = rows[0][0] 514 | else: 515 | txn.execute("SELECT MIN(first_report_time) FROM reports") 516 | first_time = txn.fetchall() 517 | if first_time is not None and len(first_time)>0 and first_time[0][0] is not None: 518 | last_filled_date = datetime.date.fromtimestamp(first_time[0][0]) 519 | else: 520 | last_filled_date = datetime.date.today() 521 | 522 | first_date = last_filled_date + datetime.timedelta(days=1) 523 | # Then fill history 524 | date = first_date 525 | while date <= last_date: 526 | update_history_txn(txn, date) 527 | date = date + datetime.timedelta(days = 1) 528 | 529 | except Exception as e: 530 | log.err(_why="Error updating history: {}".format(e)) 531 | logging.warning("Error updating history: {}".format(e)) 532 | 533 | 534 | def fixup_history_txn(txn): 535 | try: 536 | txn.execute("SELECT MIN(first_report_time) FROM reports") 537 | first_time = txn.fetchall() 538 | if first_time is not None and len(first_time)>0 and first_time[0][0] is not None: 539 | first_date = datetime.date.fromtimestamp(first_time[0][0]) 540 | else: 541 | # No data, nothing to do 542 | return 543 | 544 | last_date = datetime.date.today() - datetime.timedelta(days = 1) 545 | 546 | # Find any dates for which the history has not been filled 547 | txn.execute("SELECT date FROM history ORDER BY date ASC") 548 | rows = txn.fetchall() 549 | dates = set([row[0] for row in rows]) 550 | 551 | date = first_date 552 | while date <= last_date: 553 | if date not in dates: 554 | update_history_txn(txn, date) 555 | date = date + datetime.timedelta(days = 1) 556 | 557 | except Exception as e: 558 | log.err(_why="Error fixing up history: {}".format(e)) 559 | logging.warning("Error fixing up history: {}".format(e)) 560 | 561 | def update_recent_history(date=None): 562 | "date should be a datetime.date or None, indicating yesterday" 563 | return Registry.DBPOOL.runInteraction(update_recent_history_txn, date) 564 | 565 | def update_country_history_txn(txn, date=None, include_history = False): 566 | if date is None: 567 | date = datetime.date.today() - datetime.timedelta(days = 1) 568 | 569 | if include_history: 570 | start_time = 0 571 | else: 572 | start_time = (date - datetime.date(1970, 1, 1)).total_seconds() 573 | end_time = (date + datetime.timedelta(days=1) - datetime.date(1970, 1, 1)).total_seconds() 574 | 575 | txn.execute("SELECT country_code,country,num_reports FROM country_history") 576 | rows = txn.fetchall() 577 | result = {row[0]:(row[1],row[2]) for row in rows} 578 | 579 | gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE) 580 | 581 | txn.execute(database.translate_query( 582 | """SELECT crackers.ip_address,COUNT(*) as count 583 | FROM crackers LEFT JOIN reports ON reports.cracker_id = crackers.id 584 | WHERE reports.first_report_time >= ? AND reports.first_report_time < ? 585 | GROUP BY crackers.id 586 | """), (start_time, end_time)) 587 | 588 | while True: 589 | rows = txn.fetchmany(size=100) 590 | if len(rows) == 0: 591 | break 592 | for row in rows: 593 | ip = row[0] 594 | count = row[1] 595 | try: 596 | country_code = gi.country_code_by_addr(ip) 597 | country = gi.country_name_by_addr(ip) 598 | if country_code is None: 599 | country_code = "ZZ" 600 | if country is None: 601 | country = "(Unknown)" 602 | if country_code in result: 603 | count += result[country_code][1] 604 | result[country_code] = (country,count) 605 | except Exception, e: 606 | logging.debug("Exception looking up country for {}: {}".format(ip, e)) 607 | 608 | logging.debug("result: ".format(result)) 609 | 610 | for country_code in result: 611 | country,count = result[country_code] 612 | txn.execute(database.translate_query( 613 | """REPLACE INTO country_history (country_code,country,num_reports) 614 | VALUES (?,?,?)"""), 615 | (country_code,country,count)) 616 | 617 | def update_country_history(date=None, include_history=False): 618 | "date should be a datetime.date or None, indicating yesterday" 619 | return Registry.DBPOOL.runInteraction(update_country_history_txn, date, include_history) 620 | 621 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 622 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published 637 | by the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . 662 | --------------------------------------------------------------------------------