├── config ├── __init__.py └── config.py ├── core ├── __init__.py └── color.py ├── modules ├── __init__.py └── retcodes.py ├── config.ini ├── requirements.txt ├── README.md ├── install.sh └── squidmagic.py /config/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /core/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /modules/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /config.ini: -------------------------------------------------------------------------------- 1 | [spamhaus] 2 | query_url = http://www.spamhaus.org/query/bl?ip= 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | sh 2 | termcolor 3 | configparser 4 | dnspython 5 | pyzmq 6 | -------------------------------------------------------------------------------- /core/color.py: -------------------------------------------------------------------------------- 1 | def colors(): 2 | 3 | SQUID_MAGIC_COLORS = { 4 | 'BOLD': { 5 | 'code': '\033[1m', 6 | }, 7 | 'END': { 8 | 'code': '\033[0m', 9 | }, 10 | } 11 | return SQUID_MAGIC_COLORS 12 | -------------------------------------------------------------------------------- /config/config.py: -------------------------------------------------------------------------------- 1 | from configparser import ConfigParser 2 | 3 | def spamhaus(filename='config.ini', section='spamhaus'): 4 | parser = ConfigParser() 5 | parser.read(filename) 6 | 7 | spamhaus = {} 8 | if parser.has_section(section): 9 | items = parser.items(section) 10 | for item in items: 11 | spamhaus[item[0]] = item[1] 12 | return item[1] 13 | else: 14 | raise Exception('{0} not found in the {1} file'.format(section, filename)) 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Squidmagic is a tool designed to analyze a web-based network traffic to detect central command and control (C&C) servers and Malicious site, using Squid proxy server and Spamhaus. 2 | 3 | ### Install dependencies 4 | 5 | ``` 6 | pip install -r requirements.txt 7 | ``` 8 | 9 | ### Usage 10 | 11 | ``` 12 | python squidmagic.py /squid/access.log 13 | 14 | _ _ _ 15 | (_) | | (_) 16 | ___ __ _ _ _ _ __| |_ __ ___ __ _ __ _ _ ___ 17 | / __|/ _` | | | | |/ _` | '_ ` _ \ / _` |/ _` | |/ __| 18 | \__ \ (_| | |_| | | (_| | | | | | | (_| | (_| | | (__ 19 | |___/\__, |\__,_|_|\__,_|_| |_| |_|\__,_|\__, |_|\___| 20 | | | __/ | 21 | |_| |___/ 22 | Analyzing... 23 | 24 | Analyzing by SBL Advisory... 25 | Spam server detected, ip is 65.182.101.221 26 | Analyzing by SBL_CSS Advisory... 27 | safe server detected, host or ip is 65.182.101.221 28 | Analyzing by PBL Advisory... 29 | safe server detected, host or ip is 65.182.101.221 30 | 31 | ``` 32 | -------------------------------------------------------------------------------- /modules/retcodes.py: -------------------------------------------------------------------------------- 1 | def sblcodes(): 2 | 3 | SBL_RET_CODES = { 4 | '127.0.0.2': { 5 | 'status': 'spam', 6 | }, 7 | } 8 | return SBL_RET_CODES 9 | 10 | def sblzencodes(): 11 | 12 | ZEN_RETURN_CODES_SBL = {'127.0.0.2': 'Spam'} 13 | return ZEN_RETURN_CODES_SBL 14 | 15 | def sblcsscodes(): 16 | SBL_CSS_RET_CODES = { 17 | '127.0.0.3': { 18 | 'status': 'spam', 19 | }, 20 | } 21 | return SBL_CSS_RET_CODES 22 | 23 | def sblcsszencodes(): 24 | 25 | ZEN_RETURN_CODES = {'127.0.0.3': 'Spam'} 26 | return ZEN_RETURN_CODES 27 | 28 | def rblcodes(): 29 | PBL_RET_CODES = { 30 | '127.0.0.10': { 31 | 'status': 'spam', 32 | }, 33 | '127.0.0.11': { 34 | 'status': 'spam', 35 | }, 36 | } 37 | return PBL_RET_CODES 38 | 39 | def zenrblcodes(): 40 | ZEN_RETURN_CODES = {'127.0.0.10': 'Spam', '127.0.0.11': 'Spam'} 41 | return ZEN_RETURN_CODES 42 | 43 | def xblcodes(): 44 | XBL_RET_CODES = { 45 | '127.0.0.4': { 46 | 'status': 'worms/viruses', 47 | }, 48 | '127.0.0.5': { 49 | 'status': 'worms/viruses', 50 | }, 51 | '127.0.0.6': { 52 | 'status': 'worms/viruses', 53 | }, 54 | '127.0.0.7': { 55 | 'status': 'worms/viruses', 56 | }, 57 | } 58 | return XBL_RET_CODES 59 | 60 | def zenxblcodes(): 61 | ZEN_RETURN_CODES = {'127.0.0.4': 'worms/viruses', '127.0.0.5': 'worms/viruses', 62 | '127.0.0.6': 'worms/viruses', '127.0.0.7': 'worms/viruses'} 63 | return ZEN_RETURN_CODES 64 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SQUID_REPO='https://github.com/ch3k1/squidmagic' 4 | 5 | declare -a packages 6 | declare -a python_packages 7 | python_packages=(sh configparser termcolor dnspython pyzmq) 8 | packages["ubuntu"]="python-pip git libzmq-dev" 9 | 10 | Color_Off='\033[0m' # Text Reset 11 | LOG=$(mktemp) 12 | # Regular Colors 13 | Red='\033[0;31m' # Red 14 | log_icon="\e[31m✓\e[0m" 15 | log_icon_ok="\e[32m✓\e[0m" 16 | log_icon_nok="\e[31m✗\e[0m" 17 | 18 | if [[ $EUID -ne 0 ]]; then 19 | echo -e "$Red \n You must be a root user.. $Color_Off" 20 | exit 1 21 | else 22 | 23 | [[ ! -e /etc/debian_version ]] && { 24 | echo -e "$Red \n This script currently works only ubuntu 16.04 distro $Color_Off" 25 | exit 1 26 | } 27 | 28 | run_and_log() { 29 | $1 &> ${LOG} && { 30 | _log_icon=$log_icon_ok 31 | } || { 32 | _log_icon=$log_icon_nok 33 | exit_=1 34 | } 35 | echo -e "${_log_icon} ${2}" 36 | [[ $exit_ ]] && { echo -e "\t -> ${_log_icon} $3"; exit; } 37 | } 38 | 39 | install_packages(){ 40 | # Update and Install packages 41 | sudo apt-get update -y && sudo apt-get upgrade -y 42 | sudo apt-get install -y ${packages[@]} 43 | return 0 44 | } 45 | 46 | clone_repo(){ 47 | git clone ${SQUID_REPO} 48 | return 0 49 | } 50 | 51 | # install python packages 52 | install_python_packages(){ 53 | pip install ${python_packages[@]} 54 | return 0 55 | } 56 | 57 | run_and_log install_packages "Installing system packages" 58 | run_and_log clone_repo "Cloning repo" "Could not clone repo" 59 | run_and_log install_python_packages "Installing python packages" 60 | 61 | 62 | fi -------------------------------------------------------------------------------- /squidmagic.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys,os,argparse,re 4 | import signal,time 5 | 6 | import dns.resolver 7 | import zmq 8 | from sh import tail 9 | 10 | from termcolor import colored 11 | from config.config import spamhaus 12 | from modules import retcodes 13 | from core import color 14 | 15 | colors = color.colors() 16 | 17 | # ZeroMQ Context 18 | context = zmq.Context() 19 | 20 | # This is our new stuff 21 | zmq_socket = context.socket(zmq.PUSH) 22 | zmq_socket.connect("tcp://127.0.0.1:5555") 23 | 24 | if len(sys.argv) <= 1: 25 | print("Please give some options, type -h for more information") 26 | sys.exit() 27 | 28 | signal.signal(signal.SIGINT, signal.SIG_DFL); 29 | 30 | class squidmagic(object): 31 | def __init__(self, apiattr, species): 32 | self.apiattr = apiattr 33 | self.species = species 34 | 35 | def getapiattr(self): 36 | return self.apiattr 37 | def getSpecies(self): 38 | return self.species 39 | 40 | class Banner(squidmagic): 41 | def __init__(self, apiattr): 42 | squidmagic.__init__(self, apiattr, """ 43 | _ _ _ 44 | (_) | | (_) 45 | ___ __ _ _ _ _ __| |_ __ ___ __ _ __ _ _ ___ 46 | / __|/ _` | | | | |/ _` | '_ ` _ \ / _` |/ _` | |/ __| 47 | \__ \ (_| | |_| | | (_| | | | | | | (_| | (_| | | (__ 48 | |___/\__, |\__,_|_|\__,_|_| |_| |_|\__,_|\__, |_|\___| 49 | | | __/ | 50 | |_| |___/ 51 | """) 52 | 53 | banner = Banner("Banner") 54 | parser = argparse.ArgumentParser(description='squidmagic is a tool designed to analyze a web-based network traffic') 55 | parser.add_argument('logfile_path', help='Select a Squid server log file path') 56 | args = parser.parse_args() 57 | 58 | if os.path.isfile(args.logfile_path): 59 | sys.stdout.write(colors['BOLD']['code'] + banner.getSpecies() + colors['END']['code'] + colors['BOLD']['code'] + "Analyzing...\n" + colors['END']['code']) 60 | print 61 | else: 62 | sys.stdout.write(colors['BOLD']['code'] + "Squid server log file not found.\n" + colors['END']['code']) 63 | sys.exit(0) 64 | 65 | class sbladvisory: 66 | 67 | @staticmethod 68 | def default_response(self): 69 | self.sp_response = {'status': '0', 70 | 'response_code': '0', 71 | 'url': 'null'} 72 | 73 | def check_status(self, ip): 74 | self.default_response(self) 75 | sp_resolver = dns.resolver.Resolver() 76 | 77 | try: 78 | _r_name = dns.reversename.from_address(ip) 79 | 80 | _r_name = str(_r_name).replace("in-addr.arpa.", "zen.spamhaus.org.") 81 | answers = sp_resolver.query(_r_name) 82 | 83 | for rdata in answers: 84 | _url = spamhaus()+ip 85 | self.sp_response = {'status': '1', 86 | 'response_code': 87 | retcodes.sblcodes()[rdata.address], 88 | 'assessment': 89 | retcodes.sblzencodes()[rdata.address], 90 | 'url': _url} 91 | except Exception, e: 92 | pass 93 | 94 | return self.sp_response 95 | 96 | class sblcssadvisory: 97 | 98 | @staticmethod 99 | def default_response(self): 100 | self.sp_response = {'status': '0', 101 | 'response_code': '0', 102 | 'url': 'null'} 103 | 104 | def check_status(self, ip): 105 | self.default_response(self) 106 | sp_resolver = dns.resolver.Resolver() 107 | 108 | try: 109 | 110 | _r_name = dns.reversename.from_address(ip) 111 | 112 | _r_name = str(_r_name).replace("in-addr.arpa.", "zen.spamhaus.org.") 113 | answers = sp_resolver.query(_r_name) 114 | 115 | for rdata in answers: 116 | _url = spamhaus()+ip 117 | self.sp_response = {'status': '1', 118 | 'response_code': 119 | retcodes.sblcsscodes()[rdata.address], 120 | 'assessment': 121 | retcodes.sblcsszencodes[rdata.address], 122 | 'url': _url} 123 | except Exception, e: 124 | pass 125 | 126 | return self.sp_response 127 | 128 | class pbladvisory: 129 | 130 | @staticmethod 131 | def default_response(self): 132 | self.sp_response = {'status': '0', 133 | 'response_code': '0', 134 | 'url': 'null'} 135 | 136 | def check_status(self, ip): 137 | self.default_response(self) 138 | sp_resolver = dns.resolver.Resolver() 139 | 140 | try: 141 | 142 | _r_name = dns.reversename.from_address(ip) 143 | 144 | _r_name = str(_r_name).replace("in-addr.arpa.", "zen.spamhaus.org.") 145 | answers = sp_resolver.query(_r_name) 146 | 147 | for rdata in answers: 148 | _url = spamhaus()+ip 149 | self.sp_response = {'status': '1', 150 | 'response_code': 151 | retcodes.rblcodes()[rdata.address], 152 | 'assessment': 153 | retcodes.zenrblcodes()[rdata.address], 154 | 'url': _url} 155 | except Exception, e: 156 | pass 157 | 158 | return self.sp_response 159 | 160 | class xbladvisory: 161 | 162 | @staticmethod 163 | def default_response(self): 164 | self.sp_response = {'status': '0', 165 | 'response_code': '0', 166 | 'url': 'null'} 167 | 168 | def check_status(self, ip): 169 | self.default_response(self) 170 | sp_resolver = dns.resolver.Resolver() 171 | 172 | try: 173 | 174 | _r_name = dns.reversename.from_address(ip) 175 | 176 | _r_name = str(_r_name).replace("in-addr.arpa.", "zen.spamhaus.org.") 177 | answers = sp_resolver.query(_r_name) 178 | 179 | for rdata in answers: 180 | _url = spamhaus()+ip 181 | self.sp_response = {'status': '1', 182 | 'response_code': 183 | retcodes.xblcodes()[rdata.address], 184 | 'assessment': 185 | retcodes.zenxblcodes()[rdata.address], 186 | 'url': _url} 187 | except Exception, e: 188 | pass 189 | 190 | return self.sp_response 191 | 192 | for i, line in enumerate(tail("-f", args.logfile_path, _iter=True)): 193 | 194 | try: 195 | duration, byte, address, code_status, num_bytes, method, urls, rfc931, HIER_DIRECT, _ = line.split()[:10] 196 | 197 | def sbl_spamhaus(): 198 | time.sleep(2) 199 | sys.stdout.write(colored(colors['BOLD']['code'] + "Analyzing by SBL Advisory...\n" + colors['END']['code'], 'blue')) 200 | ips = re.findall(r'[0-9]+(?:\.[0-9]+){3}', HIER_DIRECT) 201 | if not ips: 202 | print '\t', colored("The IP address or hostname where the request was forwarded not found", 'red') 203 | return 204 | for ip in ips: 205 | domain_ip = ip.split(":")[0] 206 | 207 | sh = sbladvisory() 208 | response = sh.check_status(domain_ip) 209 | if response['status'] == '0': 210 | print '\t', colored("safe server detected, host or ip is " + domain_ip, 'green') 211 | msg = { 'squidmagic': 'squidmagic','host' : domain_ip, 'status' : 'safe' } 212 | zmq_socket.send_json(msg) 213 | else: 214 | print '\t', colored("Spam server detected, ip is " + domain_ip, 'red') 215 | msg = { 'squidmagic': 'squidmagic', 'host' : domain_ip, 'status' : 'unsafe' } 216 | zmq_socket.send_json(msg) 217 | 218 | def css_spamhaus(): 219 | time.sleep(2) 220 | sys.stdout.write(colored(colors['BOLD']['code'] + "Analyzing by SBL_CSS Advisory...\n" + colors['END']['code'], 'blue')) 221 | ips = re.findall(r'[0-9]+(?:\.[0-9]+){3}', HIER_DIRECT) 222 | if not ips: 223 | print '\t', colored("The IP address or hostname where the request was forwarded not found", 'red') 224 | return 225 | for ip in ips: 226 | domain_ip = ip.split(":")[0] 227 | 228 | sh = sblcssadvisory() 229 | response = sh.check_status(domain_ip) 230 | if response['status'] == '0': 231 | print '\t', colored("safe server detected, host or ip is " + domain_ip, 'green') 232 | msg = { 'squidmagic': 'squidmagic', 'host' : domain_ip, 'status' : 'safe' } 233 | zmq_socket.send_json(msg) 234 | else: 235 | print '\t', colored("Spam server detected, ip is " + domain_ip, 'red') 236 | msg = { 'squidmagic': 'squidmagic', 'host' : domain_ip, 'status' : 'unsafe' } 237 | zmq_socket.send_json(msg) 238 | 239 | def pbl_spamhaus(): 240 | time.sleep(2) 241 | sys.stdout.write(colored(colors['BOLD']['code'] + "Analyzing by PBL Advisory...\n" + colors['END']['code'], 'blue')) 242 | ips = re.findall(r'[0-9]+(?:\.[0-9]+){3}', HIER_DIRECT) 243 | if not ips: 244 | print '\t', colored("The IP address or hostname where the request was forwarded not found", 'red') 245 | return 246 | for ip in ips: 247 | domain_ip = ip.split(":")[0] 248 | 249 | sh = pbladvisory() 250 | response = sh.check_status(domain_ip) 251 | 252 | if response['status'] == '0': 253 | print '\t', colored("safe server detected, host or ip is " + domain_ip, 'green') 254 | msg = { 'squidmagic': 'squidmagic', 'host' : domain_ip, 'status' : 'safe' } 255 | zmq_socket.send_json(msg) 256 | else: 257 | print '\t', colored("Spam server detected, ip is " + domain_ip, 'red') 258 | msg = { 'squidmagic': 'squidmagic', 'host' : domain_ip, 'status' : 'unsafe' } 259 | zmq_socket.send_json(msg) 260 | 261 | def xbl_spamhaus(): 262 | time.sleep(2) 263 | sys.stdout.write(colored(colors['BOLD']['code'] + "Analyzing by XBL Advisory...\n" + colors['END']['code'], 'blue')) 264 | ips = re.findall(r'[0-9]+(?:\.[0-9]+){3}', HIER_DIRECT) 265 | if not ips: 266 | print '\t', colored("The IP address or hostname where the request was forwarded not found", 'red') 267 | return 268 | for ip in ips: 269 | domain_ip = ip.split(":")[0] 270 | 271 | sh = xbladvisory() 272 | response = sh.check_status(domain_ip) 273 | 274 | if response['status'] == '0': 275 | print '\t', colored("safe server detected, host or ip is " + domain_ip, 'green') 276 | msg = { 'squidmagic': 'squidmagic', 'host' : domain_ip, 'status' : 'safe' } 277 | zmq_socket.send_json(msg) 278 | else: 279 | print '\t', colored("Malicious web server detected, ip is " + domain_ip, 'red') 280 | msg = { 'squidmagic': 'squidmagic', 'host' : domain_ip, 'status' : 'unsafe' } 281 | zmq_socket.send_json(msg) 282 | 283 | def main(): 284 | sbl_spamhaus() 285 | css_spamhaus() 286 | pbl_spamhaus() 287 | xbl_spamhaus() 288 | 289 | if __name__ == '__main__': 290 | main() 291 | 292 | except ValueError: 293 | continue 294 | 295 | if rfc931 == '-': continue 296 | 297 | try: 298 | if code_status.split('/')[1] == '407': continue 299 | except IndexError: 300 | continue 301 | 302 | try: 303 | sum_bytes[rfc931] = sum_bytes[rfc931] + int(num_bytes) 304 | except KeyError: 305 | sum_bytes[rfc931] = int(num_bytes) 306 | --------------------------------------------------------------------------------