├── lib ├── Syslog.py ├── Messages.py ├── PostfixLog.py ├── Sql.py ├── DovecotLog.py ├── AuthLog.py ├── Config.py ├── ProcessLog.py ├── Logs.py ├── Queue.py └── ProcessLine.py ├── WWW ├── .htaccess ├── index.php └── lib │ ├── autoload.php │ ├── AuthLogs.php │ ├── DovecotLogs.php │ ├── PostfixLogs.php │ ├── Config.php │ ├── Logs.php │ ├── SQL.php │ └── Api.php ├── MailLog2MySQL ├── etc └── MailLog2MySQL.conf ├── test.py ├── Makefile ├── init.d └── maillog2mysql ├── test.sh └── README.md /lib/Syslog.py: -------------------------------------------------------------------------------- 1 | import syslog 2 | 3 | class Syslog: 4 | 5 | @staticmethod 6 | def write(msg): 7 | syslog.syslog(msg) -------------------------------------------------------------------------------- /WWW/.htaccess: -------------------------------------------------------------------------------- 1 | RewriteEngine On 2 | RewriteCond %{ENV:REDIRECT_STATUS} 200 3 | RewriteRule ^ - [L] 4 | RewriteCond %{REQUEST_FILENAME} !-f 5 | RewriteCond %{REQUEST_FILENAME} !-d 6 | RewriteRule . index.php [L] -------------------------------------------------------------------------------- /WWW/index.php: -------------------------------------------------------------------------------- 1 | 11 | 12 | -------------------------------------------------------------------------------- /MailLog2MySQL: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | import sys 3 | 4 | # adding Folder_2 to the system path 5 | sys.path.insert(0, '/usr/lib/MailLog2MySQL') 6 | 7 | from ProcessLog import ProcessLog 8 | logfile = open("/var/log/syslog","r") 9 | # read_lines(logfile) 10 | 11 | ProcessLog.main() 12 | -------------------------------------------------------------------------------- /etc/MailLog2MySQL.conf: -------------------------------------------------------------------------------- 1 | #mysql settings 2 | host = localhost 3 | user = sammy 4 | password = password 5 | database = maillog 6 | port = 3306 7 | 8 | #read logs 9 | log_file = /var/log/mail.log 10 | 11 | #api access 12 | access = ["127.0.0.1", "192.168.110.158"] 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | f = open("/var/log/mailogtest") 4 | 5 | 6 | i = 0 7 | while (True): 8 | line = f.readline() 9 | if (not line) : 10 | break 11 | 12 | cmd = "echo '"+line.split("\n")[0]+"' >> /var/log/mail.log" 13 | # print(cmd) 14 | 15 | os.system(cmd) 16 | 17 | # i=i+1 18 | 19 | 20 | # if ( i == 5000 ): 21 | # break 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /WWW/lib/autoload.php: -------------------------------------------------------------------------------- 1 | /var/run/MailLog2MySQL.pid 19 | fi 20 | ;; 21 | stop) 22 | kill `cat /var/run/MailLog2MySQL.pid` 23 | rm /var/run/MailLog2MySQL.pid 24 | ;; 25 | restart) 26 | $0 stop 27 | $0 start 28 | ;; 29 | status) 30 | if [ -e /var/run/MailLog2MySQL.pid ]; then 31 | echo MailLog2MySQL is running, pid=`cat /var/run/MailLog2MySQL.pid` 32 | else 33 | echo MailLog2MySQL is NOT running 34 | exit 1 35 | fi 36 | ;; 37 | *) 38 | echo "Usage: $0 {start|stop|status|restart}" 39 | esac 40 | 41 | exit 0 42 | -------------------------------------------------------------------------------- /lib/Sql.py: -------------------------------------------------------------------------------- 1 | import pymysql 2 | 3 | from Config import Config 4 | 5 | class Sql: 6 | @staticmethod 7 | def connect(): 8 | Sql.conn = pymysql.connect( 9 | host = Config.mysql['host'], 10 | user = Config.mysql['user'], 11 | password = Config.mysql['password'], 12 | db = Config.mysql['database'], 13 | port = int(Config.mysql['port']) 14 | ) 15 | return 0 16 | 17 | 18 | @staticmethod 19 | def reconnect(): 20 | try: 21 | Sql.conn.ping(reconnect=False) 22 | except: 23 | Sql.connect() 24 | 25 | 26 | 27 | @staticmethod 28 | def insert(sql, data): 29 | Sql.reconnect() 30 | cursor = Sql.conn.cursor() 31 | cursor.executemany(sql,(data)) 32 | Sql.conn.commit() 33 | cursor.close() 34 | 35 | 36 | 37 | @staticmethod 38 | def execute(sql): 39 | Sql.connect() 40 | cursor = Sql.conn.cursor() 41 | cursor._defer_warnings = True 42 | cursor.execute(sql) 43 | Sql.conn.commit() 44 | 45 | cursor.close() 46 | Sql.conn.close() -------------------------------------------------------------------------------- /lib/DovecotLog.py: -------------------------------------------------------------------------------- 1 | from Sql import Sql 2 | from Syslog import Syslog 3 | 4 | class DovecotLog: 5 | month ="" 6 | day ="" 7 | hour ="" 8 | domain ="" 9 | email ="" 10 | msgid ="" 11 | log ="" 12 | 13 | 14 | @staticmethod 15 | def toList(dovecot): 16 | return [ 17 | dovecot.month, 18 | dovecot.day, 19 | dovecot.hour, 20 | dovecot.domain, 21 | dovecot.email, 22 | dovecot.msgid, 23 | dovecot.log 24 | ] 25 | 26 | 27 | @staticmethod 28 | def createTable(): 29 | sql = ''' 30 | CREATE TABLE IF NOT EXISTS dovecot_logs ( 31 | id int AUTO_INCREMENT, 32 | month varchar(3), 33 | day varchar(2), 34 | hour varchar(2), 35 | domain varchar(124), 36 | email varchar(64), 37 | msgid varchar(64), 38 | log varchar(64), 39 | PRIMARY KEY (id, domain, email, msgid, hour, day, month)) 40 | ''' 41 | try: 42 | Sql.execute(sql) 43 | except Exception as e: 44 | Syslog.write("Error to create dovecot_logs table") 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /lib/AuthLog.py: -------------------------------------------------------------------------------- 1 | from Sql import Sql 2 | from Syslog import Syslog 3 | 4 | class AuthLog: 5 | month ="" 6 | day ="" 7 | hour ="" 8 | domain ="" 9 | email ="" 10 | ip ="" 11 | log ="" 12 | 13 | 14 | @staticmethod 15 | def toList(auth): 16 | return [ 17 | auth.month, 18 | auth.day, 19 | auth.hour, 20 | auth.domain, 21 | auth.email, 22 | auth.ip, 23 | auth.log 24 | ] 25 | 26 | 27 | @staticmethod 28 | def createTable(): 29 | sql = ''' 30 | CREATE TABLE IF NOT EXISTS auth_logs ( 31 | `id` int(11) NOT NULL AUTO_INCREMENT, 32 | `month` varchar(3) DEFAULT NULL, 33 | `day` varchar(2) NOT NULL, 34 | `hour` varchar(2) DEFAULT NULL, 35 | `domain` varchar(124) NOT NULL, 36 | `ip` varchar(64) NOT NULL, 37 | `email` varchar(64) NOT NULL, 38 | `log` text DEFAULT NULL, 39 | PRIMARY KEY (`id`,`domain`,`email`,`ip`,`month`,`day`, `hour`) 40 | ) 41 | ''' 42 | try: 43 | Sql.execute(sql) 44 | except: 45 | Syslog.write("Error to create auth_logs table") 46 | -------------------------------------------------------------------------------- /lib/Config.py: -------------------------------------------------------------------------------- 1 | import os 2 | from Syslog import Syslog 3 | 4 | class Config: 5 | mysql = { 6 | "host" : None, 7 | "user" : None, 8 | "password" : None, 9 | "database" : None, 10 | "port" : None 11 | } 12 | log_file = None 13 | 14 | 15 | @staticmethod 16 | def read(): 17 | file="/etc/MailLog2MySQL.conf" 18 | 19 | if ( not (os.path.exists(file) ) ): 20 | Syslog.write(f"File {file} not exists") 21 | raise Exception (f"File {file} not exists") 22 | 23 | conf = open(file, "r") 24 | 25 | while (True): 26 | line = conf.readline() 27 | 28 | if (not line): 29 | break 30 | 31 | line= line.split("\n")[0] 32 | line=line.replace(" ", "") 33 | splitted = line.split("=") 34 | 35 | # if (len(splitted) != 2 and line !=""): 36 | # Syslog.write(f"error at {line} ") 37 | # raise Exception (f"error at {line} ") 38 | 39 | if ( splitted[0] in Config.mysql ): 40 | Config.mysql[splitted[0]] = splitted[1] 41 | continue 42 | 43 | if ( splitted[0] == "log_file"): 44 | Config.log_file = splitted[1] 45 | 46 | return 1 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /WWW/lib/Config.php: -------------------------------------------------------------------------------- 1 | setAccess( $type[1] ); 21 | } 22 | 23 | if ($type[0] == "host"){ 24 | $api->db->setHost( $type[1] ); 25 | } 26 | 27 | if ($type[0] == "user"){ 28 | $api->db->setUser( $type[1] ); 29 | } 30 | 31 | if ($type[0] == "password"){ 32 | $api->db->setPassword( $type[1] ); 33 | } 34 | 35 | if ($type[0] == "database"){ 36 | $api->db->setDatabase( $type[1] ); 37 | } 38 | 39 | if ($type[0] == "port"){ 40 | $api->db->setPort( $type[1] ); 41 | } 42 | } 43 | 44 | fclose($handle); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /WWW/lib/Logs.php: -------------------------------------------------------------------------------- 1 | db->connect(); 19 | $stmt= $api->db->conn->prepare($sql); 20 | SQL::doBind($toBind, $stmt); 21 | $stmt->execute(); 22 | 23 | $res = json_encode( $stmt->get_result()->fetch_all(MYSQLI_ASSOC) ); 24 | 25 | $sql="SELECT count(*) as 'number of logs' FROM ".$log::TABLE." ".$where; 26 | $stmt=$api->db->conn->prepare($sql); 27 | SQL::doBind($toBind, $stmt); 28 | $stmt->execute(); 29 | $num_logs = $stmt->get_result()->fetch_all()[0][0]; 30 | $num_pages = ceil($num_logs/$log::LIMIT); 31 | 32 | $msg ="{"; 33 | $msg.="\"number of logs\":".$num_logs.","; 34 | $msg.="\"number of pages\":".$num_pages.","; 35 | $msg.="\"data\":".$res; 36 | $msg.="}"; 37 | 38 | Api::response($msg); 39 | } 40 | 41 | static public function createWhere($params,&$where, &$toBind){ 42 | $where="WHERE "; 43 | for ($i=0; $ihost =$host; 13 | } 14 | 15 | public function setUser($user){ 16 | $this->user =$user; 17 | } 18 | 19 | public function setPassword($password){ 20 | $this->password =$password; 21 | } 22 | 23 | public function setDatabase($database){ 24 | $this->database =$database; 25 | } 26 | 27 | public function setPort($port){ 28 | $this->port =$port; 29 | } 30 | 31 | public function connect(){ 32 | // Create connection 33 | $this->conn = new mysqli( 34 | $this->host, 35 | $this->user, 36 | $this->password, 37 | $this->database, 38 | $this->port 39 | ); 40 | 41 | if ($this->conn->connect_error){ 42 | $this->showError($this->conn->connect_error); 43 | } 44 | 45 | } 46 | 47 | public function close(){ 48 | $this->conn =null; 49 | } 50 | 51 | public function showError($msg){ 52 | ob_clean(); 53 | header('Content-Type: application/json; charset=utf-8'); 54 | die("{\"msg\":\"".$msg."\"}"); 55 | http_response_code(403); 56 | } 57 | 58 | public function showConfig(){ 59 | echo $this->host ."
"; 60 | echo $this->user ."
"; 61 | echo $this->password ."
"; 62 | echo $this->database ."
"; 63 | echo $this->port ."
"; 64 | } 65 | 66 | public function print($sql ){ 67 | $this->connect(); 68 | $stmt =$this->conn->prepare($sql); 69 | $stmt->execute($sql); 70 | var_dump($stmt->get_result()); 71 | $stmt->close(); 72 | $this->close(); 73 | } 74 | 75 | static public function doBind($toBind, $stmt){ 76 | $bindParams=""; 77 | for ($i=0; $ibind_param(\"".str_repeat("s", count($toBind))."\",".$bindParams.");"); 84 | } 85 | } 86 | } -------------------------------------------------------------------------------- /lib/ProcessLog.py: -------------------------------------------------------------------------------- 1 | import os, sys, time 2 | from ProcessLine import ProcessLine 3 | from Sql import Sql 4 | from Logs import Logs 5 | from AuthLog import AuthLog 6 | from DovecotLog import DovecotLog 7 | from PostfixLog import PostfixLog 8 | from Queue import Queue 9 | from Config import Config 10 | from Syslog import Syslog 11 | 12 | 13 | class ProcessLog: 14 | @staticmethod 15 | def main(): 16 | Config.read() 17 | 18 | file = ProcessLog.openFile() 19 | inode = os.fstat(file.fileno()).st_ino 20 | 21 | logs = Logs() 22 | 23 | AuthLog .createTable() 24 | DovecotLog.createTable() 25 | PostfixLog.createTable() 26 | 27 | tik = time.time() 28 | i=0 29 | Syslog.write("started") 30 | while True: 31 | try: 32 | line = file.readline() 33 | line = line.split("\r")[0] 34 | 35 | if not line: 36 | time.sleep(1) 37 | if os.stat(Config.log_file).st_ino != inode: 38 | Syslog.write("LOGROTATE") 39 | file.close() 40 | file = ProcessLog.openFile() 41 | inode = os.fstat(file.fileno()).st_ino 42 | 43 | else: 44 | ProcessLine.main(line, logs) 45 | 46 | tok = time.time() 47 | if ( (tok - tik) > 1 ): 48 | tik=time.time() 49 | Queue.preareToInsert(logs) 50 | logs.insertToMySQL() 51 | # print(logs.num_logs) 52 | 53 | if (Queue.NUM_LOGS > 4): 54 | Queue.preareToInsert(logs) 55 | 56 | if (logs.num_logs > 10 ): 57 | logs.insertToMySQL() 58 | i=i+1 59 | except: 60 | pass 61 | 62 | 63 | @staticmethod 64 | def openFile(): 65 | file=Config.log_file 66 | 67 | if ( not (os.path.exists(file) ) ): 68 | Syslog.write(f"File {file} not exists") 69 | raise Exception (f"File {file} not exists") 70 | 71 | f= open(file,"r") 72 | f.seek(0, os.SEEK_END) 73 | return f 74 | 75 | -------------------------------------------------------------------------------- /WWW/lib/Api.php: -------------------------------------------------------------------------------- 1 | db=new SQL(); 9 | Config::readFile($this); 10 | //check IP 11 | $this->checkIP(); 12 | } 13 | 14 | public function setAccess($access){ 15 | 16 | $this->access = json_decode($access); 17 | if ($this->access == NULL) { 18 | Api::block("Invalid access IP's format in conf file"); 19 | } 20 | } 21 | 22 | public function setKey($key){ 23 | $this->api_key = $key; 24 | } 25 | 26 | public function checkIP( ) { 27 | $ip = Api::getIP(); 28 | foreach($this->access as $ip_allowed){ 29 | if ( $ip === $ip_allowed ){ 30 | return; 31 | } 32 | } 33 | Api::block("Permission denied"); 34 | } 35 | 36 | 37 | 38 | public static function getIP(){ 39 | return $_SERVER['REMOTE_ADDR']; 40 | } 41 | 42 | 43 | public static function block($msg){ 44 | ob_clean(); 45 | header('Content-Type: application/json; charset=utf-8'); 46 | die("{\"msg\":\"".$msg."\"}"); 47 | http_response_code(403); 48 | } 49 | 50 | public static function response($msg){ 51 | ob_clean(); 52 | header('Content-Type: application/json; charset=utf-8'); 53 | die($msg); 54 | http_response_code(200); 55 | } 56 | 57 | public static function main(){ 58 | 59 | $api=new Api(); 60 | 61 | /*** 62 | * 63 | * ?table=auth_logs 64 | * ?table=dovecot_logs 65 | * ?table=postfix_logs 66 | */ 67 | 68 | if ( isset($_GET['table']) && $_GET['table']==="auth_logs"){ 69 | AuthLogs::main($api); 70 | } 71 | if ( isset($_GET['table']) && $_GET['table']==="dovecot_logs"){ 72 | DovecotLogs::main($api); 73 | } 74 | if ( isset($_GET['table']) && $_GET['table']==="postfix_logs"){ 75 | PostfixLogs::main($api); 76 | } 77 | 78 | $resp="{"; 79 | $resp.="\"auth_logs\":[\"month\",\"day\",\"hour\",\"domain\",\"ip\",\"email\",\"log\"],"; 80 | $resp.="\"dovecot_logs\":[\"month\",\"day\",\"hour\",\"domain\",\"email\",\"msgid\",\"log\"],"; 81 | $resp.="\"postfix_logs\":[\"month\",\"day\",\"hour\",\"mail_to\",\"mail_to_domain\",\"mail_from\",\"mail_from_domain\",\"status\",\"msgid\"]"; 82 | $resp.="}"; 83 | Api::response($resp); 84 | } 85 | } 86 | 87 | 88 | /** 89 | * 90 | * 91 | * 92 | */ -------------------------------------------------------------------------------- /lib/Logs.py: -------------------------------------------------------------------------------- 1 | from AuthLog import AuthLog 2 | from DovecotLog import DovecotLog 3 | from Sql import Sql 4 | from Syslog import Syslog 5 | import time 6 | 7 | class Logs: 8 | TYPE_AUTH = 1 9 | TYPE_MAIL = 2 10 | TYPE_POSTFIX = 3 11 | TYPE_DOVECOT = 4 12 | 13 | 14 | def __init__(self): 15 | self.auth = [] 16 | self.mail = [] 17 | self.postfix = [] 18 | self.dovecot = [] 19 | self.num_logs = 0 20 | 21 | 22 | #set auth log 23 | def set(self, log, type): 24 | self.num_logs = self.num_logs + 1 25 | 26 | if ( type == Logs.TYPE_AUTH ): 27 | self.auth.append(AuthLog.toList(log)) 28 | return 0 29 | 30 | if ( type == Logs.TYPE_MAIL ): 31 | self.mail.append(log) 32 | return 0 33 | 34 | if ( type == Logs.TYPE_POSTFIX ): 35 | self.postfix.append(log) 36 | return 0 37 | 38 | if ( type == Logs.TYPE_DOVECOT ): 39 | self.dovecot.append(DovecotLog.toList(log)) 40 | 41 | return 0 42 | 43 | return 1 44 | 45 | 46 | def get(self): 47 | pass 48 | 49 | 50 | def insertToMySQL(self) : 51 | sql = ''' 52 | INSERT IGNORE INTO auth_logs 53 | (month, day, hour, domain, email, ip, log) 54 | VALUES (%s,%s,%s,%s,%s,%s,%s);''' 55 | try: 56 | if ( len(self.auth) > 0 ): 57 | Sql.insert(sql, self.auth) 58 | self.auth=[] 59 | 60 | except Exception as e : 61 | Syslog.write(e) 62 | 63 | sql = ''' 64 | INSERT IGNORE INTO dovecot_logs 65 | (month, day, hour, domain, email, msgid, log) 66 | VALUES (%s,%s,%s,%s,%s,%s,%s);''' 67 | try: 68 | if ( len(self.dovecot) > 0 ): 69 | Sql.insert(sql, self.dovecot) 70 | self.dovecot=[] 71 | 72 | except Exception as e : 73 | Syslog.write(e) 74 | 75 | sql = ''' 76 | INSERT IGNORE INTO postfix_logs 77 | (month, day, hour, mail_to, mail_to_domain, 78 | mail_from, mail_from_domain, status, msgid) 79 | VALUES (%s,%s,%s,%s,%s,%s,%s, %s, %s);''' 80 | try: 81 | if ( len(self.postfix) > 0 ): 82 | Sql.insert(sql, self.postfix) 83 | idx=0 84 | self.postfix=[] 85 | 86 | except Exception as e : 87 | Syslog.write(e) 88 | 89 | 90 | self.num_logs=0 91 | 92 | 93 | def clear(self): 94 | self.auth.clear() 95 | self.num_logs = 0 96 | 97 | 98 | def clearAuth(self): 99 | pass 100 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | LOG_PATH="/var/log/mail.log" 3 | 4 | 5 | echo " 6 | Sep 22 04:04:15 mail postfix/cleanup[50522]: QueueID: message-id= 7 | Sep 22 04:04:15 mail postfix/qmgr[861]: QueueID: from=, size=7028, nrcpt=1 (queue active) 8 | Sep 22 04:04:15 mail postfix/local[50524]: QueueID: to=, relay=local, delay=0.12, delays=0.12/0/0/0, dsn=2.0.0, status=sent 9 | Sep 22 04:04:15 mail postfix/qmgr[861]: QueueID: removed 10 | " >> $LOG_PATH 11 | # output: 12 | # *************************** 1. row *************************** 13 | # id: 681 14 | # month: Sep 15 | # day: 22 16 | # hour: 04 17 | # mail_to: user 18 | # mail_to_domain: test.ro 19 | # mail_from: user 20 | # mail_from_domain: test.ro 21 | # status: sent 22 | # msgid: MsgID 23 | 24 | 25 | 26 | echo " 27 | Sep 22 04:02:55 mail dovecot: auth-worker(2506): conn unix:auth-worker (pid=2420,uid=0): auth-worker<48865241>: sql(user1@test.ro,192.168.1.23,): Password mismatch 28 | Sep 22 02:05:00 mail dovecot: imap-login: Login: user=, method=PLAIN, rip=192.168.123, lip=192.168.1.3, pid=48597, TLS 29 | Sep 22 04:05:19 mail dovecot: pop3-login: Login: user=, method=PLAIN, rip=192.168.123, lip=192.168.1.3, pid=52386, TLS 30 | " >> $LOG_PATH 31 | # output: 32 | # *************************** 1. row *************************** 33 | # id: 185214 34 | # month: Sep 35 | # day: 22 36 | # hour: 04 37 | # domain: test.ro 38 | # ip: 192.168.1.23 39 | # email: user1 40 | # log: Password mismatch 41 | # *************************** 2. row *************************** 42 | # id: 185215 43 | # month: Sep 44 | # day: 22 45 | # hour: 02 46 | # domain: test.ro 47 | # ip: 192.168.123 48 | # email: user 49 | # log: logged in imap-login 50 | # *************************** 3. row *************************** 51 | # id: 185216 52 | # month: Sep 53 | # day: 22 54 | # hour: 04 55 | # domain: test.ro 56 | # ip: 192.168.123 57 | # email: user 58 | # log: logged in pop3-login 59 | 60 | 61 | 62 | echo " 63 | Sep 22 04:02:44 mail dovecot: pop3(user@test.ro)<16586>: expunge: box=INBOX, uid=216970, msgid=, size=271290 64 | Sep 22 04:02:55 mail dovecot: imap(user@test.ro)<51452>: delete: box=Drafts, uid=2942, msgid=, size=3122 65 | Sep 22 04:02:55 mail dovecot: imap(user@test.ro)<51452>: expunge: box=Drafts, uid=2942, msgid=, size=3122 66 | " >> $LOG_PATH 67 | # output: 68 | # *************************** 1. row *************************** 69 | # id: 2476 70 | # month: Sep 71 | # day: 22 72 | # hour: 04 73 | # domain: test.ro 74 | # email: user 75 | # msgid: MsdID 76 | # log: expunge: box=INBOX 77 | # *************************** 2. row *************************** 78 | # id: 2477 79 | # month: Sep 80 | # day: 22 81 | # hour: 04 82 | # domain: test.ro 83 | # email: user 84 | # msgid: MsgID 85 | # log: delete: box=Drafts 86 | # *************************** 3. row *************************** 87 | # id: 2478 88 | # month: Sep 89 | # day: 22 90 | # hour: 04 91 | # domain: test.ro 92 | # email: user 93 | # msgid: MsgID 94 | # log: expunge: box=Drafts 95 | 96 | 97 | mysql -p maillog <<'EOF' 98 | select * from postfix_logs \G 99 | select * from auth_logs \G 100 | select * from dovecot_logs \G 101 | EOF 102 | 103 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MailLog2MySQL 2 | MailLog2MySQL is a Python script for parsing and storing email server log (from Dovecot and Postfix) data into a MySQL database. This can be useful for monitoring email traffic, analyzing email server performance, and generating reports. 3 | 4 | Inspired by [mysqmail-dovecot-logger](https://packages.debian.org/stable/mail/mysqmail-dovecot-logger) and [mysqmail-postfix-logger](https://packages.debian.org/sid/mysqmail-postfix-logger) with some improvements and an API interface. 5 | 6 | 7 | ## Features 8 | - Parses email server log files in various formats. 9 | - Extracts relevant information such as sender, recipient, subject, date, and more. 10 | - Stores parsed data in a MySQL database for easy querying and analysis. 11 | - Customizable configuration for log file formats and database connection. 12 | 13 | # Installation 14 | 15 | ### mysql 16 | ``` 17 | CREATE DATABASE maillog; 18 | GRANT ALL PRIVILEGES ON maillog.* TO 'sammy'@'localhost' IDENTIFIED BY 'password'; 19 | FLUSH PRIVILEGES; 20 | 21 | ``` 22 | 23 | ### apache config 24 | ``` 25 | a2enmod mod rewrite 26 | ``` 27 | 28 | ``` 29 | create file /etc/apache2/sites-available/MailLog2MySQL.conf and add 30 | 31 | Listen 8888 32 | 33 | DocumentRoot /var/www/MailLog2MySQL 34 | 35 | 36 | Options Indexes FollowSymLinks MultiViews 37 | AllowOverride All 38 | Order allow,deny 39 | allow from all 40 | 41 | 42 | ErrorLog ${APACHE_LOG_DIR}/error.log 43 | CustomLog ${APACHE_LOG_DIR}/access.log combined 44 | 45 | ``` 46 | ``` 47 | a2ensite MailLog2MySQL.conf 48 | ``` 49 | 50 | ``` 51 | install and enable PHP 52 | ``` 53 | 54 | ``` 55 | apt install python3-pymysql 56 | apt install php-mysql 57 | ``` 58 | 59 | ### install MailLog2MySQL 60 | ``` 61 | make install 62 | 63 | ``` 64 | 65 | 66 | 67 | ### create /etc/rc.local or add line /etc/init.d/maillog2mysql start 68 | ``` 69 | #!/bin/sh -e 70 | /etc/init.d/maillog2mysql start 71 | exit 0 72 | ``` 73 | ``` 74 | chmod +x /etc/rc.local 75 | ``` 76 | 77 | ### run 78 | ``` 79 | /etc/init.d/maillog2mysql start 80 | ``` 81 | 82 | ### config 83 | ``` 84 | cat /etc/MailLog2MySQL.conf 85 | 86 | #mysql settings 87 | host = localhost 88 | user = sammy 89 | password = password 90 | database = maillog 91 | port = 3306 92 | 93 | #read logs 94 | log_file = /var/log/mail.log 95 | 96 | #api access 97 | access = ["127.0.0.1", "192.168.1.1"] 98 | ``` 99 | 100 | # Example API 101 | ``` 102 | curl http://127.0.0.1:8888/api?table=dovecot_logs | jq . 103 | % Total % Received % Xferd Average Speed Time Time Time Current 104 | Dload Upload Total Spent Left Speed 105 | 100 3793 100 3793 0 0 1136k 0 --:--:-- --:--:-- --:--:-- 1234k 106 | { 107 | "number of logs": 710, 108 | "number of pages": 36, 109 | "data": [ 110 | { 111 | "id": 1, 112 | "month": "Mar", 113 | "day": "12", 114 | "hour": "08", 115 | "domain": "test.ro", 116 | "email": "dan", 117 | "msgid": "fdsajfnasfqq123412@mailtest2.ro", 118 | "log": "delete: box=INBOX" 119 | }, 120 | { 121 | "id": 2, 122 | "month": "Mar", 123 | "day": "13", 124 | "hour": "09", 125 | "domain": "test.ro", 126 | "email": "dan", 127 | "msgid": "fdsajfnasfqq123412@mailtest2.ro", 128 | "log": "expunge: box=INBOX" 129 | }, 130 | 131 | curl http://127.0.0.1:8888/api?table=dovecot_logs&page=21 132 | curl http://127.0.0.1:8888/api?table=dovecot_logs&month=Mar&day=12&hour=08&domain=test.ro&email=dan 133 | 134 | 135 | ``` 136 | ``` 137 | curl http://127.0.0.1:8888/api?table=auth_logs | jq . 138 | % Total % Received % Xferd Average Speed Time Time Time Current 139 | Dload Upload Total Spent Left Speed 140 | 100 2900 100 2900 0 0 306k 0 --:--:-- --:--:-- --:--:-- 314k 141 | { 142 | "number of logs": 2669, 143 | "number of pages": 133, 144 | "data": [ 145 | { 146 | "id": 1, 147 | "month": "Mar", 148 | "day": "12", 149 | "hour": "10", 150 | "domain": "test.ro", 151 | "ip": "192.168.1.1", 152 | "email": "dan2", 153 | "log": "unknown user" 154 | }, 155 | { 156 | "id": 2, 157 | "month": "Mar", 158 | "day": "12", 159 | "hour": "10", 160 | "domain": "test.ro", 161 | "ip": "192.168.1.1", 162 | "email": "dan", 163 | "log": "logged in imap-login" 164 | }, 165 | ``` 166 | ``` 167 | curl http://127.0.0.1:8888/api?table=postfix_logs | jq . 168 | % Total % Received % Xferd Average Speed Time Time Time Current 169 | Dload Upload Total Spent Left Speed 170 | 100 6287 100 6287 0 0 403k 0 --:--:-- --:--:-- --:--:-- 409k 171 | { 172 | "number of logs": 1416, 173 | "number of pages": 71, 174 | "data": [ 175 | { 176 | "id": 1, 177 | "month": "Mar", 178 | "day": "13", 179 | "hour": "00", 180 | "mail_to": "dan3", 181 | "mail_to_domain": "test.ro", 182 | "mail_from": "dan", 183 | "mail_from_domain": "test.ro", 184 | "status": "sent", 185 | "msgid": "jbhfdsan32@mail.test.ro" 186 | }, 187 | 188 | ``` 189 | 190 | ### Delete 191 | 192 | 193 | ``` 194 | make delete 195 | or 196 | rm /usr/local/bin/MailLog2MySQL 197 | rm -rf /usr/lib/MailLog2MySQL 198 | rm -rf /var/www/MailLog2MySQL 199 | rm /etc/MailLog2MySQL.conf 200 | rm /etc/init.d/maillog2mysql 201 | ``` 202 | 203 | ``` 204 | a2dissite MailLog2MySQL.conf 205 | 206 | remove /etc/apache2/sites-available/MailLog2MySQL.conf 207 | 208 | delete database 209 | 210 | delete /etc/rc.local or delete line /etc/init.d/maillog2mysql start 211 | ``` 212 | -------------------------------------------------------------------------------- /lib/Queue.py: -------------------------------------------------------------------------------- 1 | import time 2 | from Logs import Logs 3 | from Messages import Messages 4 | 5 | class Queue: 6 | q={} 7 | AVAILABLE = 5*60*60 #5 hour 8 | LIMIT = 1000 #maximum number of objects in Queue 9 | NUM_LOGS = 0 10 | 11 | def __init__ (self, id, timeout, month, day, hour): 12 | 13 | self.id = id 14 | self.month = month 15 | self.day = day 16 | self.hour = hour 17 | self.timeout = timeout 18 | self.To = [] 19 | self.ToDomain = [] 20 | self.From = None 21 | self.FromDomain = None 22 | self.Status = [] 23 | self.msgID = "None" 24 | self.Subject = None 25 | self.toInsert = False 26 | Queue.NUM_LOGS = Queue.NUM_LOGS + 1 27 | 28 | 29 | 30 | 31 | def insertTo(self, To, Status): 32 | self.Status.append(Status) 33 | loc = To.find('@') 34 | if (loc == -1): 35 | self.To.append(To[:loc]) 36 | self.ToDomain.append("") 37 | return 38 | 39 | self.To.append(To[:loc]) 40 | self.ToDomain.append(To[loc+1:]) 41 | 42 | 43 | 44 | def insertFrom(self, From): 45 | loc = From.find('@') 46 | if (loc == -1): 47 | self.From = From 48 | self.FromDomain = "" 49 | return 50 | 51 | self.From = From[:loc] 52 | self.FromDomain = From[loc+1:] 53 | 54 | 55 | 56 | def insertToFrom(self, To, From, Status): 57 | self.insertTo(To, Status) 58 | self.insertFrom(From) 59 | 60 | 61 | 62 | def insertMsgID(self, msgID): 63 | if (len(msgID)> 254 ): 64 | msgID=msgID[:254] 65 | self.msgID = msgID 66 | 67 | 68 | 69 | def toList(self): 70 | response = [] 71 | for i in range(0, len(self.To) ): 72 | response.append([ 73 | self.month, 74 | self.day, 75 | self.hour, 76 | self.To[i], 77 | self.ToDomain[i], 78 | self.From, 79 | self.FromDomain, 80 | self.Status[i], 81 | self.msgID, 82 | ]) 83 | 84 | return response 85 | 86 | 87 | @staticmethod 88 | def add(id, month, day, hour): 89 | Queue.limit() 90 | if (id in Queue.q): 91 | return 0 92 | Queue.q[id] = Queue( 93 | id, 94 | int ( time.mktime(time.localtime() ) ) + Queue.AVAILABLE, 95 | month, 96 | day, 97 | hour 98 | ) 99 | return 0 100 | 101 | 102 | 103 | @staticmethod 104 | def removeOld(): 105 | time = int ( time.mktime(time.localtime() ) ) + Queue.AVAILABLE 106 | for k in Queue.q: 107 | if ( Queue.q[k] < time ): 108 | del Queue.q[k] 109 | 110 | 111 | @staticmethod 112 | def remove(id): 113 | try: 114 | Queue.q[id].toInsert = True 115 | except: 116 | pass 117 | return 0 118 | 119 | 120 | @staticmethod 121 | def addFrom(id, From): 122 | try: 123 | Queue.q[id].insertFrom(From) 124 | except Exception as e: 125 | pass 126 | return 0 127 | 128 | 129 | @staticmethod 130 | def addTo(id, To, Status): 131 | try: 132 | Queue.q[id].insertTo(To, Status) 133 | except Exception as e: 134 | pass 135 | return 0 136 | 137 | 138 | 139 | @staticmethod 140 | def addToFrom(id, To, From, Status): 141 | try: 142 | Queue.q[id].insertToFrom(To, From, Status) 143 | Queue.remove(id) 144 | except Exception as e: 145 | #print(e) 146 | pass 147 | return 0 148 | 149 | 150 | 151 | @staticmethod 152 | def addMsgID(id, msgID): 153 | if (id not in Queue.q): 154 | return 0 155 | 156 | if (len(msgID)> 254 ): 157 | msgID=msgID[:254] 158 | 159 | #add and check if msgid already exists with different queue id 160 | if (Messages.addID(msgID) == 0): 161 | del Queue.q[id] 162 | Queue.NUM_LOGS = Queue.NUM_LOGS - 1 163 | return 0 164 | 165 | try: 166 | Queue.q[id].insertMsgID(msgID) 167 | except Exception as e: 168 | pass 169 | 170 | return 0 171 | 172 | 173 | 174 | @staticmethod 175 | def print(): 176 | print("-----------Queue----------") 177 | for id in Queue.q: 178 | print(Queue.q[id].id) 179 | print(Queue.q[id].From) 180 | print(Queue.q[id].FromDomain) 181 | print(Queue.q[id].To) 182 | print(Queue.q[id].ToDomain) 183 | print(Queue.q[id].Status) 184 | print(Queue.q[id].msgID) 185 | print(Queue.q[id].timeout) 186 | print(Queue.q[id].toInsert) 187 | print("-------------------------") 188 | 189 | 190 | 191 | @staticmethod 192 | def preareToInsert(logs): 193 | toDelete = [] 194 | currentTime = time.time() 195 | 196 | for id in Queue.q: 197 | if (currentTime - Queue.q[id].timeout > 0 ): 198 | #if expired, insert with status unknown 199 | Queue.q[id].Status = "unknown" 200 | Queue.q[id].toInsert= True 201 | 202 | if ( Queue.q[id].toInsert != True ): 203 | continue 204 | 205 | for data in Queue.q[id].toList(): 206 | logs.set(data, logs.TYPE_POSTFIX ) 207 | 208 | toDelete.append(id) 209 | 210 | 211 | #delete from queue 212 | for id in toDelete: 213 | Messages.delete(Queue.q[id].msgID) 214 | del Queue.q[id] 215 | Queue.NUM_LOGS = Queue.NUM_LOGS - 1 216 | 217 | 218 | 219 | 220 | @staticmethod 221 | def limit(): 222 | if ( Queue.NUM_LOGS > Queue.LIMIT ): 223 | try: 224 | # remove first 10 elements, if queue is to large 225 | #print( Queue.NUM_LOGS) 226 | #print( Queue.LIMIT) 227 | for i in range(1,10) : 228 | id = next(iter(Queue.q)) 229 | Queue.q[id].Status = "unknown" 230 | Queue.q[id].toInsert= True 231 | # Queue.q.pop(next(iter(Queue.q))) 232 | 233 | Queue.preareToInsert() 234 | 235 | except Exception as e: 236 | pass 237 | return 1 238 | 239 | return 0 240 | 241 | 242 | -------------------------------------------------------------------------------- /lib/ProcessLine.py: -------------------------------------------------------------------------------- 1 | from AuthLog import AuthLog 2 | from DovecotLog import DovecotLog 3 | from Logs import Logs 4 | from Queue import Queue 5 | 6 | class ProcessLine: 7 | 8 | @staticmethod 9 | def main(line, logs): 10 | words = line.split(" ") 11 | len_words = len(words) 12 | 13 | try: 14 | if ( len(words) < 5 ): 15 | return 1 16 | 17 | if (words[5][:11] == "auth-worker" or 18 | words[5][:11] == "imap-login:" or 19 | words[5][:11] == "pop3-login:" or 20 | words[5][:17] == "managesieve-login"): 21 | return ProcessLine.auth_log(words, len_words, logs ) 22 | 23 | if (words[4][:7] == "dovecot"): 24 | return ProcessLine.dovecot_log(words, len_words, logs ) 25 | 26 | if (words[4][:7] == "postfix"): 27 | return ProcessLine.postfix_log(words, len_words, logs) 28 | except: 29 | pass 30 | return 1 31 | 32 | 33 | 34 | @staticmethod 35 | def auth_log(words, len_words, logs ): 36 | while(len_words == 13): 37 | if ( words[10][0:4] != "sql(" ): 38 | break 39 | 40 | aux_word = words[10].split(",") 41 | loc = aux_word[0].find('@') 42 | 43 | flag1 = True 44 | if (len(aux_word) < 2 ): 45 | if ( words[10].find(")") == -1 ): 46 | break 47 | else: 48 | flag1 = False 49 | 50 | flag2 = True 51 | if (loc == -1 ): 52 | flag2 = False 53 | 54 | authLog = AuthLog() 55 | authLog.month = words[0][:3] 56 | authLog.day = words[1][:3] 57 | authLog.hour = words[2].split(":")[0][:2] 58 | authLog.email = aux_word[0][4:loc][:64] 59 | 60 | if (flag2 == True): 61 | authLog.domain = aux_word[0][loc+1:][:124] 62 | 63 | authLog.ip = "" 64 | if ( flag1 == True ): 65 | authLog.ip = aux_word[1].split(")")[0][:64] 66 | authLog.domain = aux_word[0][loc+1:][:124] 67 | else: 68 | authLog.domain = aux_word[0][loc+1:].split(")")[0][:124] 69 | 70 | if (words[11]=="Password" and words[12][0:8]=="mismatch" ): 71 | authLog.log = "Password mismatch" 72 | return logs.set(authLog, logs.TYPE_AUTH) 73 | 74 | if (words[11]=="unknown" and words[12][0:4]== "user"): 75 | authLog.log = "unknown user" 76 | return logs.set(authLog, logs.TYPE_AUTH) 77 | break 78 | 79 | while (len_words == 13): 80 | if (words[6] != "Login:"): 81 | break 82 | if (words[7][:6]!= "user=<"): 83 | break 84 | if (words[9][:4]!= "rip="): 85 | break 86 | 87 | authLog = AuthLog() 88 | authLog.month = words[0][:3] 89 | authLog.day = words[1][:2] 90 | authLog.hour = words[2].split(":")[0][:2] 91 | 92 | loc = words[7].find("@") 93 | if ( loc == -1 ): 94 | authLog.email = words[7][6:-2][:64] 95 | authLog.domain = "" 96 | else: 97 | authLog.email = words[7][ 6:loc][:64] 98 | authLog.domain = words[7][loc+1:-2][:124] 99 | 100 | authLog.ip = words[9][4:-1][:64] 101 | authLog.log = "logged in "+ words[5][:-1] 102 | return logs.set(authLog, logs.TYPE_AUTH) 103 | 104 | return 1 105 | 106 | 107 | 108 | @staticmethod 109 | def dovecot_log(words, len_words, logs): 110 | while( len_words > 9 ): 111 | if ( words[5][:5] != "imap(" and words[5][:5] != "pop3("): 112 | break 113 | 114 | if ( words[-2][:7] != "msgid=<"): 115 | break 116 | 117 | loc = words[5].find('@') 118 | dovecotLog = DovecotLog() 119 | dovecotLog.month = words[0][:3] 120 | dovecotLog.day = words[1][:2] 121 | dovecotLog.hour = words[2].split(":")[0][:2] 122 | 123 | if ( loc == -1 ): 124 | dovecotLog.email = words[5][5:].split(")")[0][:64] 125 | else: 126 | dovecotLog.email = words[5][5:].split("@")[0][:64] 127 | dovecotLog.domain = words[5][5:].split("@")[1].split(")")[0][:124] 128 | 129 | dovecotLog.msgid = words[-2][7:7+64][:-2] 130 | dovecotLog.log = " ".join(words[6:-3])[:64][:-1] 131 | return logs.set(dovecotLog, logs.TYPE_DOVECOT ) 132 | 133 | return 1 134 | 135 | 136 | 137 | @staticmethod 138 | def postfix_log(words, len_words, logs): 139 | 140 | # if (words[6][:7] == "client="): 141 | # return Queue.add(words[5][:-1], words[0], words[1], words[2].split(":")[0]) 142 | 143 | if (words[6][:7] == "removed"): 144 | return Queue.remove(words[5][:-1]) 145 | 146 | if (words[6][:12] == "message-id=<"): 147 | Queue.add(words[5][:-1], words[0], words[1], words[2].split(":")[0]) 148 | return Queue.addMsgID(words[5][:-1], words[6][12:-2] ) 149 | 150 | 151 | if (words[6][0:6] == "from=<"): 152 | return Queue.addFrom(words[5][:-1], words[6][6:-2]) 153 | 154 | if (words[6][0:4] == "to=<"): 155 | i = 7 156 | while True: 157 | if ( i == len_words ): 158 | break 159 | 160 | if (words[i][0:7] == "status="): 161 | return Queue.addTo(words[5][:-1], words[6][4:-2], words[i][7:].split("\n")[0]) 162 | 163 | i=i+1 164 | 165 | if (words[6] == "milter-reject:" and 166 | words[-4][:6] == "from=<" and 167 | words[-3][:4] == "to=<" ): 168 | return Queue.addToFrom(words[5][:-1], words[-3][4:-1], words[-4][6:-1], words[14][:-1].split("\n")[0] ) 169 | 170 | 171 | if (words[5] == "NOQUEUE:" and 172 | words[-4][:6] == "from=<" and 173 | words[-3][:4] == "to=<" ): 174 | 175 | loc = words[-3].find("@") 176 | if (loc == -1 ): 177 | To = words[-3][4:-1] 178 | ToDomain = "" 179 | else: 180 | To = words[-3][4:loc] 181 | ToDomain = words[-3][loc+1:-1] 182 | 183 | loc = words[-4].find("@") 184 | if (loc == -1 ): 185 | From = words[-4][6:-1] 186 | FromDomain = "" 187 | else: 188 | From = words[-4][6:loc] 189 | FromDomain = words[-4][loc+1:-1] 190 | NOQUEUE = [ 191 | words[0], 192 | words[1], 193 | words[2].split(":")[0], 194 | To, 195 | ToDomain, 196 | From, 197 | FromDomain, 198 | " ".join(words[6:-4]), 199 | " - " 200 | ] 201 | return logs.set(NOQUEUE, logs.TYPE_POSTFIX ) 202 | return 1 203 | --------------------------------------------------------------------------------