├── README.md ├── latestMailQueue.py └── mailConf.ini /README.md: -------------------------------------------------------------------------------- 1 | # log_error_sendmail 2 | 监控日志，分析日志当发现错误时，发邮件，实现实时监控 3 | -------------------------------------------------------------------------------- /latestMailQueue.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding:utf-8 -*- 3 | #author:liugao,516587331@qq.com 4 | #desc: read config to send mail by python 5 | #created on 2016/04/25 6 | 7 | import sys,os 8 | import ConfigParser 9 | import linecache 10 | import math 11 | 12 | import signal 13 | import time 14 | import re 15 | from multiprocessing import Process,Queue,Pool 16 | import multiprocessing 17 | #from multiprocessing import Process,Queue 18 | 19 | import smtplib 20 | from email.mime.text import MIMEText 21 | from email.header import Header 22 | 23 | #strPattern='\[(.*)\]\s+<(.*?)>\s+[Ff]ailed to decode(.*?)' 24 | #logZoneName = "GameZoneServer_" 25 | #logPostfix = ".log" 26 | #dateFormat = '%Y-%m-%d' 27 | 28 | # 29 | g_strGlobal = "Global" 30 | g_strInfoSize = "InfoSize" 31 | g_strMonSize = "servermonitor" 32 | g_strUserInfo = "UserInfo_%d" 33 | g_UsrName="Username" 34 | g_strPwd="Password" 35 | g_strStmp="MailStmp" 36 | g_strMailinfo="MailInfo_%d" 37 | g_strFrom="From" 38 | g_strTo="To" 39 | g_strEncoding="Encoding" 40 | g_strServerinfo="ServerInfo_%d" 41 | g_strServerID="ServerID" 42 | g_strOnceSendlins="OnceSendLines" 43 | g_strLoginfo="logInfo_%d" 44 | g_strLogPath = "LogPath" 45 | g_strLogName= "LogName" 46 | g_strDateFormat= "DateFormat" 47 | g_strLogPostfix= "LogPostfix" 48 | g_strOnceReadlines= "OnceReadLines" 49 | g_strHasReadlines= "hasreadlines" 50 | g_strRegpattern= "RegPattern" 51 | g_strHasReadDate= "hasreaddate" 52 | 53 | 54 | 55 | #input msg into Queue 56 | def write(q,lock,linetext): 57 | if linetext: 58 | lock.acquire() #add lock 59 | q.put(linetext) 60 | lock.release() #release lock 61 | 62 | #you can define your email strategy with mutiprocess. 63 | #for some reason,some email service providers might forbit your email address, 64 | #so you had to use serval email address. 65 | #lineline:[4](index)[10](linenum) 66 | def read(q,list_mailobj,InfoSize,list_indexlinenum,cfgobj): 67 | iCount = 0 68 | bSendMail = 0 69 | while True: 70 | index = iCount % InfoSize 71 | iGetCount = 0 72 | strContext = '' 73 | time.sleep(10) 74 | print "read" 75 | MaxSendlines = list_mailobj[index].getSendLines() 76 | bSendMail = 0 77 | while not q.empty(): 78 | 79 | #set read line limited,so that send many lines by one email (add on 2016/04/28 by liugao) 80 | lineline = q.get(False) 81 | if lineline: 82 | iIndex = int(lineline[0:4]) 83 | iLineNum = int(lineline[4:14]) 84 | print iIndex,iLineNum 85 | #write linenum 86 | if iLineNum > list_indexlinenum[iIndex]: 87 | list_indexlinenum[iIndex] = iLineNum + 1 88 | lineline = lineline[14:] 89 | iGetCount += 1 90 | strContext += lineline 91 | else: 92 | print 'context is null' 93 | #iGetCount reach max sendlines ,then send mail right now and sleep some seconds (add on 2016/04/28 by liugao) 94 | # 95 | #print "Read PID:"+str(os.getpid())+"index:"+str(index) + "now:"+str(iGetCount)+"sendlines:"+str(MaxSendlines) 96 | if iGetCount == int(MaxSendlines): 97 | try: 98 | list_mailobj[index].setsubject('Server Error') 99 | linelinetext = "ServerID: " + list_mailobj[index].getSeverID() + "\n"+ strContext + "\nnow :" +str(time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time()))) 100 | list_mailobj[index].settext(linelinetext) 101 | list_mailobj[index].sendmail() 102 | list_mailobj[index].clearContext() 103 | linelinetext = '' 104 | bSendMail = 1 105 | except Execption,e: 106 | bSendMail = 0 107 | print str(e) 108 | 109 | #print "####################:",str(bSendMail) 110 | if bSendMail: 111 | for i in range(len(list_indexlinenum)): 112 | DateFormat=cfgobj.get(g_strLoginfo%i,g_strDateFormat) 113 | strNow = str(time.strftime(DateFormat,time.localtime(time.time()))) 114 | cfgobj.setWrite(g_strLoginfo%i,g_strHasReadDate,strNow) 115 | cfgobj.setWrite(g_strLoginfo%i,g_strHasReadlines,str(list_indexlinenum[i])) 116 | else : 117 | print "else list_indexlinenum",len(list_indexlinenum) 118 | #print "max:",iGetCount 119 | iGetCount = 0 120 | strContext = '' 121 | #wait 10s and send mail 122 | time.sleep(10) 123 | break 124 | 125 | # if iGetCunt less oncesendlines and queue is empty 126 | if iGetCount > 0 and iGetCount < int(MaxSendlines): 127 | # send mail rightnow 128 | try: 129 | list_mailobj[index].setsubject('Server Error') 130 | linelinetext = "ServerID: " + list_mailobj[index].getSeverID() + "\n"+ strContext + "\nnow :" +str(time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time()))) 131 | list_mailobj[index].settext(linelinetext) 132 | list_mailobj[index].sendmail() 133 | list_mailobj[index].clearContext() 134 | linelinetext = '' 135 | bSendMail = 1 136 | except Execption,e: 137 | bSendMail = 0 138 | print str(e) 139 | 140 | #print "####################:",str(bSendMail) 141 | if bSendMail: 142 | for i in range(len(list_indexlinenum)): 143 | DateFormat=cfgobj.get(g_strLoginfo%i,g_strDateFormat) 144 | strNow = str(time.strftime(DateFormat,time.localtime(time.time()))) 145 | cfgobj.set(g_strLoginfo%i,g_strHasReadDate,strNow) 146 | cfgobj.setWrite(g_strLoginfo%i,g_strHasReadlines,str(list_indexlinenum[i])) 147 | else : 148 | print "else list_indexlinenum",len(list_indexlinenum) 149 | 150 | #print "other:",iGetCount,MaxSendlines 151 | iGetCount = 0 152 | strContext = '' 153 | 154 | iCount += 1 155 | 156 | class ReadConf: 157 | def __init__(self,config_file_path): 158 | self.path = config_file_path 159 | self.cf = ConfigParser.ConfigParser() 160 | try: 161 | self.cf.read(self.path) 162 | except: 163 | print 'read config file failed' 164 | sys.exit(1) 165 | 166 | def checkSection(self): 167 | if not self.cf.has_section(g_strGlobal): 168 | return False 169 | if not self.cf.has_option(g_strGlobal,g_strInfoSize): 170 | return False 171 | if not self.cf.has_option(g_strGlobal,g_strMonSize): 172 | return False 173 | 174 | infosize = self.cf.getint(g_strGlobal,g_strInfoSize) 175 | monitorsize = self.cf.getint(g_strGlobal,g_strMonSize) 176 | print infosize,monitorsize 177 | for i in range(infosize): 178 | if not self.cf.has_section(g_strUserInfo%i): 179 | print "has no %s"%g_strUserInfo 180 | return False 181 | if not self.cf.has_option(g_strUserInfo%i,g_UsrName): 182 | print "has no %s"%g_UsrName 183 | return False 184 | if not self.cf.has_option(g_strUserInfo%i,g_strPwd): 185 | print "has no %s"%g_strPwd 186 | return False 187 | if not self.cf.has_option(g_strUserInfo%i,g_strStmp): 188 | print "has no %s"%g_strStmp 189 | return False 190 | 191 | if not self.cf.has_section(g_strMailinfo%i): 192 | print "has no %s"%g_strMailinfo 193 | return False 194 | if not self.cf.has_option(g_strMailinfo%i,g_strFrom): 195 | print "has no %s"%g_strFrom 196 | return False 197 | if not self.cf.has_option(g_strMailinfo%i,g_strTo): 198 | print "has no %s"%g_strTo 199 | return False 200 | if not self.cf.has_option(g_strMailinfo%i,g_strEncoding): 201 | print "has no %s"%g_strEncoding 202 | return False 203 | 204 | if not self.cf.has_section(g_strServerinfo%i): 205 | print "has no %s"%g_strServerinfo 206 | return False 207 | if not self.cf.has_option(g_strServerinfo%i,g_strServerID): 208 | print "has no %s"%g_strServerID 209 | return False 210 | if not self.cf.has_option(g_strServerinfo%i,g_strOnceSendlins): 211 | print "has no %s"%g_strOnceSendlins 212 | return False 213 | print "check mail info success" 214 | for j in range(monitorsize): 215 | if not self.cf.has_section(g_strLoginfo%j): 216 | print "has no %s,%d"%(g_strLoginfo,j) 217 | return False 218 | if not self.cf.has_option(g_strLoginfo%j,g_strLogPath): 219 | print "has no %s"%g_strLogPath 220 | return False 221 | if not self.cf.has_option(g_strLoginfo%j,g_strLogName): 222 | print "has no %s"%g_strLogName 223 | return False 224 | if not self.cf.has_option(g_strLoginfo%j,g_strDateFormat): 225 | print "has no %s"%g_strDateFormat 226 | return False 227 | if not self.cf.has_option(g_strLoginfo%j,g_strLogPostfix): 228 | print "has no %s"%g_strLogPostfix 229 | return False 230 | if not self.cf.has_option(g_strLoginfo%j,g_strOnceReadlines): 231 | print "has no %s"%g_strOnceReadlines 232 | return False 233 | if not self.cf.has_option(g_strLoginfo%j,g_strHasReadlines): 234 | print "has no %s"%g_strHasReadlines 235 | return False 236 | if not self.cf.has_option(g_strLoginfo%j,g_strRegpattern): 237 | print "has no %s"%g_strRegpattern 238 | return False 239 | if not self.cf.has_option(g_strLoginfo%j,g_strHasReadDate): 240 | print "has no %s"%g_strHasReadDate 241 | return False 242 | return True 243 | 244 | def get(self,field,key): 245 | result="" 246 | try: 247 | result = self.cf.get(field,key) 248 | except: 249 | result="" 250 | return result; 251 | def getInt(self,field,key): 252 | iResult = 0 253 | try: 254 | iResult = self.cf.getint(field,key) 255 | except: 256 | iResult = 0 257 | return iResult 258 | def set(self,section,option,value): 259 | try: 260 | self.cf.set(section,option,value) 261 | except Exception,e: 262 | print str(e) 263 | 264 | def setWrite(self,section,option,value): 265 | try: 266 | self.cf.set(section,option,value) 267 | print self.cf.get(section,option) 268 | self.cf.write(open(self.path,'w')) 269 | except Exception,e: 270 | print str(e) 271 | 272 | 273 | 274 | 275 | class MailHelper: 276 | def __init__(self,Subject,Username,Pwd,Stmp,From,To,ServerID,Encoding,OnceSendLines): 277 | self.subject = Subject 278 | self.username = Username 279 | self.pwd = Pwd 280 | self.stmp = Stmp 281 | self.sender = From 282 | self.recevier = To 283 | self.encoding = Encoding 284 | self.ServerID = ServerID 285 | self.onceSendLines = OnceSendLines 286 | 287 | def getSendLines(self): 288 | return self.onceSendLines 289 | 290 | def getSubject(self): 291 | return self.subject 292 | 293 | def getUsername(self): 294 | return self.username 295 | 296 | def getPwd(self): 297 | return self.pwd 298 | 299 | def getSeverID(self): 300 | return self.ServerID 301 | 302 | def clearContext(self): 303 | self.text = '' 304 | self.subject='' 305 | 306 | def settext(self,strText): 307 | self.text = strText 308 | 309 | def setsubject(self,strSubject): 310 | self.subject = strSubject 311 | 312 | def sendmail(self): 313 | msg = MIMEText(_text=self.text, _charset=self.encoding) #encode using utf-8 314 | msg['Subject'] = Header(self.subject, self.encoding) 315 | msg['From'] = self.sender 316 | msg['To'] = self.recevier 317 | try: 318 | smtp = smtplib.SMTP() 319 | smtp.connect(self.stmp) 320 | smtp.ehlo() 321 | smtp.starttls() 322 | smtp.ehlo() 323 | smtp.set_debuglevel(1) 324 | smtp.login(self.username, self.pwd) 325 | listRecevier = self.recevier.split(',') 326 | smtp.sendmail(self.sender, listRecevier, msg.as_string()) 327 | except: 328 | #smtp.close() 329 | self.text = '' 330 | self.subject='' 331 | print "Send mail failed" 332 | smtp.quit() 333 | 334 | #read log file per 5s 335 | class MonitorLog: 336 | def __init__(self,index,OnceReadLines,zoneLogPath,contentQueue,lock,ReadLines): 337 | self.index = index 338 | self.linespilte = OnceReadLines 339 | self.logZonePath = zoneLogPath 340 | self.canSend = 0 341 | self.dateFormat = '' 342 | self.logZoneName = '' 343 | self.logPostfix = '' 344 | self.contentQueue = contentQueue 345 | self.lock = lock 346 | self.regPattern = '' 347 | self.readlines = ReadLines 348 | 349 | def setRegPattern(self,strPattern): 350 | self.regPattern = strPattern 351 | #send mailor not 352 | def setSendMailFlag(self,bCanSend): 353 | self.canSend = bCanSend 354 | 355 | def setDateFormat(self,strFormat): 356 | self.dateFormat = strFormat 357 | 358 | def setLogName(self,strlogName): 359 | self.logZoneName = strlogName 360 | 361 | def GetLogName(self): 362 | return self.logZoneName 363 | 364 | def setLogPostfix(self,strlogfix): 365 | self.logPostfix = strlogfix 366 | 367 | # 368 | def monitor(self): 369 | #print 'filename %s' % self.logZonePath 370 | filename="" 371 | #had read lines 372 | count = int(self.readlines) 373 | # 374 | waittime = 1.25 375 | waittimescount = 0 376 | print 'monitor(self)' + str(os.getpid()) 377 | while True: 378 | # get date 379 | thistime = time.strftime(self.dateFormat, time.localtime(time.time())) 380 | currfilename = self.logZonePath + "/" + self.logZoneName+ str(thistime)+ self.logPostfix 381 | 382 | if not os.path.exists(currfilename) : 383 | print "%s not exist,please check path of config file." % currfilename 384 | sleeptime = waittime**waittimescount 385 | #print sleeptime,waittimescount,int(sleeptime) 386 | time.sleep(int(sleeptime)) 387 | waittimescount += 1 388 | continue 389 | else : 390 | waittimescount = 0 391 | 392 | print currfilename 393 | if filename == "" : 394 | filename = currfilename 395 | 396 | if filename != "" and filename != currfilename: 397 | count=0 398 | filename = currfilename 399 | linecache.clearcache() 400 | #fileobject = open(filename,'r') 401 | cache_data = linecache.getlines(filename) 402 | cachelines = len(cache_data) 403 | #print "Lines: " + str(cachelines) + " Now:"+str(time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time()))) 404 | #all_the_text = file_object.read() 405 | #if count == (cachelines - 1): 406 | # continue 407 | #print "====================:",count 408 | if count > cachelines: 409 | count = cachelines 410 | for lineNum in range(count,cachelines): 411 | lineline = cache_data[lineNum] 412 | #print lineline 413 | if lineline: 414 | # 415 | pattern = re.compile(self.regPattern) 416 | items = re.findall(pattern,lineline) 417 | if items: 418 | for item in items: 419 | print item[0],item[1] 420 | #send email to notify someone 421 | print "lineNum:" ,lineNum 422 | #for itimes in range(2): 423 | if self.canSend : 424 | ''' 425 | try : 426 | self.mailobj.setsubject('Server Error') 427 | linelinetext = "ServerID: " + self.serverID + "\n"+ lineline + "\nnow :" +str(time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time()))) 428 | self.mailobj.settext(linelinetext) 429 | self.mailobj.sendmail() 430 | self.mailobj.clearContext() 431 | linelinetext = "" 432 | except Exception,e: 433 | print str(e) 434 | time.sleep(5) 435 | ''' 436 | linelinetext = '' 437 | strlineIndex = '' 438 | lineCount = '' 439 | strlineIndex = str(self.index) 440 | #print "strlineIndex:",strlineIndex 441 | strlineIndex = strlineIndex.zfill(4) 442 | #print strlineIndex2 443 | iCounLineNum = lineNum 444 | lineCount = str(iCounLineNum) 445 | lineCount = lineCount.zfill(10) 446 | linelinetext = strlineIndex + lineCount + self.logZoneName +"--"+lineline 447 | print linelinetext 448 | self.lock.acquire() #add lock 449 | self.contentQueue.put(linelinetext) 450 | lineline = '' 451 | self.lock.release() #release lock 452 | 453 | #write(self.contentQueue,self.lock,lineline) 454 | #itimes += 1 455 | #print "times :" + str(itimes) 456 | else: 457 | print lineNum, " Not Found" 458 | else: 459 | print "now line is null" 460 | 461 | 462 | if lineNum > 0 and ((lineNum+1) % self.linespilte == 0): 463 | count = lineNum + 1 464 | break; 465 | 466 | # 467 | endlinenum = cachelines - 1 468 | #print str(lineNum)+":::::"+str(endlinenum) 469 | if int(lineNum) == int(endlinenum): 470 | count = cachelines 471 | break 472 | lineNum += 1 473 | 474 | #except Exception,e: 475 | #print str(e) 476 | #fileobject.close() 477 | time.sleep(5) 478 | linecache.clearcache() 479 | 480 | def monitorlogprocess(monitorobjprc): 481 | monitorobjprc.monitor() 482 | 483 | if __name__ == "__main__": 484 | 485 | ConfFile = "mailConf.ini" 486 | if not os.path.exists(ConfFile) : 487 | print "Config file not exist,please check path of config file." 488 | sys.exit(1) 489 | 490 | # get configuration infomation 491 | cfgobj = ReadConf(ConfFile) 492 | if not cfgobj.checkSection(): 493 | print "check cfg fialed,please check ini" 494 | sys.exit(1) 495 | else : 496 | print "check ini config file success" 497 | #sys.exit(1) 498 | 499 | #get mail service providers number 500 | InfoSize = cfgobj.getInt(g_strGlobal,g_strInfoSize) 501 | ServerMonitor = cfgobj.getInt(g_strGlobal,g_strMonSize) 502 | list_mailobj = [] 503 | list_indexlinenum = [] 504 | for i in range(InfoSize): 505 | Username = cfgobj.get(g_strUserInfo%i,g_UsrName) 506 | Pwd = cfgobj.get(g_strUserInfo%i,g_strPwd) 507 | Stmp = cfgobj.get(g_strUserInfo%i,g_strStmp) 508 | 509 | From = cfgobj.get(g_strMailinfo%i,g_strFrom) 510 | To = cfgobj.get(g_strMailinfo%i,g_strTo) 511 | Encoding = cfgobj.get(g_strMailinfo%i,g_strEncoding) 512 | 513 | ServerID = cfgobj.get(g_strServerinfo%i,g_strServerID) 514 | 515 | OnceSendLines = cfgobj.get(g_strServerinfo%i,g_strOnceSendlins) 516 | 517 | #init mailobj 518 | mailobj = MailHelper("This is python obj",Username,Pwd,Stmp,From,To,ServerID,Encoding,OnceSendLines) 519 | list_mailobj.append(mailobj) 520 | 521 | manager = multiprocessing.Manager() 522 | contentQueue = manager.Queue() 523 | lock = manager.Lock() #init lock 524 | p = multiprocessing.Pool(ServerMonitor+1) 525 | 526 | for index in range(ServerMonitor): 527 | LogPath = cfgobj.get(g_strLoginfo%index,g_strLogPath) 528 | LogName=cfgobj.get(g_strLoginfo%index,g_strLogName) 529 | DateFormat=cfgobj.get(g_strLoginfo%index,g_strDateFormat) 530 | LogPostfix =cfgobj.get(g_strLoginfo%index,g_strLogPostfix) 531 | OnceReadLines = cfgobj.getInt(g_strLoginfo%index,g_strOnceReadlines) 532 | ReadLines = cfgobj.getInt(g_strLoginfo%index,g_strHasReadlines) 533 | RegPattern = cfgobj.get(g_strLoginfo%index,g_strRegpattern) 534 | hasreaddate=cfgobj.get(g_strLoginfo%index,g_strHasReadDate) 535 | strNow = str(time.strftime(DateFormat,time.localtime(time.time()))) 536 | if strNow != hasreaddate.strip(): 537 | print "data is not the same day" 538 | ReadLines = 0 539 | 540 | list_indexlinenum.append(ReadLines) 541 | #print "$$$$$$$$$$$$$$",ReadLines 542 | monitorobj= MonitorLog(index,OnceReadLines,LogPath,contentQueue,lock,ReadLines) 543 | monitorobj.setDateFormat(DateFormat) 544 | monitorobj.setLogName(LogName) 545 | monitorobj.setLogPostfix(LogPostfix) 546 | monitorobj.setRegPattern(RegPattern) 547 | monitorobj.setSendMailFlag(1) 548 | #print str(index) + LogPath + LogName + DateFormat + LogPostfix + str(OnceReadLines) 549 | p.apply_async(monitorlogprocess,args=(monitorobj,)) 550 | 551 | pr = p.apply_async(read,args=(contentQueue,list_mailobj,InfoSize,list_indexlinenum,cfgobj)) 552 | p.close() 553 | while True: 554 | time.sleep(9) 555 | #print "main" 556 | -------------------------------------------------------------------------------- /mailConf.ini: -------------------------------------------------------------------------------- 1 | [Global] 2 | servermonitor = 2 3 | infosize = 3 4 | 5 | [UserInfo_0] 6 | username = 516587331@qq.com 7 | password = XXXXX 8 | mailstmp = smtp.qq.com 9 | 10 | [MailInfo_0] 11 | from = 516587331@qq.com 12 | to = xxxx@foxmail.com,xxxxx@sina.com,xxxx@xxx.com 13 | encoding = utf-8 14 | 15 | [ServerInfo_0] 16 | serverid = 251 17 | oncesendlines = 3 18 | 19 | [UserInfo_1] 20 | username = xxxx@126.com 21 | password = XXXXX 22 | mailstmp = smtp.126.com 23 | 24 | [MailInfo_1] 25 | from = xxxxx@126.com 26 | to = 516587331@qq.com,xxxx@sina.com 27 | encoding = utf-8 28 | 29 | [ServerInfo_1] 30 | serverid = 252 31 | oncesendlines = 3 32 | 33 | [UserInfo_2] 34 | username = xxxxx@sina.com 35 | password = XXXXXX 36 | mailstmp = smtp.sina.com 37 | 38 | [MailInfo_2] 39 | from = xxxxxt@sina.com 40 | to = xxxxx@foxmail.com,xxxxx@126.com 41 | encoding = utf-8 42 | 43 | [ServerInfo_2] 44 | serverid = 253 45 | oncesendlines = 3 46 | 47 | [logInfo_0] 48 | oncereadlines = 100 49 | dateformat = %Y-%m-%d 50 | readlines = 424 51 | logpath = /data/qmonster/liugao/RuntimeEnv/RuntimeEnv/Zone/ZoneServer/log 52 | logname = GameZoneServer 53 | logpostfix = .log 54 | RegPattern =\[(.*)\]\s+<(.*?)>\s+[Ff]ailed to decode(.*?) 55 | 56 | [logInfo_1] 57 | oncereadlines = 100 58 | dateformat = %Y-%m-%d 59 | readlines = 3 60 | logpath = /data/qmonster/liugao/RuntimeEnv/RuntimeEnv/World/WorldServer/log 61 | logname = GameWorldServer 62 | logpostfix = .log 63 | RegPattern =\[(.*)\]\s+<(.*?)>\s+[Ff]ailed to decode(.*?) 64 | --------------------------------------------------------------------------------