├── monitor_client ├── __init__.py ├── core │ ├── __init__.py │ ├── __init__.pyc │ ├── heartbeat.pyc │ └── heartbeat.py ├── plugins │ ├── __init__.py │ ├── cpu.pyc │ ├── load.pyc │ ├── __init__.pyc │ ├── plugin_api.pyc │ ├── plugin_api.py │ ├── load.py │ └── cpu.py ├── main.pyc ├── main.py └── runserver.py ├── monitor_server ├── __init__.py ├── conf │ ├── __init__.py │ ├── services │ │ ├── __init__.py │ │ ├── linux.pyc │ │ ├── __init__.pyc │ │ ├── generic.pyc │ │ ├── generic.py │ │ └── linux.py │ ├── hosts.pyc │ ├── __init__.pyc │ ├── templates.pyc │ ├── templates.py │ └── hosts.py └── runserver.tac └── README.md /monitor_client/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /monitor_server/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /monitor_client/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /monitor_server/conf/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /monitor_client/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /monitor_server/conf/services/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /monitor_client/main.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_client/main.pyc -------------------------------------------------------------------------------- /monitor_server/conf/hosts.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_server/conf/hosts.pyc -------------------------------------------------------------------------------- /monitor_client/core/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_client/core/__init__.pyc -------------------------------------------------------------------------------- /monitor_client/plugins/cpu.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_client/plugins/cpu.pyc -------------------------------------------------------------------------------- /monitor_client/plugins/load.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_client/plugins/load.pyc -------------------------------------------------------------------------------- /monitor_server/conf/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_server/conf/__init__.pyc -------------------------------------------------------------------------------- /monitor_client/core/heartbeat.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_client/core/heartbeat.pyc -------------------------------------------------------------------------------- /monitor_server/conf/templates.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_server/conf/templates.pyc -------------------------------------------------------------------------------- /monitor_client/plugins/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_client/plugins/__init__.pyc -------------------------------------------------------------------------------- /monitor_client/plugins/plugin_api.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_client/plugins/plugin_api.pyc -------------------------------------------------------------------------------- /monitor_server/conf/services/linux.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_server/conf/services/linux.pyc -------------------------------------------------------------------------------- /monitor_server/conf/services/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_server/conf/services/__init__.pyc -------------------------------------------------------------------------------- /monitor_server/conf/services/generic.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deskmen/twisted-monitor/HEAD/monitor_server/conf/services/generic.pyc -------------------------------------------------------------------------------- /monitor_server/conf/services/generic.py: -------------------------------------------------------------------------------- 1 | 2 | class BaseService(object): 3 | def __init__(self): 4 | self.name = "Basename" 5 | self.interval = 300 6 | self.plugin_name = "plugin_name" 7 | self.triggers = {} 8 | -------------------------------------------------------------------------------- /monitor_client/plugins/plugin_api.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import cpu,load 4 | 5 | 6 | def get_cpu_status(): 7 | return cpu.monitor() 8 | def get_load_status(): 9 | return load.monitor() 10 | 11 | 12 | -------------------------------------------------------------------------------- /monitor_client/plugins/load.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import commands 4 | 5 | 6 | def monitor(): 7 | shell_command = "uptime" 8 | status,result = commands.getstatusoutput(shell_command) 9 | if status != 0: 10 | value_dic = {"status":status} 11 | else: 12 | value_dic = {} 13 | uptime = result.split(',')[:-1][0] 14 | load1,load5,load15 = result.split('load average:')[1].split(',') 15 | value_dic = { 16 | 'uptime':uptime, 17 | 'load1':load1, 18 | 'load5':load5, 19 | 'load15':load15, 20 | 'status':status 21 | } 22 | return value_dic 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #twisted-monitor 2 | 3 | 此程序用于监控任何需要监控的服务 4 | 服务端程序可以分布部署在不同地区的服务器上,客户端连接的不同地域服务端程序 5 | 6 | #服务端需要安装twisted 7 | pip install twisted 8 | 9 | #添加服务端监控主机: 10 | monitor_server/conf/hosts.py 11 | web_clusters.hosts,mysql_groups.hosts中添加监控客户端 12 | web_clusters.hosts = ['192.168.1.11'] 13 | mysql_groups.hosts = ['192.168.1.11','192.168.1.12'] 14 | 15 | 16 | #启动服务端: 17 | cd monitor_server 18 | twistd -y runserver.tac 19 | #查看监控log: 20 | tail -f twistd.log 21 | 22 | 23 | 24 | #配置客户端: 25 | monitor_client/core/heartbeat.py 26 | self.host = '192.168.1.10' 27 | 28 | #启动客户端: 29 | cd monitor_client 30 | python runserver.py start 31 | #查看监控 32 | tail -f runserver.log 33 | -------------------------------------------------------------------------------- /monitor_client/plugins/cpu.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import commands 4 | 5 | 6 | def monitor(): 7 | shell_command = "sar 1 3| grep 'Average:'" 8 | status,result = commands.getstatusoutput(shell_command) 9 | if status != 0: 10 | value_dic = {"status":status} 11 | else: 12 | value_dic = {} 13 | user,nice,system,iowait,steal,idle = result.split()[2:] 14 | value_dic = { 15 | 'user':user, 16 | 'nice':nice, 17 | 'system':system, 18 | 'iowait':iowait, 19 | 'steal':steal, 20 | 'idle':idle, 21 | 'status':status 22 | } 23 | return value_dic 24 | -------------------------------------------------------------------------------- /monitor_server/conf/templates.py: -------------------------------------------------------------------------------- 1 | 2 | from services import linux 3 | 4 | 5 | class BaseTemplate(object): 6 | def __init__(self): 7 | self.name = 'name' 8 | self.hosts = [] 9 | self.services = [] 10 | 11 | 12 | class linuxgenerictemplate(BaseTemplate): 13 | def __init__(self): 14 | super(linuxgenerictemplate,self).__init__() 15 | self.name = "linuxcommonservices" 16 | self.services = [ 17 | linux.CPU(), 18 | linux.LOAD(), 19 | ] 20 | 21 | self.services[0].interval = 60 22 | 23 | class linux2(BaseTemplate): 24 | def __init__(self): 25 | super(linux2,self).__init__() 26 | self.name = "linux2" 27 | self.services = [ 28 | linux.CPU(), 29 | linux.NETWORK() 30 | ] 31 | -------------------------------------------------------------------------------- /monitor_client/core/heartbeat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import socket 4 | import json 5 | 6 | 7 | 8 | class connection_server(object): 9 | def __init__(self): 10 | self.host = "192.168.1.10" 11 | self.port = 8007 12 | def client_config(self): 13 | s = socket.socket(socket.AF_INET,socket.SOCK_STREAM) 14 | s.connect((self.host,self.port)) 15 | mark = s.recv(1024) 16 | sgin = json.loads(mark) 17 | s.close() 18 | return sgin 19 | def send_data(self,data): 20 | s = socket.socket(socket.AF_INET,socket.SOCK_STREAM) 21 | s.connect((self.host,self.port)) 22 | mark = s.send(data) 23 | s.close() 24 | 25 | -------------------------------------------------------------------------------- /monitor_server/conf/hosts.py: -------------------------------------------------------------------------------- 1 | 2 | import templates 3 | 4 | web_clusters = templates.linuxgenerictemplate() 5 | 6 | web_clusters.hosts = [ 7 | '192.168.1.11', 8 | ] 9 | 10 | 11 | 12 | mysql_groups = templates.linux2() 13 | 14 | mysql_groups.hosts = [ 15 | '192.168.1.11', 16 | '192.168.1.12', 17 | ] 18 | 19 | monitor_group = [web_clusters,mysql_groups] 20 | 21 | 22 | def send_config(): 23 | host_config_dict = {} 24 | for group in monitor_group: 25 | for host in group.hosts: 26 | if host not in host_config_dict: 27 | host_config_dict[host] = {} 28 | for s in group.services: 29 | host_config_dict[host][s.name] = [s.plugin_name,s.interval] 30 | return host_config_dict 31 | 32 | def all_config(client): 33 | host_config_dict = {} 34 | for group in monitor_group: 35 | if client in group.hosts: 36 | for s in group.services: 37 | host_config_dict[s.name] = [s.triggers] 38 | return host_config_dict 39 | 40 | -------------------------------------------------------------------------------- /monitor_client/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import socket 4 | import json 5 | import time 6 | import threading 7 | from core.heartbeat import * 8 | from plugins import plugin_api 9 | 10 | 11 | class MonitorClient(object): 12 | def __init__(self): 13 | self.sgin = connection_server() 14 | self.host_config = self.sgin.client_config() 15 | print self.host_config 16 | def start(self): 17 | self.handle() 18 | def get_host_config(self): 19 | pass 20 | def handle(self): 21 | if self.host_config: 22 | while 1: 23 | for service,val in self.host_config.items(): 24 | if len(val) < 3: 25 | self.host_config[service].append(0) 26 | plugin_name,interval,last_run_timestrf = val 27 | now_time_strftime = time.time() 28 | last_time = now_time_strftime - last_run_timestrf 29 | if last_time < interval: 30 | next_run_time = interval - last_time 31 | print "service %s next run time %s"%(service,next_run_time) 32 | else: 33 | print "going to run the %s again"%service 34 | self.host_config[service][2] = time.time() 35 | t = threading.Thread(target=self.call_plugin,args=(service,plugin_name,interval)) 36 | t.start() 37 | time.sleep(1) 38 | else: 39 | print "cannot get host config" 40 | def call_plugin(self,service,plugin_name,interval): 41 | func = getattr(plugin_api,plugin_name) 42 | report_data = { 43 | 'service':service, 44 | 'data':func(), 45 | 'timestrf':time.time(), 46 | 'interval':interval 47 | } 48 | r_data = json.dumps(report_data) 49 | self.sgin.send_data(r_data) 50 | -------------------------------------------------------------------------------- /monitor_client/runserver.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #coding:utf-8 3 | 4 | import sys 5 | import os 6 | from main import MonitorClient 7 | 8 | def daemonize(stdin='/dev/null',stdout='/dev/null',stderr='/dev/null'): 9 | try: 10 | pid = os.fork() 11 | if pid > 0: 12 | sys.exit(0) 13 | except OSError,e: 14 | sys.stderr.write("fork #1 failed:(%d) %s\n"%(e.errno,e.strerror)) 15 | sys.exit(1) 16 | 17 | os.chdir("/") 18 | os.umask(0) 19 | os.setsid() 20 | try: 21 | pid = os.fork() 22 | if pid > 0: 23 | sys.exit(0) 24 | except OSError,e: 25 | sys.stderr.write("fork #2 failed:(%d) %s\n"%(e.errno,e.strerror)) 26 | sys.exit(1) 27 | for f in sys.stdout,sys.stderr:f.flush() 28 | si = file(stdin,'r') 29 | so = file(stdout,'a+') 30 | se = file(stderr,'a+',0) 31 | os.dup2(si.fileno(),sys.stdin.fileno()) 32 | os.dup2(so.fileno(),sys.stdout.fileno()) 33 | os.dup2(se.fileno(),sys.stderr.fileno()) 34 | 35 | 36 | def run(): 37 | sys.stdout.write('daemon started with pid %d\n'%os.getpid()) 38 | pid = os.getpid() 39 | with open(process_pid,"w") as f: 40 | f.write("%d"%pid) 41 | sys.stdout.flush() 42 | dk = MonitorClient() 43 | dk.start() 44 | 45 | 46 | 47 | if __name__ == '__main__': 48 | BASE_DIR = os.path.abspath(os.path.dirname(__file__)) 49 | process_pid = "%s/%s.pid"%(BASE_DIR,sys.argv[0].strip(".py")) 50 | process_log = "%s/%s.log"%(BASE_DIR,sys.argv[0].strip(".py")) 51 | if len(sys.argv) != 2: 52 | print "参数数目输入错误,ps:python %s start|stop|restart"%sys.argv[0] 53 | elif sys.argv[1] != "start" and sys.argv[1] != "stop" and sys.argv[1] != "restart": 54 | print "参数输入错误,ps:start|stop|restart" 55 | elif sys.argv[1] == "start": 56 | print "the process is start" 57 | daemonize('/dev/null',process_log,process_log) 58 | run() 59 | elif sys.argv[1] == "stop": 60 | with open(process_pid,"r") as f: 61 | pid = f.read() 62 | os.kill(int(pid),9) 63 | print "the process is stop" 64 | elif sys.argv[1] == "restart": 65 | try: 66 | with open(process_pid,"r") as f: 67 | pid = f.read() 68 | os.kill(int(pid),9) 69 | print "the process is stop" 70 | print "the process is start" 71 | daemonize('/dev/null',process_log,process_log) 72 | run() 73 | except OSError,e: 74 | print e 75 | print "the process is start" 76 | daemonize('/dev/null',process_log,process_log) 77 | run() 78 | except: 79 | print "error" 80 | 81 | -------------------------------------------------------------------------------- /monitor_server/runserver.tac: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #coding:utf-8 3 | 4 | from twisted.internet.protocol import Protocol 5 | from twisted.internet.protocol import Factory 6 | from twisted.internet import reactor 7 | from twisted.application import service,internet 8 | import time 9 | import operator 10 | import json 11 | from conf.hosts import * 12 | 13 | class Echo(Protocol): 14 | '''协议类实现用户的服务协议，例如 http,ftp,ssh 等''' 15 | def __init__(self, factory): 16 | self.factory = factory 17 | 18 | def connectionMade(self): 19 | '''连接建立时被回调的方法''' 20 | client = self.transport.getPeer().host 21 | hosts_template = send_config() 22 | print "%s 已连接"%client 23 | if client not in hosts_template: 24 | print "%s 没有加入主机监控组"%client 25 | self.transport.loseConnection() 26 | else: 27 | host_template = hosts_template[client] 28 | self.transport.write(json.dumps(host_template)) 29 | 30 | 31 | def connectionLost(self, reason): 32 | '''连接关闭时被回调的方法''' 33 | client = self.transport.getPeer().host 34 | print "%s 已断开"%client 35 | #self.factory.numProtocols = self.factory.numProtocols - 1 36 | 37 | def dataReceived(self, data): 38 | '''接收数据的函数，当有数据到达时被回调''' 39 | client = self.transport.getPeer().host 40 | client_info = json.loads(data) 41 | client_service = client_info["service"] 42 | client_data = client_info["data"] 43 | client_timestrf = client_info["timestrf"] 44 | client_interval = client_info["interval"] 45 | client_config_all = all_config(client) 46 | 47 | client_config_info = client_config_all[client_service] 48 | client_config = client_config_info[0] 49 | if time.time() - client_timestrf < client_interval: 50 | if client_info["data"]["status"] == 0: 51 | print "%s service %s data valid"%(client,client_service) 52 | self.service_item_handle(client_config,client_data,client_service,client) 53 | else: 54 | print "%s service %s plugin error"%(client,client_service) 55 | else: 56 | expired_time = time.time() - client_timestrf - client_interval 57 | print "%s service %s data expired"%(client,client_service) 58 | def service_item_handle(self,client_config,client_data,client_service,client): 59 | for k,v in client_config.items(): 60 | print k,client_data[k] 61 | oper = v["operator"] 62 | warning_val = v["warning"] 63 | critical_val = v["critical"] 64 | oper_func = getattr(operator,oper) 65 | 66 | if v["data_type"] is float: 67 | item_data = float(client_data[k]) 68 | warning_res = oper_func(item_data,warning_val) 69 | critical_res = oper_func(item_data,critical_val) 70 | print "warning:%s critical:%s"%(warning_val,critical_val) 71 | print "warning:%s critical:%s"%(warning_res,critical_res) 72 | 73 | class EchoFactory(Factory): 74 | '''协议工厂类，当客户端建立连接的时候，创建协议对象，协议对象与客户端连接一一对应''' 75 | def buildProtocol(self, addr): 76 | return Echo(self) 77 | 78 | if __name__ == '__builtin__': 79 | # 创建监听端口 80 | 81 | application = service.Application("echo") 82 | echoService = internet.TCPServer(8007,EchoFactory()) 83 | echoService.setServiceParent(application) 84 | -------------------------------------------------------------------------------- /monitor_server/conf/services/linux.py: -------------------------------------------------------------------------------- 1 | from generic import BaseService 2 | 3 | class CPU(BaseService): 4 | def __init__(self): 5 | super(CPU,self).__init__() 6 | self.interval = 30 7 | self.name = "linux_cpu" 8 | self.plugin_name = "get_cpu_status" 9 | self.triggers = { 10 | 'idle':{ 11 | 'func':'avg', 12 | 'last':10*60, 13 | 'count':1, 14 | 'operator':'lt', 15 | 'warning':40, 16 | 'critical':30, 17 | "data_type":float 18 | }, 19 | 'iowait':{ 20 | 'func':'hit', 21 | 'last':10*60, 22 | 'count':5, 23 | 'operator':'gt', 24 | 'warning':30, 25 | 'critical':40, 26 | 'data_type':float 27 | } 28 | } 29 | 30 | class LOAD(BaseService): 31 | def __init__(self): 32 | super(LOAD,self).__init__() 33 | self.interval = 30 34 | self.name = "linux_load" 35 | self.plugin_name = "get_load_status" 36 | self.triggers = { 37 | 'load1':{ 38 | 'func':'hit', 39 | 'last':10*60, 40 | 'count':1, 41 | 'operator':'gt', 42 | 'warning':5, 43 | 'critical':10, 44 | "data_type":float 45 | }, 46 | 'load5':{ 47 | 'func':'hit', 48 | 'last':10*60, 49 | 'count':1, 50 | 'operator':'gt', 51 | 'warning':1, 52 | 'critical':10, 53 | 'data_type':float 54 | }, 55 | 'load15':{ 56 | 'func':'hit', 57 | 'last':10*60, 58 | 'count':1, 59 | 'operator':'gt', 60 | 'warning':5, 61 | 'critical':10, 62 | 'data_type':float 63 | } 64 | 65 | } 66 | 67 | 68 | class MEMORY(BaseService): 69 | def __init__(self): 70 | super(MEMORY,self).__init__() 71 | self.interval = 20 72 | self.name = "linux_memory" 73 | self.plugin_name = "get_memory_status" 74 | self.triggers = { 75 | 'usage':{ 76 | 'func':'avg', 77 | 'last':5*60, 78 | 'count':1, 79 | 'operator':'gt', 80 | 'warning':80, 81 | 'critical':90, 82 | 'data_type':float 83 | } 84 | } 85 | 86 | 87 | class NETWORK(BaseService): 88 | def __init__(self): 89 | super(NETWORK,self).__init__() 90 | self.interval = 60 91 | self.name = "linux_network" 92 | self.plugin_name = "get_network_status" 93 | self.triggers = { 94 | 'in':{ 95 | 'func':'hit', 96 | 'last':10*60, 97 | 'count':5, 98 | 'operator':'gt', 99 | 'warning':1024*1024*10, 100 | 'critical':1024*1024*15, 101 | 'data_type':float 102 | }, 103 | 'out':{ 104 | 'func':'hit', 105 | 'last':10*60, 106 | 'count':5, 107 | 'operator':'gt', 108 | 'warning':1024*1024*10, 109 | 'critical':1024*1024*15, 110 | 'data_type':float 111 | } 112 | } 113 | --------------------------------------------------------------------------------