├── .idea
├── markdown-navigator.xml
├── markdown-navigator
│ └── profiles_settings.xml
└── vcs.xml
├── __init__.py
├── bin
└── setup.sh
├── conf
├── __init__.py
├── __init__.pyc
├── __pycache__
│ ├── __init__.cpython-36.pyc
│ ├── agent.cpython-36.pyc
│ └── config.cpython-36.pyc
├── agent.py
├── config.py
├── config.pyc
└── id_rsa
├── db
└── __init__.py
├── img
├── QQ20170504-165331@2x.png
├── QQ20170504-165347@2x.png
└── QQ20170504-165654@2x.png
├── readme.md
├── requirement.txt
└── src
├── ParamikoClient.py
├── SocketServer.py
├── __init__.py
├── __pycache__
├── ParamikoClient.cpython-36.pyc
└── SocketServer.cpython-36.pyc
├── main.py
└── urlooker-to-falcon.py
/.idea/markdown-navigator.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
--------------------------------------------------------------------------------
/.idea/markdown-navigator/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/__init__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding:utf-8 -*-
3 | # Author : Leon
--------------------------------------------------------------------------------
/bin/setup.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 |
--------------------------------------------------------------------------------
/conf/__init__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding:utf-8 -*-
3 | # Author : Leon
--------------------------------------------------------------------------------
/conf/__init__.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeonQiuM/Urlooker-to-falcon/eaf619cfecdccebeda829d270be5a13778fceefa/conf/__init__.pyc
--------------------------------------------------------------------------------
/conf/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeonQiuM/Urlooker-to-falcon/eaf619cfecdccebeda829d270be5a13778fceefa/conf/__pycache__/__init__.cpython-36.pyc
--------------------------------------------------------------------------------
/conf/__pycache__/agent.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeonQiuM/Urlooker-to-falcon/eaf619cfecdccebeda829d270be5a13778fceefa/conf/__pycache__/agent.cpython-36.pyc
--------------------------------------------------------------------------------
/conf/__pycache__/config.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeonQiuM/Urlooker-to-falcon/eaf619cfecdccebeda829d270be5a13778fceefa/conf/__pycache__/config.cpython-36.pyc
--------------------------------------------------------------------------------
/conf/agent.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding:utf-8 -*-
3 | # Author : Leon
4 |
5 |
6 | #paramiko相关配置，采用私钥公钥对的方式进行登录agent,请提前进行授权
7 | private_key = "../conf/id_rsa"
8 |
9 |
10 | #各个地域的agent配置，这是一个列表，列表中的每一个agent为一个字典，添加请确保关键字段完整
11 | #并要求登录的用户对agent的脚本有执行的权限
12 |
13 | agent_list = [
14 | {
15 | "host":"192.168.30.130", # ssh ip
16 | "port":22, #ssh port
17 | "username":"root", # ssh user
18 | },
19 | ]
--------------------------------------------------------------------------------
/conf/config.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding:utf-8 -*-
3 | # Author : Leon
4 |
5 |
6 |
7 | #open-falcon API
8 | PUSHHOME = "http://127.0.0.1/v1/push"
9 |
10 | #agent需要执行的系统命令，请确保在每一个agent上都执行成功
11 | AGENTCMD = "python3 /data/scripts/urlooker_to_falcon/src/urlooker-to-falcon.py"
12 |
13 |
14 | #socket-server-config
15 | Server = "0.0.0.0"
16 | Port = 6699
17 |
18 | #时间间隔，server会定时去执行agent上的脚本，也是falcon的step间隔
19 | STEP = 300
20 |
21 |
--------------------------------------------------------------------------------
/conf/config.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeonQiuM/Urlooker-to-falcon/eaf619cfecdccebeda829d270be5a13778fceefa/conf/config.pyc
--------------------------------------------------------------------------------
/conf/id_rsa:
--------------------------------------------------------------------------------
1 | -----BEGIN RSA PRIVATE KEY-----
2 | XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
3 | -----END RSA PRIVATE KEY-----
--------------------------------------------------------------------------------
/db/__init__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding:utf-8 -*-
3 | # Author : Leon
--------------------------------------------------------------------------------
/img/QQ20170504-165331@2x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeonQiuM/Urlooker-to-falcon/eaf619cfecdccebeda829d270be5a13778fceefa/img/QQ20170504-165331@2x.png
--------------------------------------------------------------------------------
/img/QQ20170504-165347@2x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeonQiuM/Urlooker-to-falcon/eaf619cfecdccebeda829d270be5a13778fceefa/img/QQ20170504-165347@2x.png
--------------------------------------------------------------------------------
/img/QQ20170504-165654@2x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeonQiuM/Urlooker-to-falcon/eaf619cfecdccebeda829d270be5a13778fceefa/img/QQ20170504-165654@2x.png
--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | # 网站监控
2 |
3 | 
4 |
5 |
6 | ## 前言
7 |
8 | 1. 感谢
9 | + [URLooker](https://github.com/URLooker "URLooker")
10 | + [open-falcon](https://github.com/open-falcon "open-falcon")
11 |
12 |
13 | ## 摘要
14 | + 由于原urlooker在监控数据的存储、展示、报警灯跟open-falcon是基本分离的，每个地址的监控需要在全国各地的监测点上部署相同的策略，相同的告警机制，那我们为何不能用open-falcon的api将数据上传。将全国的各个监测点的数据统一到一个screen中，只配置一套监控模板、只使用一个hostgroup，这样是否会显得优雅和便捷。
15 | + 在原有的urlooker监控基础上，将存储的mysql数据提取出来回传到open-falcon做统一的监控管理
16 |
17 | ```text
18 | 1. 监控方式是在urlooker的基础上，将数据dump下来进行整理汇总，计算等操作后再push到open-falcon
19 | 2. agent与server端的数据传输，采用socket的方式通讯
20 | 3. server端控制了agent的定时执行
21 | 4. agent上传数据的同时也会将自己的dump下来的数据push到open-falcon作为独立的数据监测站点,并以一个为zone的tag来作为相同地址监控的区分
22 | ```
23 |
24 | > 1. 在URLooker的基础上将数据手机整理，重新计算后push到open-falcon
25 | > 2. 由于URLooker的部署可以分布到不同的地点，所以在处理数据的时候，不但将每个点的数据都push到falcon，\
26 | 而且会将所有点的结果进行汇总计算后push到falcon，并以tag:`zone=all`来辨识
27 | > 3. push到falcon的数据的endpoint为你在URLooker中监控的url地址
28 | > 4. metric只有两个:
29 | > + 响应平均时间`ResponseTimeAverage`
30 | > + 平均可用率`CurrentAvailableRate`
31 |
32 |
33 | ## 环境支持
34 | + Python3
35 |
36 | ## 部署
37 |
38 | + python环境
39 |
40 | + 依赖
41 | ```bash
42 | pip install -r requirement.txt
43 | ```
44 | + server端配置
45 | 1. 防火墙请打开对应端口限制，以便于agent上报的数据正常接收
46 | 2. 监听端口默认为`6699`，server修改请注意修改agent与之对应
47 | 3. 私钥请将内容添加至conf目录下的id_rsa文件中
48 |
49 | + agent相关配置
50 | 1. 公钥私钥对登录配置
51 | 2. 外网访问
52 |
53 | + 启动server
54 |
55 | ```bash
56 | cd src;nohup python3 SocketServer.py &
57 | cd src;nohup python3 main.py &
58 | ```
59 | ## 添加新的监控节点
60 |
61 | 1. 首先部署`urlooker`，从其他节点导出监控数据，这样避免了手动添加
62 | 2. 添加`open-falcon`主机对新agent的免密码登录
63 | 3. 添加`open-falcon`主机对新agent的ssh`22`端口防火墙访问权限
64 | 4. `open-falcon`外网防火墙添加agent对其`6699`端口的访问权限
65 | 5. 新节点上存在`/data/scripts/urlooker_to_falcon/src/urlooker-to-falcon.py`
66 | 6. Python3
67 | 7. 依赖：
68 | + pymysql
69 | + requests
70 | + urlparse
71 | + socket
72 | 8. 在`agent.py`中的`agent_list`列表添加一个字典，请注意包含以下三个字段：
73 | ```python
74 | {
75 | "host":"",
76 | "port":22,
77 | "username":"root"
78 | },
79 | ```
80 |
--------------------------------------------------------------------------------
/requirement.txt:
--------------------------------------------------------------------------------
1 | PyMySQL==0.7.11
2 | paramiko==2.1.2
3 | requests==2.13.0
--------------------------------------------------------------------------------
/src/ParamikoClient.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding:utf-8 -*-
3 | # Author : Leon
4 |
5 | import paramiko
6 |
7 | class SSHTcpClient(object):
8 | def __init__(self,host,port,username,pkey):
9 | self.host = host
10 | self.port = port
11 | self.username = username
12 | self.pkey = pkey
13 |
14 | def connecting(self,cmd):
15 | private_key = paramiko.RSAKey.from_private_key_file(self.pkey)
16 | ssh = paramiko.SSHClient()
17 | ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
18 | ssh.connect(hostname=self.host,
19 | port=self.port,
20 | username=self.username,
21 | pkey=private_key
22 | )
23 | stdin, stdout, stderr = ssh.exec_command(cmd)
24 | result = stdout.read()
25 | print(result.decode())
26 | ssh.close()
--------------------------------------------------------------------------------
/src/SocketServer.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding:utf-8 -*-
3 | # Author : Leon
4 |
5 | import socketserver
6 | import sys,os,json
7 | import re
8 |
9 | BaseDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
10 | sys.path.append(BaseDir)
11 | from conf.config import *
12 |
13 |
14 | class UrlLookerTcpHandler(socketserver.BaseRequestHandler):
15 |
16 | def handle(self):
17 | '''
18 | handle
19 | :return:
20 | '''
21 | li = []
22 | while True:
23 | try:
24 | self.data = self.request.recv(1024).strip()
25 | except Exception as text:
26 | print(text,"recv timeout")
27 | if not self.data:
28 | break
29 | li.append(self.data.decode())
30 | self.request.sendall("OK".encode())
31 | print(len(li))
32 | new_li = []
33 | for i in li:
34 | newli = re.sub("'","",i)
35 | new_li.append(json.loads(newli))
36 | with open("../db/db",'a') as f:
37 | for i in new_li:
38 | f.write(json.dumps(i)+"\n")
39 |
40 |
41 | HOST, PORT = Server, Port
42 | server = socketserver.ThreadingTCPServer((HOST, PORT), UrlLookerTcpHandler)
43 | server.serve_forever()
44 |
45 |
--------------------------------------------------------------------------------
/src/__init__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding:utf-8 -*-
3 | # Author : Leon
--------------------------------------------------------------------------------
/src/__pycache__/ParamikoClient.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeonQiuM/Urlooker-to-falcon/eaf619cfecdccebeda829d270be5a13778fceefa/src/__pycache__/ParamikoClient.cpython-36.pyc
--------------------------------------------------------------------------------
/src/__pycache__/SocketServer.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeonQiuM/Urlooker-to-falcon/eaf619cfecdccebeda829d270be5a13778fceefa/src/__pycache__/SocketServer.cpython-36.pyc
--------------------------------------------------------------------------------
/src/main.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding:utf-8 -*-
3 | # Author : Leon
4 |
5 | import os,sys,time
6 | import json
7 | import requests
8 |
9 | BaseDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
10 | sys.path.append(BaseDir)
11 |
12 | from conf.config import *
13 | from conf.agent import *
14 |
15 | from ParamikoClient import SSHTcpClient
16 |
17 |
18 | def push(PushDataList):
19 | response = requests.post(PUSHHOME, data=json.dumps(PushDataList))
20 | print(response.text)
21 |
22 |
23 | def analyse(AllUrlDict):
24 | PushDataList = []
25 | for key in AllUrlDict:
26 | Timedata = {}
27 | RateData = {}
28 | Timedata["endpoint"] = key
29 | RateData["endpoint"] = key
30 | ResponseTimeTotal = ResponseTimeCount = CurrentAvailableRateTotal = CurrentAvailableRateCount = 0
31 | for ValueDict in AllUrlDict[key]:
32 |
33 | if ValueDict.get("metric") == "ResponseTimeAverage":
34 | ResponseTimeTotal += ValueDict["value"]
35 | ResponseTimeCount += 1
36 | elif ValueDict.get("metric") == "CurrentAvailableRate":
37 | CurrentAvailableRateTotal += ValueDict["value"]
38 | CurrentAvailableRateCount += 1
39 | Timedata["metric"] = "ResponseTimeAverage"
40 | Timedata["value"] = ResponseTimeTotal/ResponseTimeCount
41 | Timedata["step"] = STEP
42 | Timedata["tags"] = "zone=all"
43 | Timedata["timestamp"] = int(time.time())
44 | Timedata["counterType"] = 'GAUGE'
45 | PushDataList.append(Timedata)
46 | RateData["metric"] = "CurrentAvailableRate"
47 | RateData["value"] = CurrentAvailableRateTotal / CurrentAvailableRateCount
48 | RateData["step"] = STEP
49 | RateData["tags"] = "zone=all"
50 | RateData["timestamp"] = int(time.time())
51 | RateData["counterType"] = 'GAUGE'
52 | PushDataList.append(RateData)
53 | return PushDataList
54 |
55 |
56 |
57 | def grouping():
58 | '''
59 | :return:
60 | '''
61 | AllUrlDict = {}
62 | with open("../db/db", 'r', encoding="utf-8") as f:
63 | for line in f:
64 | line = json.loads(line)
65 | if line["endpoint"] not in AllUrlDict.keys():
66 | AllUrlDict[line["endpoint"]] = []
67 | AllUrlDict[line["endpoint"]].append(line)
68 | else:
69 | AllUrlDict[line["endpoint"]].append(line)
70 | return AllUrlDict
71 |
72 |
73 | while True:
74 | # loading data
75 | for agent in agent_list:
76 | print(agent)
77 | try:
78 | SSHClient = SSHTcpClient(agent["host"],agent["port"],agent["username"],private_key)
79 | SSHClient.connecting(AGENTCMD)
80 | except Exception as text:
81 | print(text)
82 | continue
83 |
84 | AllUrlDict = grouping()
85 | PushDataList = analyse(AllUrlDict)
86 | print(PushDataList)
87 | push(PushDataList)
88 | t = time.localtime()
89 | FileTag = "{year}-{mon}-{day}:{hour}:{min}:{sec}".format(year=t.tm_year,
90 | mon=t.tm_mon,
91 | day=t.tm_mday,
92 | hour=t.tm_hour,
93 | min=t.tm_min,
94 | sec=t.tm_sec)
95 | os.rename("../db/db","../db/db.{date}".format(date=FileTag))
96 | time.sleep(STEP)
97 |
98 |
--------------------------------------------------------------------------------
/src/urlooker-to-falcon.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding:utf-8 -*-
3 | # Author : Leon
4 |
5 | import pymysql
6 | import time
7 | from urllib.parse import urlparse
8 | import requests,json
9 | import socket
10 |
11 | ServerAddress="192.168.30.130"
12 | ServerPort=6699
13 |
14 | PUSHHOME="http://127.0.0.1/v1/push"
15 | HOST="127.0.0.1"
16 | ZoneTag = ""
17 | PORT=3306
18 | USER='root'
19 | PASSWD=""
20 | DB='urlooker'
21 |
22 | class MysqlDb(object):
23 | def __init__(self,HOST,PORT,USER,PASSWD,DB):
24 | self.HOST = HOST
25 | self.PORT = PORT
26 | self.USER = USER
27 | self.PASSWD = PASSWD
28 | self.DB = DB
29 |
30 | def connect(self,sql):
31 | connect = pymysql.connect(host=self.HOST, port=self.PORT, user=self.USER, passwd=self.PASSWD, db=self.DB)
32 | cursor = connect.cursor(cursor=pymysql.cursors.DictCursor)
33 | cursor.execute(sql)
34 | raw = cursor.fetchall()
35 | connect.commit()
36 | cursor.close()
37 | connect.close()
38 | return raw
39 |
40 | def __del__(self):
41 | pass
42 | def __str__(self):
43 | return "Exception!"
44 |
45 | def query_map():
46 | '''
47 | Query the raw data table, maintaining the sid and url corresponding relation, and write the map.conf file
48 | :return: None
49 | '''
50 | _mysql_conn_obj = MysqlDb(HOST,PORT,USER,PASSWD,DB)
51 | raw = _mysql_conn_obj.connect("select * from {TABLE_NAME}".format(TABLE_NAME="strategy"))
52 | BaseData = {}
53 | for item in raw:
54 | key = item["id"]
55 | value = item["url"]
56 | BaseData[key] = value
57 | with open("map.conf",'w+') as f:
58 | for i,j in BaseData.items():
59 | line = str(i) + "\t" + str(j) + "\n"
60 | f.write(line)
61 |
62 |
63 | def query_data():
64 | '''
65 | From the database to retrieve data within ten minutes before the current point in time
66 | :return:
67 |
68 | '''
69 | CurrentTime = int(time.time())
70 | TenMinTime = CurrentTime - 600 #ten minutes
71 | _mysql_conn_obj = MysqlDb(HOST, PORT, USER, PASSWD, DB)
72 | raw = _mysql_conn_obj.connect(
73 | "select * from item_status00 where push_time>%s and push_time<%s order by Sid"
74 | %(TenMinTime, CurrentTime)
75 | )
76 | return raw
77 |
78 |
79 | def handle_data():
80 | '''
81 | The original data cleaning
82 | :param query_data_raw:
83 | :return:
84 | '''
85 | OriginalData = query_data()
86 | CleanData = {}
87 | for item in OriginalData:
88 | CleanData[item["sid"]] = {}
89 | CleanData[item["sid"]]["resp_time"] = []
90 | CleanData[item["sid"]]["result"] = []
91 | for item in OriginalData:
92 | if item["sid"] in CleanData.keys():
93 | CleanData[item["sid"]]["resp_time"].append(item["resp_time"])
94 | CleanData[item["sid"]]["result"].append(item["result"])
95 | PushBaseData = {}
96 | for i, j in CleanData.items():
97 | PushBaseData[i] = {}
98 | for i, j in CleanData.items():
99 | ResponseTimeTotal = 0
100 | for item in j["resp_time"]:
101 | ResponseTimeTotal += item
102 | ResponseTimeAverage = ResponseTimeTotal / len(j["resp_time"])
103 | PushBaseData[i]["ResponseTimeAverage"] = ResponseTimeAverage
104 | ErrorCount = NormalCount = 0
105 | for item in j["resp_code"]:
106 | if item == 0:
107 | NormalCount += 1
108 | else:
109 | ErrorCount += 1
110 | CurrentAvailableRate = NormalCount / float(NormalCount + ErrorCount)
111 | PushBaseData[i]["CurrentAvailableRate"] = CurrentAvailableRate
112 |
113 | return PushBaseData
114 |
115 |
116 | def _read_map():
117 | '''
118 | :return:
119 | '''
120 | #update local file
121 | query_map()
122 | MapDict = {}
123 | with open("map.conf", 'r') as f:
124 | for line in f:
125 | sid, url = line.split()
126 | url = urlparse(url)
127 | MapDict[sid] = url.netloc+url.path
128 | return MapDict
129 |
130 |
131 |
132 | def _push_data():
133 | '''
134 | handle push to open-falcon data
135 | :return:
136 | '''
137 | PushBaseData = handle_data()
138 | Mapdict = _read_map()
139 | PushList = []
140 |
141 | for item in PushBaseData:
142 | item = int(item)
143 | FinallyPushData = {}
144 | FinallyPushData["endpoint"] = Mapdict[str(item)]
145 | FinallyPushData["metric"] = "ResponseTimeAverage"
146 | FinallyPushData["timestamp"] = int(time.time())
147 | FinallyPushData["value"] = PushBaseData[item]["ResponseTimeAverage"]
148 | FinallyPushData["counterType"] = "GAUGE"
149 | FinallyPushData["tags"] = "Zone={ZoneTag}".format(ZoneTag=ZoneTag)
150 | FinallyPushData["step"] = 300
151 | PushList.append(FinallyPushData)
152 | FinallyPushData = {}
153 | FinallyPushData["endpoint"] = Mapdict[str(item)]
154 | FinallyPushData["metric"] = "CurrentAvailableRate"
155 | FinallyPushData["timestamp"] = int(time.time())
156 | FinallyPushData["value"] = PushBaseData[item]["CurrentAvailableRate"]
157 | FinallyPushData["counterType"] = "GAUGE"
158 | FinallyPushData["tags"] = "Zone={ZoneTag}".format(ZoneTag=ZoneTag)
159 | FinallyPushData["step"] = 300
160 | PushList.append(FinallyPushData)
161 | return PushList
162 |
163 |
164 | if __name__ == '__main__':
165 | PushDataList = _push_data()
166 | response = requests.post(PUSHHOME,data=json.dumps(PushDataList))
167 | print(response.text)
168 |
169 | client = socket.socket()
170 | client.connect((ServerAddress, ServerPort))
171 |
172 | while True:
173 | for i in PushDataList:
174 | client.sendall(json.dumps(i).encode())
175 | print(len(PushDataList))
176 | recv = client.recv(1024)
177 | print("recv status", recv.decode())
178 | break
179 |
180 | client.close()
181 |
182 |
183 |
184 |
185 |
186 |
--------------------------------------------------------------------------------