├── __init__.py ├── Src ├── __init__.py ├── ip2region.db ├── ip2Region.py └── Core.py ├── webServer ├── __init__.py ├── admin │ ├── __init__.py │ └── home.py ├── static │ ├── images │ │ ├── jt.png │ │ ├── lbx.png │ │ ├── map.png │ │ ├── head_bg.png │ │ ├── line(1).png │ │ ├── weather.png │ │ └── webserver_monitor.png │ ├── font │ │ └── DS-DIGIT.TTF │ ├── js │ │ ├── flexible.js │ │ ├── timer.js │ │ ├── config.js.bak │ │ ├── config.js │ │ ├── myMap.js │ │ └── index.js │ └── css │ │ ├── index.css │ │ └── index.less ├── start.py ├── customer.py ├── templates │ └── home │ │ └── index.html ├── divers │ ├── mongo.py │ └── mysql.py └── api_request.http ├── ParserAdapter ├── __init__.py ├── BaseAdapter.py ├── Apache.py └── Nginx.py ├── QueueAdapter ├── __init__.py ├── BaseAdapter.py ├── Redis.py └── Mongodb.py ├── StorageAdapter ├── __init__.py ├── BaseAdapter.py ├── Mongodb.py └── Mysql.py ├── requirements.txt ├── Dockerfile ├── config.ini.example ├── main.py ├── README.MD └── LICENSE /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /webServer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ParserAdapter/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /QueueAdapter/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /StorageAdapter/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /webServer/admin/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Src/ip2region.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jyolo/wLogger/HEAD/Src/ip2region.db -------------------------------------------------------------------------------- /webServer/static/images/jt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jyolo/wLogger/HEAD/webServer/static/images/jt.png -------------------------------------------------------------------------------- /webServer/static/images/lbx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jyolo/wLogger/HEAD/webServer/static/images/lbx.png -------------------------------------------------------------------------------- /webServer/static/images/map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jyolo/wLogger/HEAD/webServer/static/images/map.png -------------------------------------------------------------------------------- /webServer/static/font/DS-DIGIT.TTF: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jyolo/wLogger/HEAD/webServer/static/font/DS-DIGIT.TTF -------------------------------------------------------------------------------- /webServer/static/images/head_bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jyolo/wLogger/HEAD/webServer/static/images/head_bg.png -------------------------------------------------------------------------------- /webServer/static/images/line(1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jyolo/wLogger/HEAD/webServer/static/images/line(1).png -------------------------------------------------------------------------------- /webServer/static/images/weather.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jyolo/wLogger/HEAD/webServer/static/images/weather.png -------------------------------------------------------------------------------- /webServer/static/images/webserver_monitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jyolo/wLogger/HEAD/webServer/static/images/webserver_monitor.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask == 1.1.1 2 | flask-pymongo == 2.3.0 3 | flask_sqlalchemy == 2.4.4 4 | SQLAlchemy == 1.3 5 | configparser == 4.0.2 6 | redis == 3.5.3 7 | mongo == 0.2.0 8 | pymysql == 0.10.1 9 | click == 7.1.2 -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos 2 | RUN yum list | grep python \ 3 | && yum -y install python38 \ 4 | && yum -y install git \ 5 | && cd / && git clone https://github.com/jyolo/wLogger \ 6 | && cd /wLogger \ 7 | && pip3 install -r requirements.txt \ 8 | && echo "/usr/bin/python3 /wLogger/main.py \$@" > run.sh 9 | 10 | 11 | 12 | 13 | ENTRYPOINT ["/bin/bash","/wLogger/run.sh"] 14 | -------------------------------------------------------------------------------- /QueueAdapter/BaseAdapter.py: -------------------------------------------------------------------------------- 1 | from abc import abstractmethod,ABCMeta 2 | 3 | class Adapter(): 4 | __metaclass__ = ABCMeta 5 | 6 | 7 | def __init__(self): pass 8 | 9 | @abstractmethod 10 | def initQueue(self): pass 11 | 12 | @abstractmethod 13 | def pushDataToQueue(self ): pass 14 | 15 | @abstractmethod 16 | def getDataFromQueue(self): pass 17 | 18 | @abstractmethod 19 | def rollBackToQueue(self): pass 20 | 21 | @abstractmethod 22 | def getDataCountNum(self): pass -------------------------------------------------------------------------------- /webServer/static/js/flexible.js: -------------------------------------------------------------------------------- 1 | (function flexible(window, document) { 2 | var docEl = document.documentElement; 3 | var dpr = window.devicePixelRatio || 1; 4 | 5 | // adjust body font size 6 | function setBodyFontSize() { 7 | if (document.body) { 8 | document.body.style.fontSize = 12 * dpr + "px"; 9 | } else { 10 | document.addEventListener("DOMContentLoaded", setBodyFontSize); 11 | } 12 | } 13 | setBodyFontSize(); 14 | 15 | // set 1rem = viewWidth / 10 16 | function setRemUnit() { 17 | var rem = docEl.clientWidth / 24; 18 | docEl.style.fontSize = rem + "px"; 19 | } 20 | 21 | setRemUnit(); 22 | 23 | // reset rem unit on page resize 24 | window.addEventListener("resize", setRemUnit); 25 | window.addEventListener("pageshow", function(e) { 26 | if (e.persisted) { 27 | setRemUnit(); 28 | } 29 | }); 30 | 31 | // detect 0.5px supports 32 | if (dpr >= 2) { 33 | var fakeBody = document.createElement("body"); 34 | var testElement = document.createElement("div"); 35 | testElement.style.border = ".5px solid transparent"; 36 | fakeBody.appendChild(testElement); 37 | docEl.appendChild(fakeBody); 38 | if (testElement.offsetHeight === 1) { 39 | docEl.classList.add("hairlines"); 40 | } 41 | docEl.removeChild(fakeBody); 42 | } 43 | })(window, document); 44 | -------------------------------------------------------------------------------- /webServer/static/js/timer.js: -------------------------------------------------------------------------------- 1 | // 每个loader 定时设定 ms 毫秒 2 | var each_loader_timmer = { 3 | 4 | total_ip: global_timer_secends, 5 | total_pv: global_timer_secends, 6 | 7 | map_chart: global_timer_secends, 8 | top_ip_chart : global_timer_secends , 9 | 10 | status_code_chart : global_timer_secends, 11 | 12 | network_traffic_by_minute : global_timer_secends, 13 | request_ip_pv_by_minute : global_timer_secends, 14 | 15 | request_num_by_url : global_timer_secends, 16 | spider_by_ua : global_timer_secends, 17 | 18 | } 19 | 20 | // window.chart_load_func['total_ip']() 21 | // window.chart_load_func['total_pv']() 22 | // window.chart_load_func['map_chart']() 23 | // window.chart_load_func['top_ip_chart']() 24 | // window.chart_load_func['status_code_chart']() 25 | // window.chart_load_func['request_ip_pv_by_minute']() 26 | // window.chart_load_func['network_traffic_by_minute']() 27 | // window.chart_load_func['request_num_by_url']() 28 | // window.chart_load_func['spider_by_ua']() 29 | 30 | 31 | 32 | loader_key = Object.keys(each_loader_timmer) 33 | var timer = [] 34 | 35 | for (var i=0 ;i < loader_key.length ; i++){ 36 | 37 | let func = window.chart_load_func[loader_key[i]] 38 | // first run 39 | if (!func) continue; 40 | func() 41 | timer = setInterval(function(){ 42 | // for timer 43 | func() 44 | },each_loader_timmer[ loader_key[i] ]) 45 | 46 | } 47 | 48 | 49 | -------------------------------------------------------------------------------- /webServer/static/js/config.js.bak: -------------------------------------------------------------------------------- 1 | window.host = 'http://local.monitor.com'; 2 | window.Authorization = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJsb2NhbC5rcGkuY29tIiwiaWF0IjoxNTkxNjg1OTA1LCJzdWIiOiJ3ZWIiLCJhdWQiOiJBcGFjaGUtSHR0cENsaWVudFwvNC41LjYgKEphdmFcLzEuOC4wXzIwMi1yZWxlYXNlKSIsImV4cCI6MTYyMzIyMTkwNSwidWlkIjo4NCwibmlja25hbWUiOiJ3ZWljb25nIn0.fX1RrO4aOJ-v0QKQ4lSBcjIWyDzzl4F96yDv7_aySHnacUU-4VZbYeec4804-iJBLBmWcM3YheoO-XFqyY9ffdQTjNfobD9WiYPBNJBJAooSQJMOo2H7mwJrXgPTGlFEds1rpXfGHEH2yl7SidPwa4Hq4itR6B1aJOdEY23-8GU' 3 | window.global_timer_secends = 20000 // 全局定时器 20秒 4 | window.chart_load_func = [] 5 | 6 | //total_money 7 | window.chart_load_func['get_total_money_load'] = function(){ 8 | $.ajax({ 9 | url: host + '/v1/screen/get_tsmoney_total', 10 | type:'GET', 11 | async:true, 12 | headers: { 'Authorization':Authorization,}, 13 | success:function(msg){ 14 | let money = msg.data.toString().split('.')[0] 15 | $('.total_money').html(money) 16 | 17 | } 18 | 19 | }) 20 | } 21 | 22 | 23 | //total_member 24 | window.chart_load_func['get_total_member_load'] = function(){ 25 | $.ajax({ 26 | url: host + '/v1/screen/get_member_total', 27 | type:'GET', 28 | async:true, 29 | headers: { 'Authorization':Authorization,}, 30 | success:function(msg){ 31 | let num = msg.data.toString().split('.')[0] 32 | $('.total_member').html(num) 33 | 34 | } 35 | }) 36 | } 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /config.ini.example: -------------------------------------------------------------------------------- 1 | 2 | [nginx] 3 | pid_path = /www/server/nginx/logs/nginx.pid 4 | server_conf = /www/server/nginx/conf/nginx.conf 5 | 6 | [apache] 7 | apachectl_bin = /www/server/apache/bin/apachectl 8 | server_conf = /www/server/apache/conf/httpd.conf 9 | 10 | 11 | [mysql] 12 | host = 127.0.0.1 13 | port = 3306 14 | db = logger 15 | username = logger 16 | password = xxxxx 17 | table = logger_watcher 18 | # 输出 日志 切割 到不同的表 或者 集合 支持 [day, week, month ,year] 19 | split_save = day 20 | 21 | [redis] 22 | host = 127.0.0.1 23 | port = 6379 24 | password = xxx 25 | db = 1 26 | 27 | 28 | [mongodb] 29 | host = 127.0.0.1 30 | port = 27017 31 | username = xxxx 32 | password = xxxx 33 | db = logger_watcher 34 | collection = logger 35 | # 输出 日志 切割 到不同的表 或者 集合 支持 [day, week, month ,year] 36 | split_save = day 37 | 38 | 39 | [inputer] 40 | log_debug = True 41 | node_id = server_80 42 | queue = redis 43 | queue_name = logger_watch:logger 44 | max_batch_push_queue_size = 5000 45 | max_retry_open_file_time = 10 46 | max_retry_reconnect_time = 20 47 | 48 | 49 | [inputer.log_file.web1] 50 | server_type = nginx 51 | file_path = /www/wwwlogs/local.test3.com.log 52 | ;log_format_name = combined 53 | log_format_name = online 54 | read_type = tail 55 | cut_file_type = filesize 56 | cut_file_point = 10 57 | cut_file_save_dir = /www/wwwlogs/cut_file/ 58 | 59 | 60 | [outputer] 61 | log_debug = True 62 | ;save_engine = mongodb 63 | save_engine = mysql 64 | queue = redis 65 | ;queue = mongodb 66 | queue_name = logger_watch:logger 67 | ;server_type = apache 68 | server_type = nginx 69 | worker_process_num = 1 70 | max_batch_insert_db_size = 1 71 | max_retry_reconnect_time = 200 72 | 73 | 74 | [web] 75 | # development | production 76 | env = development 77 | debug = True 78 | secret_key = asdasdasdsadasd 79 | host = 127.0.0.1 80 | port = 5000 81 | data_engine = mysql 82 | ;data_engine = mysql 83 | 84 | 85 | -------------------------------------------------------------------------------- /webServer/static/js/config.js: -------------------------------------------------------------------------------- 1 | window.host = ''; 2 | window.Authorization = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJsb2NhbC5rcGkuY29tIiwiaWF0IjoxNTkxNjg1OTA1LCJzdWIiOiJ3ZWIiLCJhdWQiOiJBcGFjaGUtSHR0cENsaWVudFwvNC41LjYgKEphdmFcLzEuOC4wXzIwMi1yZWxlYXNlKSIsImV4cCI6MTYyMzIyMTkwNSwidWlkIjo4NCwibmlja25hbWUiOiJ3ZWljb25nIn0.fX1RrO4aOJ-v0QKQ4lSBcjIWyDzzl4F96yDv7_aySHnacUU-4VZbYeec4804-iJBLBmWcM3YheoO-XFqyY9ffdQTjNfobD9WiYPBNJBJAooSQJMOo2H7mwJrXgPTGlFEds1rpXfGHEH2yl7SidPwa4Hq4itR6B1aJOdEY23-8GU' 3 | window.global_timer_secends = 10000 // 全局定时器 20秒 4 | window.chart_load_func = [] 5 | 6 | 7 | //total ip today 8 | window.chart_load_func['total_ip'] = function(){ 9 | console.log(window.location.href) 10 | $.ajax({ 11 | url: host + '/get_total_ip', 12 | type:'GET', 13 | async:true, 14 | headers: { 'Authorization':Authorization,}, 15 | success:function(msg){ 16 | $('.total_ip').html(msg.data['total_num']) 17 | } 18 | 19 | }) 20 | } 21 | 22 | 23 | //total_pv 24 | window.chart_load_func['total_pv'] = function(){ 25 | 26 | $.ajax({ 27 | url: host + '/get_total_pv', 28 | type:'GET', 29 | async:true, 30 | headers: { 'Authorization':Authorization,}, 31 | success:function(msg){ 32 | $('.total_pv').html(msg.data['total_num']) 33 | 34 | } 35 | }) 36 | } 37 | 38 | function timestampToTime(timestamp) { 39 | var date = new Date(timestamp * 1000) ;//时间戳为10位需*1000，时间戳为13位的话不需乘1000 40 | // var Y = date.getFullYear() + '-'; 41 | // var M = (date.getMonth()+1 < 10 ? '0'+(date.getMonth()+1) : date.getMonth()+1) + '-'; 42 | // var D = (date.getDate() < 10 ? '0'+date.getDate() : date.getDate()) + ' '; 43 | var h = (date.getHours() < 10 ? '0'+date.getHours() : date.getHours()) + ':'; 44 | var m = (date.getMinutes() < 10 ? '0'+date.getMinutes() : date.getMinutes()) + ':'; 45 | // var s = (date.getSeconds() < 10 ? '0'+date.getSeconds() : date.getSeconds()); 46 | 47 | // let strDate = Y+M+D+h+m+s; 48 | m = m.replace(':','') 49 | let strDate = h+m; 50 | 　 51 | return strDate; 52 | } 53 | 54 | 55 | -------------------------------------------------------------------------------- /StorageAdapter/BaseAdapter.py: -------------------------------------------------------------------------------- 1 | from abc import abstractmethod,ABCMeta 2 | import datetime,time 3 | 4 | 5 | class Adapter(): 6 | __metaclass__ = ABCMeta 7 | 8 | save_engine = [] 9 | split_save = ['day', 'week', 'month' ,'year'] 10 | 11 | def __init__(self): pass 12 | 13 | 14 | def _getTableName(self,table_key_name): 15 | 16 | table_suffix = '' 17 | save_engine_conf = self.conf[self.conf['outputer']['save_engine']] 18 | try: 19 | self.table = save_engine_conf[table_key_name] 20 | except KeyError as e: 21 | raise Exception('配置错误: %s not exists' % e.args) 22 | 23 | 24 | if 'split_save' in save_engine_conf: 25 | 26 | if save_engine_conf['split_save'] not in self.split_save: 27 | 28 | raise Exception('outputer 配置项 split_save 只支持 %s 选项' % ','.join(self.split_save)) 29 | 30 | if save_engine_conf['split_save'] == 'day': 31 | 32 | table_suffix = time.strftime( '%Y_%m_%d' , time.localtime()) 33 | 34 | elif save_engine_conf['split_save'] == 'week': 35 | 36 | now = datetime.datetime.now() 37 | this_week_start = now - datetime.timedelta(days=now.weekday()) 38 | this_week_end = now + datetime.timedelta(days=6 - now.weekday()) 39 | 40 | table_suffix = datetime.datetime.strftime(this_week_start,'%Y_%m_%d') + datetime.datetime.strftime(this_week_end,'_%d') 41 | 42 | elif save_engine_conf['split_save'] == 'month': 43 | 44 | table_suffix = time.strftime( '%Y_%m' , time.localtime()) 45 | elif save_engine_conf['split_save'] == 'year': 46 | 47 | table_suffix = time.strftime( '%Y' , time.localtime()) 48 | 49 | if len(table_suffix) : 50 | self.table = self.table +'_' + table_suffix 51 | 52 | 53 | 54 | @abstractmethod 55 | def initStorage(self): pass 56 | 57 | @abstractmethod 58 | def analysisTraffic(self):pass 59 | 60 | @abstractmethod 61 | def pushDataToStorage(self ): pass 62 | 63 | @abstractmethod 64 | def _handle_queue_data_before_into_storage(self):pass 65 | 66 | @abstractmethod 67 | def _handle_queue_data_after_into_storage(self):pass 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /webServer/admin/home.py: -------------------------------------------------------------------------------- 1 | # coding=UTF-8 2 | from flask.blueprints import Blueprint 3 | from flask_pymongo import PyMongo 4 | from flask import render_template,request,flash,session,current_app 5 | from webServer.customer import ApiCorsResponse 6 | import time,re 7 | 8 | 9 | 10 | home = Blueprint('home',__name__) 11 | 12 | @home.route('/',methods=['GET','POST']) 13 | def index(): 14 | # flash('You were successfully logged in asdasd') 15 | if(request.method == 'GET'): 16 | return render_template('home/index.html') 17 | 18 | 19 | 20 | @home.route('/get_total_ip' , methods=['GET']) 21 | def get_total_ip(): 22 | return current_app.driver.get_total_ip() 23 | 24 | @home.route('/get_total_pv' , methods=['GET']) 25 | def get_total_pv(): 26 | return current_app.driver.get_total_pv() 27 | 28 | @home.route('/get_request_num_by_url' , methods=['GET']) 29 | def get_request_num_by_url(): 30 | return current_app.driver.get_request_num_by_url() 31 | 32 | @home.route('/get_request_urls_by_ip' , methods=['GET']) 33 | def get_request_urls_by_ip(): 34 | return current_app.driver.get_request_urls_by_ip() 35 | 36 | @home.route('/get_request_num_by_ip' , methods=['GET']) 37 | def get_request_num_by_ip(): 38 | return current_app.driver.get_request_num_by_ip() 39 | 40 | @home.route('/get_request_num_by_secends' , methods=['GET']) 41 | def get_request_num_by_secends(): 42 | return current_app.driver.get_request_num_by_secends() 43 | 44 | @home.route('/get_network_traffic_by_minute' , methods=['GET']) 45 | def get_network_traffic_by_minute(): 46 | return current_app.driver.get_network_traffic_by_minute() 47 | 48 | @home.route('/get_ip_pv_num_by_minute' , methods=['GET']) 49 | def get_ip_pv_num_by_minute(): 50 | return current_app.driver.get_ip_pv_num_by_minute() 51 | 52 | @home.route('/get_request_num_by_province' , methods=['GET']) 53 | def get_request_num_by_province(): 54 | return current_app.driver.get_request_num_by_province() 55 | 56 | @home.route('/get_request_num_by_status' , methods=['GET']) 57 | def get_request_num_by_status(): 58 | return current_app.driver.get_request_num_by_status() 59 | 60 | 61 | @home.route('/get_request_num_by_status_code' , methods=['GET']) 62 | def get_request_num_by_status_code(): 63 | return current_app.driver.get_request_num_by_status_code() 64 | 65 | @home.route('/get_spider_by_ua' , methods=['GET']) 66 | def get_spider_by_ua(): 67 | return current_app.driver.get_spider_by_ua() 68 | 69 | 70 | @home.route('/get_device_type_by_ua' , methods=['GET']) 71 | def get_device_type_by_ua(): 72 | return current_app.driver.get_device_type_by_ua() 73 | 74 | 75 | 76 | 77 | if __name__ == "__main__": 78 | pass -------------------------------------------------------------------------------- /webServer/start.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | import os,sys 3 | 4 | sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 5 | from webServer.admin.home import home 6 | from webServer.divers.mysql import MysqlDb 7 | from webServer.divers.mongo import MongoDb 8 | from webServer.customer import Func 9 | 10 | 11 | app = Flask(__name__) 12 | 13 | 14 | # start this server from the main.py 15 | def start_web(conf_dict = {}): 16 | """ 17 | when run in subprocess need conf_dict of the flask config 18 | :param conf_dict: 19 | :return: 20 | """ 21 | if(not conf_dict ): 22 | raise ValueError('miss flask config of args conf_dict') 23 | 24 | app.env = conf_dict['env'] 25 | if conf_dict['debug'] == 'True': 26 | app.debug = True 27 | elif conf_dict['debug'] == 'False': 28 | app.debug = False 29 | 30 | app.secret_key = conf_dict['secret_key'] 31 | if 'host' in conf_dict: 32 | _host = conf_dict['host'] 33 | else: 34 | _host = '0.0.0.0' 35 | 36 | if 'port' in conf_dict: 37 | _port = conf_dict['port'] 38 | else: 39 | _port = 5000 40 | 41 | if 'server_name' in conf_dict: 42 | app.config['SERVER_NAME'] = conf_dict['server_name'] 43 | 44 | app.config.from_mapping(conf_dict) 45 | app.register_blueprint(home, url_prefix='/') 46 | 47 | 48 | # init flask db engine 49 | setAppDataEngine(conf_dict) 50 | 51 | app.run( 52 | host=_host, 53 | port=_port 54 | ) 55 | 56 | 57 | 58 | def setAppDataEngine(conf_dict): 59 | args = conf_dict[conf_dict['data_engine']] 60 | app.db_engine_table = Func.getTableName(args ,data_engine = conf_dict['data_engine']) 61 | 62 | if conf_dict['data_engine'] == 'mongodb': 63 | from flask_pymongo import PyMongo 64 | 65 | if args['username'] and args['password']: 66 | mongourl = 'mongodb://%s:%s@%s:%s/%s' % ( 67 | args['username'], args['password'], args['host'], args['port'], args['db']) 68 | else: 69 | mongourl = 'mongodb://%s:%s/%s' % (args['host'], args['port'], args['db']) 70 | 71 | app.db = PyMongo(app,mongourl).db 72 | app.driver = MongoDb 73 | 74 | 75 | if conf_dict['data_engine'] == 'mysql': 76 | from flask_sqlalchemy import SQLAlchemy 77 | # from sqlalchemy import create_engine 78 | # from sqlalchemy.engine.result 79 | import pymysql 80 | 81 | #sqlalchemy docs https://docs.sqlalchemy.org/en/13/core/pooling.html 82 | pymysql.install_as_MySQLdb() 83 | 84 | sql_url = 'mysql+pymysql://%s:%s@%s:%s/%s?charset=utf8' % (args['username'],args['password'],args['host'],args['port'],args['db']) 85 | 86 | app.config['SQLALCHEMY_DATABASE_URI'] = sql_url 87 | app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True 88 | 89 | db = SQLAlchemy(app) 90 | app.db = db.engine 91 | app.driver = MysqlDb 92 | 93 | 94 | 95 | 96 | 97 | 98 | if __name__ == "__main__": 99 | 100 | pass 101 | 102 | # app.run() 103 | -------------------------------------------------------------------------------- /webServer/customer.py: -------------------------------------------------------------------------------- 1 | # coding=UTF-8 2 | from flask import Response,current_app,request,session 3 | from sqlalchemy import text 4 | from decimal import Decimal 5 | import json,time,datetime 6 | 7 | 8 | # 自定义跨域 接口 返回 class 9 | class ApiCorsResponse(): 10 | 11 | @staticmethod 12 | def response(data ,success = True,status_code= 200): 13 | if success : 14 | re_data = {'msg':'ok','data':data} 15 | else: 16 | re_data = {'msg': 'fail', 'error_info': data} 17 | 18 | 19 | rep = Response( 20 | response=json.dumps(re_data) + "\n" , 21 | status=status_code, 22 | mimetype= current_app.config["JSONIFY_MIMETYPE"], 23 | headers={ 24 | 'Access-Control-Allow-Origin':'*', 25 | 'Access-Control-Allow-Method':'GET,POST,OPTIONS,PUT,DELETE', 26 | 'Access-Control-Allow-Headers':'*', 27 | } 28 | ) 29 | 30 | 31 | return rep 32 | # 自定义函数class 33 | class Func(): 34 | 35 | split_save = ['day', 'week', 'month', 'year'] 36 | @classmethod 37 | def getTableName(cls,conf,data_engine): 38 | table_suffix = '' 39 | 40 | try: 41 | if data_engine == 'mysql': 42 | table = conf['table'] 43 | if data_engine == 'mongodb': 44 | table = conf['collection'] 45 | 46 | except KeyError as e: 47 | raise Exception('配置错误: %s not exists' % e.args) 48 | 49 | 50 | 51 | if 'split_save' in conf: 52 | 53 | if conf['split_save'] not in cls.split_save: 54 | raise Exception('webserver 配置项 split_save 只支持 %s 选项' % ','.join(cls.split_save)) 55 | 56 | if conf['split_save'] == 'day': 57 | 58 | table_suffix = time.strftime('%Y_%m_%d', time.localtime()) 59 | 60 | elif conf['split_save'] == 'week': 61 | 62 | now = datetime.datetime.now() 63 | this_week_start = now - datetime.timedelta(days=now.weekday()) 64 | this_week_end = now + datetime.timedelta(days=6 - now.weekday()) 65 | 66 | table_suffix = datetime.datetime.strftime(this_week_start, '%Y_%m_%d') + datetime.datetime.strftime( 67 | this_week_end, '_%d') 68 | 69 | elif conf['split_save'] == 'month': 70 | 71 | table_suffix = time.strftime('%Y_%m', time.localtime()) 72 | elif conf['split_save'] == 'year': 73 | 74 | table_suffix = time.strftime('%Y', time.localtime()) 75 | 76 | if len(table_suffix): 77 | table = table + '_' + table_suffix 78 | 79 | 80 | return table 81 | @classmethod 82 | def fetchone(cls,resultObj): 83 | return cls.fetchall(resultObj)[0] 84 | 85 | @classmethod 86 | def fetchall(cls, resultObj): 87 | _list = [] 88 | for i in resultObj: 89 | _dict = {} 90 | item = i.items() 91 | 92 | for j in item: 93 | if isinstance(j[1],Decimal): 94 | vl = float(Decimal(j[1]).quantize(Decimal('.001'))) 95 | _dict[j[0]] = vl 96 | else: 97 | _dict[j[0]] = j[1] 98 | 99 | _list.append(_dict) 100 | 101 | return _list 102 | 103 | 104 | 105 | 106 | 107 | 108 | -------------------------------------------------------------------------------- /webServer/templates/home/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Document 8 | 9 | 10 | 11 | 12 |
13 |

web流量数据可视化实时监控系统

14 |
当前时间：
15 |
16 |
17 | 18 |
19 |
20 |

21 | 请求量最高的IP TOP50 22 | 23 |

24 |
25 |
26 |
27 |
28 |

最近10分钟每分钟流量情况

29 |
30 |
31 |
32 |
33 |

热门接口URL请求TOP 10

34 |
35 |
36 |
37 |
38 | 39 | 40 |
41 |
42 |
43 |
    44 |
  • 45 |
  • 46 |
47 |
48 |
49 |
    50 |
  • 今日IP数
    • 51 |
  • 今日PV数
    • 52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 | 63 | 64 |
65 |
66 |

非200状态码

67 |
68 |
69 |
70 |
71 |

最近10分钟每分钟IP/PV

72 |
73 |
74 |
75 |
76 |

搜索引擎蜘蛛占比

77 |
78 |
79 |
80 |
81 |
82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 110 | 111 | 112 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | # coding=UTF-8 2 | import time 3 | 4 | from Src.Core import OutputCustomer,Reader,Base 5 | from multiprocessing import Process 6 | from threading import Thread 7 | from webServer.start import start_web 8 | import multiprocessing,os,click 9 | 10 | 11 | 12 | 13 | def runReader(log_files_conf,config_name): 14 | 15 | r = Reader(log_file_conf=log_files_conf ,config_name=config_name) 16 | 17 | pushQueue = ['pushDataToQueue'] * multiprocessing.cpu_count() 18 | jobs = ['readLog','cutFile'] + pushQueue 19 | 20 | t = [] 21 | for i in jobs: 22 | th = Thread(target=r.runMethod, args=(i, )) 23 | t.append(th) 24 | 25 | for i in t: 26 | i.start() 27 | 28 | for i in t: 29 | i.join() 30 | 31 | def customer( config_name ): 32 | 33 | OutputCustomer(config_name).saveToStorage() 34 | 35 | def analysis(confg_name): 36 | 37 | OutputCustomer(confg_name).watchTraffic() 38 | 39 | def getLogFilsDict(base): 40 | logFiles = [] 41 | 42 | for i in list(base.conf): 43 | if 'inputer.log_file' in i: 44 | item = dict(base.conf[i]) 45 | item['app_name'] = i.split('.')[-1] 46 | logFiles.append(item) 47 | 48 | return logFiles 49 | 50 | 51 | def startInputer(base , config): 52 | 53 | logFiles = getLogFilsDict(base) 54 | 55 | plist = [] 56 | for i in logFiles: 57 | p = Process(target=runReader, args=(i, config,)) 58 | plist.append(p) 59 | 60 | for i in plist: 61 | i.start() 62 | 63 | for i in plist: 64 | i.join() 65 | 66 | def startOutputer(base , config): 67 | p_list = [] 68 | for start_webi in range(int(base.conf['outputer']['worker_process_num'])): 69 | p = Process(target=customer, args=(config,)) 70 | p_list.append(p) 71 | 72 | for i in p_list: 73 | i.start() 74 | 75 | for i in p_list: 76 | i.join() 77 | 78 | @click.command() 79 | @click.option('-r', '--run', help="run type" ,type=click.Choice(['inputer', 'outputer','traffic','web'])) 80 | @click.option('-s', '--stop', help="stop the proccess" ,type=click.Choice(['inputer', 'outputer'])) 81 | @click.option('-c', '--config', help="config file name" ) 82 | def enter(run,stop,config): 83 | 84 | if (config == None): 85 | print('please use "-c" to bind config.ini file') 86 | exit() 87 | 88 | base = Base(config_name=config) 89 | 90 | if (run == 'inputer'): 91 | 92 | pid = os.fork() 93 | if pid > 0 : 94 | exit() 95 | else: 96 | startInputer(base, config) 97 | 98 | 99 | 100 | if (run == 'outputer'): 101 | pid = os.fork() 102 | if pid > 0: 103 | exit() 104 | else: 105 | startOutputer(base, config) 106 | 107 | 108 | 109 | if (run == 'traffic'): 110 | 111 | analysis(config) 112 | 113 | if (run == 'web'): 114 | web_conf = dict(base.conf['web']) 115 | web_conf[web_conf['data_engine']] = dict(base.conf[web_conf['data_engine']]) 116 | start_web(web_conf) 117 | 118 | 119 | if (stop ): 120 | if isinstance(config,str) and len(config) : 121 | cmd = 'ps -ax | grep "main.py -r %s -c %s"' % (stop, config) 122 | else: 123 | cmd = 'ps -ax | grep "main.py -r %s"' % stop 124 | 125 | res = os.popen(cmd) 126 | pids = [] 127 | print('|============================================================') 128 | for i in res.readlines(): 129 | if i.find('grep') != -1: 130 | continue 131 | print('| %s ' % i.strip()) 132 | pids.append(i.strip().split(' ')[0]) 133 | 134 | 135 | if len(pids) == 0: 136 | print('| %s is not running ' % stop) 137 | print('|============================================================') 138 | exit('nothing happened . bye bye !') 139 | 140 | 141 | print('|============================================================') 142 | 143 | confirm = input('confirm: please enter [ yes | y ] or [ no | n ] : ') 144 | 145 | if confirm in ['yes','y'] and len(pids) > 0: 146 | 147 | os.popen('kill %s' % ' '.join(pids)) 148 | exit('pid: %s was killed and %s is stoped. bye bye !' % (' '.join(pids) ,stop) ) 149 | else: 150 | exit('nothing happened . bye bye !') 151 | 152 | 153 | 154 | 155 | if __name__ == "__main__": 156 | 157 | enter() 158 | -------------------------------------------------------------------------------- /ParserAdapter/BaseAdapter.py: -------------------------------------------------------------------------------- 1 | from abc import abstractmethod,ABCMeta 2 | from Src.ip2Region import Ip2Region 3 | import os,time 4 | 5 | # 解析错误 6 | class ParseError(Exception):pass 7 | # 预编译错误 8 | class ReCompile(Exception):pass 9 | 10 | 11 | 12 | class Adapter(): 13 | __metaclass__ = ABCMeta 14 | 15 | LOG_FORMAT_SPLIT_TAG = '<@>' 16 | 17 | ip_parser = None 18 | log_line_pattern_dict = {} 19 | 20 | def __init__(self,*args,**kwargs): 21 | 22 | if self.ip_parser == None: 23 | ip_data_path = os.path.dirname(os.path.dirname(__file__)) + '/Src/ip2region.db' 24 | 25 | if not os.path.exists(ip_data_path): 26 | raise FileNotFoundError('ip2region.db 数据库不存在') 27 | 28 | self.ip_parser = Ip2Region(ip_data_path) 29 | 30 | @abstractmethod 31 | def getLogFormat(self): pass 32 | 33 | @abstractmethod 34 | def parse(self): pass 35 | 36 | @abstractmethod 37 | def getLogFormatByConfStr(self): pass 38 | 39 | def getLoggerFormatByServerConf(self):pass 40 | 41 | @abstractmethod 42 | def rotatelog(self):pass 43 | 44 | # 解析 ip 新增地域字段 isp city city_id province country 45 | @abstractmethod 46 | def parse_ip_to_area(self,ip): 47 | data = {} 48 | try: 49 | res = self.ip_parser.memorySearch(ip) 50 | 51 | _arg = res['region'].decode('utf-8').split('|') 52 | 53 | # _城市Id|国家|区域|省份|城市|ISP_ 54 | data['isp'] = _arg[-1] 55 | data['city'] = _arg[-2] 56 | data['city_id'] = int(res['city_id']) 57 | data['province'] = _arg[-3] 58 | data['country'] = _arg[0] 59 | 60 | except Exception as e: 61 | data['isp'] = 0 62 | data['city'] = 0 63 | data['city_id'] = 0 64 | data['province'] = 0 65 | data['country'] = 0 66 | 67 | return data 68 | 69 | # 解析requset字段 变成 request_method ,request_url ,args ,server_protocol 70 | @abstractmethod 71 | def parse_request_to_extend(self,request_data): 72 | data = {} 73 | 74 | try: 75 | 76 | 77 | if len(request_data.strip()) == 0: 78 | data['request_method'] = '' 79 | data['request_url'] = '' 80 | data['args'] = '' 81 | data['server_protocol'] = '' 82 | return data 83 | 84 | _strarr = request_data.split(' ') 85 | 86 | if(len(_strarr) == 1) : 87 | data['request_method'] = '' 88 | data['request_url'] = _strarr[0] 89 | data['args'] = '' 90 | data['server_protocol'] = '' 91 | return data 92 | 93 | 94 | 95 | data['request_method'] = _strarr[0] 96 | _url = _strarr[1].split('?') 97 | 98 | if len(_url) > 1: 99 | data['request_url'] = _url[0] 100 | data['args'] = _url[1] 101 | else: 102 | data['request_url'] = _url[0] 103 | data['args'] = '' 104 | 105 | 106 | data['server_protocol'] = _strarr[2] 107 | 108 | 109 | if 'request_uri' in data: 110 | _strarr = data['request_uri'].split('?') 111 | 112 | data['request_url'] = _url[0] 113 | data['args'] = _url[1] 114 | 115 | 116 | return data 117 | except IndexError as e: 118 | # 异常攻击流量数据 解析错误 不在抛出exception 直接原样返回数据 以供分析 119 | data['request_method'] = '' 120 | data['request_url'] = request_data 121 | data['args'] = '' 122 | data['server_protocol'] = '' 123 | 124 | return data 125 | 126 | 127 | # 解析 time_iso8601 time_local 变成 time_str timestamp 128 | @abstractmethod 129 | def parse_time_to_str(self,time_type , time_data): 130 | 131 | data = {} 132 | time_data = time_data.replace('[', '').replace(']', '') 133 | 134 | if 'time_iso8601' == time_type: 135 | _strarr = time_data.split('+') 136 | ts = time.strptime(_strarr[0], '%Y-%m-%dT%H:%M:%S') 137 | 138 | 139 | 140 | if 'time_local' == time_type: 141 | _strarr = time_data.split('+') 142 | ts = time.strptime(_strarr[0].strip(), '%d/%b/%Y:%H:%M:%S') 143 | 144 | 145 | data['time_str'] = time.strftime('%Y-%m-%d %H:%M:%S', ts) 146 | data['timestamp'] = int(time.mktime(ts)) 147 | 148 | 149 | 150 | 151 | return data -------------------------------------------------------------------------------- /webServer/static/js/myMap.js: -------------------------------------------------------------------------------- 1 | //固定的深圳市 2 | var toCityName = GEO[440300] 3 | // 1. 实例化对象 4 | var map_chart = echarts.init(document.querySelector(".map .chart")); 5 | window.addEventListener("resize", function() { 6 | map_chart.resize(); 7 | }); 8 | window.chart_load_func['map_chart'] = function() { 9 | $.ajax({ 10 | url:host + '/get_request_num_by_province', 11 | type:'GET', 12 | headers: { 'Authorization':Authorization,}, 13 | success:function(msg){ 14 | var api_data = msg.data 15 | var province_geo = GEO[100000] 16 | var map_lines_data = [] 17 | var map_effectScatter_data = [] 18 | for (var i= 0 ; i < api_data.length ;i++){ 19 | let province_name = api_data[i]['province'].slice(0,2) 20 | for (var j = 0 ; j < province_geo.childrenNum ; j++){ 21 | if ( province_geo.children[j]['name'].slice(0,2) == province_name){ 22 | let _lines_data = { 23 | fromName: province_geo.children[j]['name'], 24 | toName: toCityName.name, 25 | coords: [province_geo.children[j].center, toCityName.center], 26 | value: api_data[i]['value'] 27 | } 28 | 29 | let _effectScatter_data = { 30 | name :province_geo.children[j]['name'], 31 | value: province_geo.children[j].center.concat(api_data[i]['value']) 32 | } 33 | map_lines_data.push(_lines_data) 34 | map_effectScatter_data.push(_effectScatter_data) 35 | break 36 | } 37 | } 38 | } 39 | 40 | var color = ["#fff"]; //航线的颜色 41 | var series = []; 42 | series.push( 43 | { 44 | name: " 运动方向", 45 | type: "lines", 46 | zlevel: 1, 47 | effect: { 48 | show: true, 49 | period: 6, 50 | trailLength: 0.7, 51 | color: "white", //arrow箭头的颜色 52 | symbolSize: 3 53 | }, 54 | lineStyle: { 55 | normal: { 56 | color: color[i], 57 | width: 0, 58 | curveness: 0.2 59 | } 60 | }, 61 | data: map_lines_data 62 | }, 63 | // { 64 | // name: "运动方向 实心线", 65 | // type: "lines", 66 | // zlevel: 2, 67 | // symbol: ["none", "arrow"], 68 | // symbolSize: 10, 69 | // effect: { 70 | // show: true, 71 | // period: 6, 72 | // trailLength: 0, 73 | // symbol: planePath, 74 | // symbolSize: 15 75 | // }, 76 | // lineStyle: { 77 | // normal: { 78 | // color: "#fff", 79 | // width: 1, 80 | // opacity: 0.6, 81 | // curveness: 0.2 82 | // } 83 | // }, 84 | // data: map_lines_data 85 | // }, 86 | { 87 | name: "effectScatter Top3", 88 | type: "effectScatter", 89 | coordinateSystem: "geo", 90 | zlevel: 2, 91 | rippleEffect: { 92 | brushType: "stroke" 93 | }, 94 | label: { 95 | color:'#fff', 96 | normal: { 97 | show: true, 98 | position: "right", 99 | formatter: function(params){ 100 | 101 | return params.data.name +' ：' + params.data.value[2] 102 | } 103 | } 104 | }, 105 | symbolSize: function(val) { 106 | return val[2] / 10000; 107 | // return 10; 108 | }, 109 | itemStyle: { 110 | normal: { 111 | color: '#fff' 112 | }, 113 | emphasis: { 114 | areaColor: "#fff" 115 | } 116 | }, 117 | data: map_effectScatter_data 118 | } 119 | ); 120 | 121 | var option = { 122 | 123 | geo: { 124 | map: "china", 125 | label: { 126 | show:false, 127 | color:"#fff", 128 | emphasis:{ 129 | itemStyle: { 130 | areaColor: null, 131 | shadowOffsetX: 0, 132 | shadowOffsetY: 0, 133 | shadowBlur: 20, 134 | borderWidth: 0, 135 | shadowColor: 'rgba(0, 0, 0, 0.5)' 136 | } 137 | } 138 | }, 139 | roam: false, 140 | // 放大我们的地图 141 | zoom: 1.2, 142 | itemStyle: { 143 | normal: { 144 | areaColor: "rgba(43, 196, 243, 0.42)", 145 | borderColor: "rgba(43, 196, 243, 1)", 146 | borderWidth: 1 147 | }, 148 | emphasis: { 149 | areaColor: "#2f89cf" 150 | } 151 | } 152 | }, 153 | series: series 154 | }; 155 | map_chart.setOption(option); 156 | 157 | } 158 | 159 | }) 160 | 161 | } 162 | 163 | -------------------------------------------------------------------------------- /QueueAdapter/Redis.py: -------------------------------------------------------------------------------- 1 | from QueueAdapter.BaseAdapter import Adapter 2 | from redis import Redis,exceptions as redis_exceptions 3 | import time,threading,os,json 4 | 5 | 6 | class QueueAp(Adapter): 7 | 8 | db = None 9 | runner = None 10 | 11 | 12 | @classmethod 13 | def initQueue(cls,runnerObject): 14 | self = cls() 15 | self.runner = runnerObject 16 | self.conf = self.runner.conf 17 | self.logging = self.runner.logging 18 | 19 | try: 20 | 21 | self.db = Redis( 22 | host=self.conf['redis']['host'], 23 | port=int(self.conf['redis']['port']), 24 | password=str(self.conf['redis']['password']), 25 | db=self.conf['redis']['db'], 26 | ) 27 | 28 | 29 | except redis_exceptions.RedisError as e: 30 | self.runner.event['stop'] = e.args[0] 31 | 32 | 33 | return self 34 | 35 | 36 | def pushDataToQueue(self ): 37 | 38 | pipe = self.db.pipeline() 39 | 40 | retry_reconnect_time = 0 41 | 42 | while True: 43 | time.sleep(1) 44 | if self.runner.event['stop']: 45 | self.logging.error( '%s ; pushQueue threading stop pid: %s ---- tid: %s ' % (self.runner.event['stop'] ,os.getpid() ,threading.get_ident() )) 46 | return 47 | 48 | try: 49 | 50 | # 重试连接queue的时候; 不再从 dqueue 中拿数据 51 | if retry_reconnect_time == 0: 52 | 53 | start_time = time.perf_counter() 54 | # self.logging.debug("\n pushQueue -------pid: %s -tid: %s- started \n" % ( os.getpid(), threading.get_ident())) 55 | 56 | for i in range(self.runner.max_batch_push_queue_size): 57 | try: 58 | line = self.runner.dqueue.pop() 59 | except IndexError as e: 60 | # self.logging.info("\n pushQueue -------pid: %s -tid: %s- wait for data ;queue len: %s---- start \n" % ( os.getpid(), threading.get_ident(), len(list(self.dqueue)))) 61 | break 62 | 63 | data = {} 64 | data['node_id'] = self.runner.node_id 65 | data['app_name'] = self.runner.app_name 66 | data['log_format_name'] = self.runner.log_format_name 67 | 68 | data['line'] = line.strip() 69 | 70 | try: 71 | # 日志的原始字符串 72 | data['log_format_str'] = self.runner.server_conf[self.runner.log_format_name]['log_format_str'].strip() 73 | # 日志中提取出的日志变量 74 | data['log_format_vars'] = self.runner.server_conf[self.runner.log_format_name]['log_format_vars'].strip() 75 | 76 | except KeyError as e: 77 | self.runner.event['stop'] = self.runner.log_format_name + '日志格式不存在' 78 | break 79 | 80 | 81 | data = json.dumps(data) 82 | pipe.lpush(self.runner.queue_key, data) 83 | 84 | 85 | res = pipe.execute() 86 | if len(res): 87 | retry_reconnect_time = 0 88 | end_time = time.perf_counter() 89 | self.logging.debug("\n pushQueue -------pid: %s -tid: %s- push data to queue :%s ; queue_len : %s----耗时:%s \n" 90 | % (os.getpid(), threading.get_ident(), len(res),self.db.llen(self.runner.queue_key), round(end_time - start_time, 2))) 91 | 92 | 93 | except redis_exceptions.RedisError as e: 94 | 95 | 96 | retry_reconnect_time = retry_reconnect_time + 1 97 | 98 | if retry_reconnect_time >= self.runner.max_retry_reconnect_time : 99 | self.runner.event['stop'] = 'pushQueue 重试连接 queue 超出最大次数' 100 | else: 101 | time.sleep(2) 102 | self.logging.debug('pushQueue -------pid: %s -tid: %s- push data fail; reconnect Queue %s times' % (os.getpid() , threading.get_ident() , retry_reconnect_time)) 103 | 104 | continue 105 | pass 106 | 107 | def getDataFromQueue(self): 108 | start_time = time.perf_counter() 109 | 110 | pipe = self.db.pipeline() 111 | 112 | db_queue_len = self.db.llen(self.runner.queue_key) 113 | 114 | if db_queue_len >= self.runner.max_batch_insert_db_size: 115 | num = self.runner.max_batch_insert_db_size 116 | else: 117 | num = db_queue_len 118 | 119 | for i in range(num): 120 | # 后进先出 121 | pipe.rpop(self.runner.queue_key) 122 | 123 | queue_list = pipe.execute() 124 | 125 | 126 | # 过滤掉None 127 | if queue_list.count(None): 128 | queue_list = list(filter(None, queue_list)) 129 | 130 | end_time = time.perf_counter() 131 | if len(queue_list): 132 | self.logging.debug("\n pid: %s ;tid : %s-- take len: %s ; queue db len : %s ; ----end 耗时: %s \n" % 133 | (os.getpid(), threading.get_ident(), len(queue_list), self.db.llen(self.runner.queue_key), 134 | round(end_time - start_time, 2))) 135 | 136 | return queue_list 137 | 138 | 139 | # 退回队列 140 | def rollBackToQueue(self,data): 141 | pipe = self.db.pipeline() 142 | for i in data: 143 | i = bytes(i,encoding='utf-8') 144 | pipe.rpush(self.runner.queue_key,i) 145 | 146 | pipe.execute() 147 | 148 | 149 | def getDataCountNum(self): 150 | return self.db.llen(self.runner.queue_key) 151 | 152 | -------------------------------------------------------------------------------- /webServer/static/css/index.css: -------------------------------------------------------------------------------- 1 | * { 2 | margin: 0; 3 | padding: 0; 4 | box-sizing: border-box; 5 | } 6 | li { 7 | list-style: none; 8 | } 9 | @font-face { 10 | font-family: electronicFont; 11 | src: url(../font/DS-DIGIT.TTF); 12 | } 13 | body { 14 | font-family: Arial, Helvetica, sans-serif; 15 | margin: 0; 16 | padding: 0; 17 | background-repeat: no-repeat; 18 | background-color: #06164A; 19 | background-size: cover; 20 | /* 行高是字体1.15倍 */ 21 | line-height: 1.15; 22 | } 23 | header { 24 | position: relative; 25 | height: 1.25rem; 26 | background: url(../images/head_bg.png) no-repeat top center; 27 | background-size: 100% 100%; 28 | } 29 | header h1 { 30 | font-size: 0.475rem; 31 | color: #fff; 32 | text-align: center; 33 | line-height: 1rem; 34 | } 35 | header .showTime { 36 | position: absolute; 37 | top: 0; 38 | right: 0.375rem; 39 | line-height: 0.9375rem; 40 | font-size: 0.25rem; 41 | color: rgba(255, 255, 255, 0.7); 42 | } 43 | .mainbox { 44 | min-width: 1024px; 45 | /*max-width: 1920px;*/ 46 | padding: 0.125rem 0.125rem 0; 47 | display: flex; 48 | } 49 | .mainbox .column { 50 | flex: 3; 51 | } 52 | .mainbox .column:nth-child(2) { 53 | flex: 5; 54 | margin: 0 0.125rem 0.1875rem; 55 | overflow: hidden; 56 | } 57 | .panel { 58 | position: relative; 59 | height: 3.875rem; 60 | border: 1px solid rgba(25, 186, 139, 0.17); 61 | background: rgba(255, 255, 255, 0.04) url(../images/line\(1\).png); 62 | padding: 0 0.1875rem 0.5rem; 63 | margin-bottom: 0.1875rem; 64 | } 65 | .panel::before { 66 | position: absolute; 67 | top: 0; 68 | left: 0; 69 | content: ""; 70 | width: 10px; 71 | height: 10px; 72 | border-top: 2px solid #02a6b5; 73 | border-left: 2px solid #02a6b5; 74 | border-radius: 20%; 75 | } 76 | .panel::after { 77 | position: absolute; 78 | top: 0; 79 | right: 0; 80 | content: ""; 81 | width: 10px; 82 | height: 10px; 83 | border-top: 2px solid #02a6b5; 84 | border-right: 2px solid #02a6b5; 85 | border-radius: 20%; 86 | } 87 | .panel .panel-footer { 88 | position: absolute; 89 | left: 0; 90 | bottom: 0; 91 | width: 100%; 92 | } 93 | .panel .panel-footer::before { 94 | position: absolute; 95 | bottom: 0; 96 | left: 0; 97 | content: ""; 98 | width: 10px; 99 | height: 10px; 100 | border-bottom: 2px solid #02a6b5; 101 | border-left: 2px solid #02a6b5; 102 | border-radius: 20%; 103 | } 104 | .panel .panel-footer::after { 105 | position: absolute; 106 | bottom: 0; 107 | right: 0; 108 | content: ""; 109 | width: 10px; 110 | height: 10px; 111 | border-bottom: 2px solid #02a6b5; 112 | border-right: 2px solid #02a6b5; 113 | border-radius: 20%; 114 | } 115 | .panel h2 { 116 | height: 0.6rem; 117 | line-height: 0.6rem; 118 | text-align: center; 119 | color: #fff; 120 | font-size: 0.25rem; 121 | font-weight: 400; 122 | } 123 | .panel h2 a { 124 | margin: 0 0.1875rem; 125 | color: #fff; 126 | text-decoration: underline; 127 | } 128 | .panel .chart { 129 | height: 3rem; 130 | } 131 | .no { 132 | background: rgba(101, 132, 226, 0.1); 133 | padding: 0.1875rem; 134 | } 135 | .no .no-hd { 136 | position: relative; 137 | border: 1px solid rgba(25, 186, 139, 0.17); 138 | } 139 | .no .no-hd::before { 140 | content: ""; 141 | position: absolute; 142 | width: 30px; 143 | height: 10px; 144 | border-top: 2px solid #02a6b5; 145 | border-left: 2px solid #02a6b5; 146 | top: 0; 147 | left: 0; 148 | } 149 | .no .no-hd::after { 150 | content: ""; 151 | position: absolute; 152 | width: 30px; 153 | height: 10px; 154 | border-bottom: 2px solid #02a6b5; 155 | border-right: 2px solid #02a6b5; 156 | right: 0; 157 | bottom: 0; 158 | } 159 | .no .no-hd ul { 160 | display: flex; 161 | } 162 | .no .no-hd ul li { 163 | position: relative; 164 | flex: 1; 165 | text-align: center; 166 | height: 1rem; 167 | line-height: 1rem; 168 | font-size: 0.875rem; 169 | color: #ffeb7b; 170 | padding: 0.05rem 0; 171 | font-family: electronicFont; 172 | font-weight: bold; 173 | } 174 | .no .no-hd ul li:first-child::after { 175 | content: ""; 176 | position: absolute; 177 | height: 50%; 178 | width: 1px; 179 | background: rgba(255, 255, 255, 0.2); 180 | right: 0; 181 | top: 25%; 182 | } 183 | .no .no-bd ul { 184 | display: flex; 185 | } 186 | .no .no-bd ul li { 187 | flex: 1; 188 | height: 0.5rem; 189 | line-height: 0.5rem; 190 | text-align: center; 191 | font-size: 0.225rem; 192 | color: rgba(255, 255, 255, 0.7); 193 | padding-top: 0.125rem; 194 | } 195 | .map { 196 | position: relative; 197 | height: 10.125rem; 198 | } 199 | .map .chart { 200 | position: absolute; 201 | top: 0; 202 | left: 0; 203 | z-index: 5; 204 | height: 10.125rem; 205 | width: 100%; 206 | } 207 | .map .map1, 208 | .map .map2, 209 | .map .map3 { 210 | position: absolute; 211 | top: 50%; 212 | left: 50%; 213 | transform: translate(-50%, -50%); 214 | width: 6.475rem; 215 | height: 6.475rem; 216 | background: url(../images/map.png) no-repeat; 217 | background-size: 100% 100%; 218 | opacity: 0.3; 219 | } 220 | .map .map2 { 221 | width: 8.0375rem; 222 | height: 8.0375rem; 223 | background-image: url(../images/lbx.png); 224 | opacity: 0.6; 225 | animation: rotate 15s linear infinite; 226 | z-index: 2; 227 | } 228 | .map .map3 { 229 | width: 7.075rem; 230 | height: 7.075rem; 231 | background-image: url(../images/jt.png); 232 | animation: rotate1 10s linear infinite; 233 | } 234 | @keyframes rotate { 235 | from { 236 | transform: translate(-50%, -50%) rotate(0deg); 237 | } 238 | to { 239 | transform: translate(-50%, -50%) rotate(360deg); 240 | } 241 | } 242 | @keyframes rotate1 { 243 | from { 244 | transform: translate(-50%, -50%) rotate(0deg); 245 | } 246 | to { 247 | transform: translate(-50%, -50%) rotate(-360deg); 248 | } 249 | } 250 | @media screen and (max-width: 1024px) { 251 | html { 252 | font-size: 42px !important; 253 | } 254 | } 255 | @media screen and (min-width: 1920) { 256 | html { 257 | font-size: 80px !important; 258 | } 259 | } 260 | -------------------------------------------------------------------------------- /QueueAdapter/Mongodb.py: -------------------------------------------------------------------------------- 1 | from QueueAdapter.BaseAdapter import Adapter 2 | from pymongo import MongoClient,errors as pyerrors 3 | import time,threading,os,json 4 | 5 | try: 6 | # Python 3.x 7 | from urllib.parse import quote_plus 8 | except ImportError: 9 | # Python 2.x 10 | from urllib import quote_plus 11 | 12 | class QueueAp(Adapter): 13 | 14 | db = None 15 | runner = None 16 | 17 | @classmethod 18 | def initQueue(cls,runnerObject): 19 | self = cls() 20 | self.runner = runnerObject 21 | self.conf = self.runner.conf 22 | self.logging = self.runner.logging 23 | 24 | if self.conf['mongodb']['username'] and self.conf['mongodb']['password']: 25 | mongo_url = 'mongodb://%s:%s@%s:%s/?authSource=%s' % \ 26 | ( 27 | quote_plus(self.conf['mongodb']['username']), 28 | quote_plus(self.conf['mongodb']['password']), 29 | self.conf['mongodb']['host'], 30 | int(self.conf['mongodb']['port']), 31 | self.conf['mongodb']['db'] 32 | ) 33 | 34 | else: 35 | mongo_url = 'mongodb://%s:%s/?authSource=%s' % \ 36 | ( 37 | self.conf['mongodb']['host'], 38 | int(self.conf['mongodb']['port']), 39 | self.conf['mongodb']['db'] 40 | ) 41 | 42 | mongodb = MongoClient(mongo_url) 43 | self.db = mongodb[self.conf['mongodb']['db']] 44 | 45 | # 创建一个过期索引 过时间 46 | ttl_seconds = 120 47 | self.db[self.runner.queue_key].create_index([("ttl", 1)], expireAfterSeconds=ttl_seconds) 48 | self.db[self.runner.queue_key].create_index([("out_queue", 1)], background=True) 49 | self.db[self.runner.queue_key].create_index([("add_time", -1)], background=True) 50 | 51 | # self.db['asdasd'] 52 | 53 | return self 54 | 55 | 56 | def pushDataToQueue(self ): 57 | 58 | retry_reconnect_time = 0 59 | 60 | while True: 61 | time.sleep(1) 62 | if self.runner.event['stop']: 63 | self.logging.error('%s ; pushQueue threading stop pid: %s ---- tid: %s ' % ( self.runner.event['stop'], os.getpid(), threading.get_ident())) 64 | return 65 | 66 | try: 67 | 68 | # 重试连接queue的时候; 不再从 dqueue 中拿数据 69 | if retry_reconnect_time == 0: 70 | _queuedata = [] 71 | 72 | start_time = time.perf_counter() 73 | 74 | for i in range(self.runner.max_batch_push_queue_size): 75 | try: 76 | line = self.runner.dqueue.pop() 77 | except IndexError as e: 78 | break 79 | 80 | q_data = {} 81 | data = {} 82 | data['node_id'] = self.runner.node_id 83 | data['app_name'] = self.runner.app_name 84 | data['log_format_name'] = self.runner.log_format_name 85 | 86 | data['line'] = line.strip() 87 | 88 | try: 89 | data['log_format_str'] = self.runner.server_conf[self.runner.log_format_name].strip() 90 | except KeyError as e: 91 | self.runner.event['stop'] = self.runner.log_format_name + '日志格式不存在' 92 | break 93 | 94 | data = json.dumps(data) 95 | 96 | q_data['out_queue'] = 0 97 | q_data['add_time'] = time.time() 98 | q_data['data'] = data 99 | 100 | _queuedata.append(q_data) 101 | 102 | 103 | if len(_queuedata): 104 | res = self.db[self.runner.queue_key].insert_many(_queuedata, ordered=False) 105 | 106 | end_time = time.perf_counter() 107 | self.logging.debug( 108 | "\n pushQueue -------pid: %s -tid: %s- push data to queue :%s ; queue_len : %s----耗时:%s \n" 109 | % (os.getpid(), threading.get_ident(), len(res.inserted_ids), 0, 110 | round(end_time - start_time, 2))) 111 | 112 | 113 | except pyerrors.PyMongoError as e: 114 | 115 | retry_reconnect_time = retry_reconnect_time + 1 116 | 117 | if retry_reconnect_time >= self.runner.max_retry_reconnect_time: 118 | self.logging.error('pushQueue 重试连接 queue 超出最大次数') 119 | raise Exception('pushQueue 重试连接 queue 超出最大次数') 120 | else: 121 | time.sleep(1) 122 | self.logging.debug('pushQueue -------pid: %s -tid: %s- push data fail: %s ; reconnect Queue %s times' % ( 123 | os.getpid(), e.args, threading.get_ident(), retry_reconnect_time)) 124 | 125 | continue 126 | 127 | 128 | def getDataFromQueue(self): 129 | db_queue_len = self.getDataCountNum() 130 | 131 | if db_queue_len == 0: 132 | return [] 133 | 134 | if db_queue_len >= self.runner.max_batch_insert_db_size: 135 | takenum = self.runner.max_batch_insert_db_size 136 | else: 137 | takenum = db_queue_len 138 | 139 | 140 | start_time = time.perf_counter() 141 | 142 | _data = [] 143 | for i in range(takenum): 144 | res = self.db[self.runner.queue_key].find_and_modify( 145 | query={'out_queue': 0}, 146 | update={'$set': {'out_queue': 1}, '$currentDate': {'ttl': True}}, 147 | sort=[('add_time', -1)] 148 | ) 149 | 150 | if res: 151 | _data.append(res['data']) 152 | 153 | end_time = time.perf_counter() 154 | self.logging.debug('\n pid: %s take data from queue %s ,queue db len : %s ;.耗时: %s \n' % ( 155 | os.getpid(), len(_data),self.getDataCountNum(), round(end_time - start_time ,3))) 156 | 157 | return _data 158 | 159 | 160 | 161 | 162 | def getDataCountNum(self): 163 | return self.db[self.runner.queue_key].count_documents({'out_queue':0}) 164 | -------------------------------------------------------------------------------- /webServer/static/css/index.less: -------------------------------------------------------------------------------- 1 | * { 2 | margin: 0; 3 | padding: 0; 4 | box-sizing: border-box; 5 | } 6 | li { 7 | list-style: none; 8 | } 9 | @font-face { 10 | font-family: electronicFont; 11 | src: url(../font/DS-DIGIT.TTF); 12 | } 13 | body { 14 | font-family: Arial, Helvetica, sans-serif; 15 | margin: 0; 16 | padding: 0; 17 | /* 背景图定位 / 背景图尺寸 cover 完全铺满容器 contain 完整显示在容器内 */ 18 | background: url(../images/bg.jpg) no-repeat #000; 19 | background-size: cover; 20 | /* 行高是字体1.15倍 */ 21 | line-height: 1.15; 22 | } 23 | header { 24 | position: relative; 25 | height: 1.25rem; 26 | background: url(../images/head_bg.png) no-repeat top center; 27 | background-size: 100% 100%; 28 | h1 { 29 | font-size: 0.475rem; 30 | color: #fff; 31 | text-align: center; 32 | line-height: 1rem; 33 | } 34 | .showTime { 35 | position: absolute; 36 | top: 0; 37 | right: 0.375rem; 38 | line-height: 0.9375rem; 39 | font-size: 0.25rem; 40 | color: rgba(255, 255, 255, 0.7); 41 | } 42 | } 43 | .mainbox { 44 | min-width: 1024px; 45 | max-width: 1920px; 46 | padding: 0.125rem 0.125rem 0; 47 | display: flex; 48 | .column { 49 | flex: 3; 50 | &:nth-child(2) { 51 | flex: 5; 52 | margin: 0 0.125rem 0.1875rem; 53 | overflow: hidden; 54 | } 55 | } 56 | } 57 | .panel { 58 | position: relative; 59 | height: 3.875rem; 60 | border: 1px solid rgba(25, 186, 139, 0.17); 61 | background: rgba(255, 255, 255, 0.04) url(../images/line\(1\).png); 62 | padding: 0 0.1875rem 0.5rem; 63 | margin-bottom: 0.1875rem; 64 | &::before { 65 | position: absolute; 66 | top: 0; 67 | left: 0; 68 | content: ""; 69 | width: 10px; 70 | height: 10px; 71 | border-top: 2px solid #02a6b5; 72 | border-left: 2px solid #02a6b5; 73 | border-radius: 20%; 74 | } 75 | &::after { 76 | position: absolute; 77 | top: 0; 78 | right: 0; 79 | content: ""; 80 | width: 10px; 81 | height: 10px; 82 | border-top: 2px solid #02a6b5; 83 | border-right: 2px solid #02a6b5; 84 | border-radius: 20%; 85 | } 86 | .panel-footer { 87 | position: absolute; 88 | left: 0; 89 | bottom: 0; 90 | width: 100%; 91 | &::before { 92 | position: absolute; 93 | bottom: 0; 94 | left: 0; 95 | content: ""; 96 | width: 10px; 97 | height: 10px; 98 | border-bottom: 2px solid #02a6b5; 99 | border-left: 2px solid #02a6b5; 100 | border-radius: 20%; 101 | } 102 | &::after { 103 | position: absolute; 104 | bottom: 0; 105 | right: 0; 106 | content: ""; 107 | width: 10px; 108 | height: 10px; 109 | border-bottom: 2px solid #02a6b5; 110 | border-right: 2px solid #02a6b5; 111 | border-radius: 20%; 112 | } 113 | } 114 | 115 | h2 { 116 | height: 0.6rem; 117 | line-height: 0.6rem; 118 | text-align: center; 119 | color: #fff; 120 | font-size: 0.25rem; 121 | font-weight: 400; 122 | a { 123 | margin: 0 0.1875rem; 124 | color: #fff; 125 | text-decoration: underline; 126 | } 127 | } 128 | .chart { 129 | height: 3rem; 130 | } 131 | } 132 | .no { 133 | background: rgba(101, 132, 226, 0.1); 134 | padding: 0.1875rem; 135 | .no-hd { 136 | position: relative; 137 | border: 1px solid rgba(25, 186, 139, 0.17); 138 | &::before { 139 | content: ""; 140 | position: absolute; 141 | width: 30px; 142 | height: 10px; 143 | border-top: 2px solid #02a6b5; 144 | border-left: 2px solid #02a6b5; 145 | top: 0; 146 | left: 0; 147 | } 148 | &::after { 149 | content: ""; 150 | position: absolute; 151 | width: 30px; 152 | height: 10px; 153 | border-bottom: 2px solid #02a6b5; 154 | border-right: 2px solid #02a6b5; 155 | right: 0; 156 | bottom: 0; 157 | } 158 | ul { 159 | display: flex; 160 | li { 161 | position: relative; 162 | flex: 1; 163 | text-align: center; 164 | height: 1rem; 165 | line-height: 1rem; 166 | font-size: 0.875rem; 167 | color: #ffeb7b; 168 | padding: 0.05rem 0; 169 | font-family: electronicFont; 170 | font-weight: bold; 171 | &:first-child::after { 172 | content: ""; 173 | position: absolute; 174 | height: 50%; 175 | width: 1px; 176 | background: rgba(255, 255, 255, 0.2); 177 | right: 0; 178 | top: 25%; 179 | } 180 | } 181 | } 182 | } 183 | .no-bd ul { 184 | display: flex; 185 | li { 186 | flex: 1; 187 | height: 0.5rem; 188 | line-height: 0.5rem; 189 | text-align: center; 190 | font-size: 0.225rem; 191 | color: rgba(255, 255, 255, 0.7); 192 | padding-top: 0.125rem; 193 | } 194 | } 195 | } 196 | .map { 197 | position: relative; 198 | height: 10.125rem; 199 | .chart { 200 | position: absolute; 201 | top: 0; 202 | left: 0; 203 | z-index: 5; 204 | height: 10.125rem; 205 | width: 100%; 206 | } 207 | .map1, 208 | .map2, 209 | .map3 { 210 | position: absolute; 211 | top: 50%; 212 | left: 50%; 213 | transform: translate(-50%, -50%); 214 | width: 6.475rem; 215 | height: 6.475rem; 216 | background: url(../images/map.png) no-repeat; 217 | background-size: 100% 100%; 218 | opacity: 0.3; 219 | } 220 | .map2 { 221 | width: 8.0375rem; 222 | height: 8.0375rem; 223 | background-image: url(../images/lbx.png); 224 | opacity: 0.6; 225 | animation: rotate 15s linear infinite; 226 | z-index: 2; 227 | } 228 | .map3 { 229 | width: 7.075rem; 230 | height: 7.075rem; 231 | background-image: url(../images/jt.png); 232 | animation: rotate1 10s linear infinite; 233 | } 234 | 235 | @keyframes rotate { 236 | from { 237 | transform: translate(-50%, -50%) rotate(0deg); 238 | } 239 | to { 240 | transform: translate(-50%, -50%) rotate(360deg); 241 | } 242 | } 243 | @keyframes rotate1 { 244 | from { 245 | transform: translate(-50%, -50%) rotate(0deg); 246 | } 247 | to { 248 | transform: translate(-50%, -50%) rotate(-360deg); 249 | } 250 | } 251 | } 252 | 253 | @media screen and (max-width: 1024px) { 254 | html { 255 | font-size: 42px !important; 256 | } 257 | } 258 | @media screen and (min-width: 1920) { 259 | html { 260 | font-size: 80px !important; 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | # 介绍 2 | 3 | > wLogger 介绍 4 | 5 | * 介绍 6 | 7 | wLogger 是一款集合 日志采集,日志解析持久化存储，web流量实时监控 。三位一体的web服务流量监控应用。 8 | 三大功能模块均可独立部署启用互不干扰。目前已内置 nginx 和 apache 的日志解析存储器，简单配置一下，开箱即用。 9 | 虽然市面上已经很多类似的开源日志采集监控服务比如goaccess，用了一圈之后始终没有一款用的特别舒心。 10 | 11 | * 它可以在日志采集的时候可以按照日志文件的大小，或者在指定时间内自动对日志进行切割日志，存储到指定的目录 （已测2W并发切割日志不丢数据） 12 | * 它可以不用像goaccess那样必须配置指定格式才能解析到数据，只用指定当前使用的 nginx/apache 日志格式名称 即可解析数据 13 | * 它可以指定不同的项目走不同的队列服务，分别解析存储到不同的数据库，完全可以自己按需灵活配置 14 | * 它天然支持分布式，日志采集服务队列已内置redis LIST结构，可自行拓展kafka ,mq等其它队列服务 15 | * 它支持自定义持久化存储引擎，日志解析持久化存储服务已内置 mongodb 和 mysql ,可自行拓展其它数据库 16 | * 简单配置，开箱即用，无入侵，高拓展，灵活配置，按需应用 17 | * 运行环境：python3+ linux平台 18 | 19 | 如果该项目有帮助到您，请不要吝啬随手给个star 20 | 21 | 您也可以从数据库中取数据自己定义流量监控的UI界面和数据展现方式; 22 | 23 | 大屏实时监控效果图 本人显示器太小，截图略显拥挤; 24 | 25 | QQ交流群 : 862251895 26 | 27 | ![image](https://cdn.jsdelivr.net/gh/jyolo/wLogger/webServer/static/images/webserver_monitor.png) 28 | 29 | > 功能说明 30 | 31 | 采集器 inputer 32 | 33 | * 实时日志采集,同时支持多个web日志同时采集 34 | * 可指定按照日志文件大小或指定时间,自动切割文件到指定目录, （日志切割不丢数据.） 35 | * 可自定义队列服务软件,接受采集的日志信息. 已内置redis 如需kafka 等其它mq队列可自行拓展 36 | * 极低的cpu内存占用 ,低配小主机也能愉快的玩耍 37 | 38 | 解析存储器 outputer 39 | 40 | * 实时解析日志并存储到指定的数据库, 已内置 mysql 和 mongodb 如需使用elastic全家桶或其它存储引擎 可自行拓展 41 | * 采集器,解析器,web服务均可 独立分布到不同的服务器节点上运行 42 | * 目前已内置 nginx，apache 解析器, 可随意指定日志格式, 只需在配置文件里面指定格式名称即可正确解析并存储 43 | * 支持按日期天,周,月,年. 自动分表或集合存储日志 44 | * 支持指定工作进程来快速消费队列数据,大流量也能实时解析并存储日志, 虚拟机中ab 实测2W并发延迟小于1秒 45 | * 注: 当海量流量来的时候发现解析存储延迟过高的情况,可将解析器部署到集群中其它多个节点同时消费队列数据,提升解析存储效率 46 | 47 | 48 | web服务 web 49 | 50 | * 已内置大屏监控web面板,流量情况一目了然 51 | * 同时支持 mysql 或者 mongodb 作为 数据源 52 | 53 | 54 | 55 | # 快速开始 56 | > 安装拓展 57 | 58 | sudo pip3 install -r requirements.txt 59 | 60 | > 启动 采集器 61 | 62 | sudo python3 main.py -r inputer -c config.ini 63 | 64 | > 启动 解析存储器 65 | 66 | sudo python3 main.py -r outputer -c config.ini 67 | 68 | > 启动 web服务 69 | 70 | sudo python3 main.py -r web -c config.ini 71 | 72 | > 查看命令行帮助 73 | 74 | python3 main.py --help 75 | 76 | * 以上三个应用均可单独部署和启用 77 | 78 | -r --run ; start ['inputer', 'outputer','web'] 79 | -s --stop ; stop ['inputer', 'outputer'] 80 | -c --config ; bind config.ini file 81 | 82 | > docker 支持 83 | 84 | docker pull jyolo/wlogger:v1.3 或者 docker build -t yourTagName . 85 | 86 | example: 87 | # 启动 web 服务 88 | docker run jyolo/wlogger:v1.3 -r web -c config.ini # 需要把配置文件复制或者挂载进 容器中/wLogger 目录内 89 | 90 | # 启动 解析存储器 服务 91 | docker run jyolo/wlogger:v1.3 -r outputer -c config.ini # 需要把配置文件复制或者挂载进 容器中/wLogger 目录内 92 | 93 | * 由于采集器 inputer 中切割日志操作,需要操作容器外部 nginx/apache 相关服务器,因此无法在docker中隔离环境下运行 . 94 | * 如果容器中有部署nginx 或者 apache 则可以 95 | 96 | # 配置详解 97 | 98 | > 公共配置 99 | 100 | # 当 inputer 和 outputer 中指定了 server_type = nginx 才需此配置 101 | [nginx] 102 | pid_path = /www/server/nginx/logs/nginx.pid # 指定 nginx.pid 的绝对路径 103 | server_conf = /www/server/nginx/conf/nginx.conf # 指定 nginx 配置文件的绝对路径 104 | 105 | # 当 inputer 和 outputer 中指定了 server_type = apache 才需此配置 106 | [apache] 107 | apachectl_bin = /www/server/apache/bin/apachectl # 指定 apachectl 命令的绝对路径 108 | server_conf = /www/server/apache/conf/httpd.conf # 指定 apache 配置文件的绝对路径 109 | 110 | # 当 inputer 和 outputer 中指定了 queue = redis 才需此配置 111 | [redis] 112 | host = 127.0.0.1 113 | port = 6379 114 | password = xxxxxxxx 115 | db = 1 116 | 117 | # 当 outputer 中 save_engine = mysql 或 web 中 data_engine = mysql 才需此配置 118 | [mysql] 119 | host = 127.0.0.1 120 | port = 3306 121 | username = nginx_logger 122 | password = xxxxxxxx 123 | db = nginx_logger 124 | table = logger 125 | split_save = day # 当有该配置项则代表开启自动分表 目前支持按 天，周，月，年 ；参数：[day, week, month ,year] ,进行存储 126 | 127 | # 当 outputer 中save_engine = mongodb 或 web 中 data_engine = mongodb 需此配置 128 | [mongodb] 129 | host = 127.0.0.1 130 | port = 27017 131 | username = logger_watcher 132 | password = xxxxxxxx 133 | db = nginx_logger 134 | collection = logger 135 | split_save = day # 当有该配置项则代表开启自动分集合 目前支持按 天，周，月，年 ；参数：[day, week, month ,year] ,进行存储 136 | 137 | 138 | > 日志采集端 配置 139 | 140 | [inputer] 141 | log_debug = True # 开启日志debug模式 会在项目中生成日志文件。 类似 ： inputer_config.ini.log 名称的日志文件 142 | node_id = server_80 # 当前节点ID 唯一 143 | queue = redis # 队列配置 目前内置了 [redis , mongodb] 144 | queue_name = queue_logger # 队列 key 的名称 145 | max_batch_push_queue_size = 5000 # 每次最多批量插入队列多少条数据 146 | max_retry_open_file_time = 10 # 当文件读取失败之后重新打开日志文件，最多重试多少次 147 | max_retry_reconnect_time = 20 # 连接队列失败的时候，最多重试多少次 148 | 149 | [inputer.log_file.web1] # inputer.log_file.web1 中的 web1 代表应用名称 唯一 app_name 150 | server_type = nginx # 服务器应用 [nginx ,apache] 151 | file_path = /wwwlogs/ww.aaa.com.log # 日志绝对路径 152 | log_format_name = online # 配置文件中 日志名称 example : "access_log /www/wwwlogs/xxx.log online;" 中的 `online` 则代表启用的日志配置名称 153 | read_type = tail # 读取文件方式 支持 tail 从末尾最后一行开始 ; head 从头第一行开始 * 当文件较大的时候 建议使用 tail 154 | cut_file_type = filesize # 切割文件方式 支持 filesize 文件大小单位M ;time 指定当天时间 24:00 155 | cut_file_point = 200 # 切割文件条件节点 当 filesize 时 200 代表200M 切一次 ; 当 time 时 24:00 代表今天该时间 切一次 156 | cut_file_save_dir = /wwwlogs/cut_file/ # 日志切割后存储绝对路径 157 | 158 | 159 | [inputer.log_file.web2] # 支持同时采集多个应用日志 追加配置即可 160 | .......................... 161 | 162 | 163 | 164 | > 日志解析存储端 165 | 166 | [outputer] 167 | log_debug = True # 开启日志debug模式 会在项目中生成日志文件。 类似 ： outpuer_config.ini.log 名称的日志文件 168 | save_engine = mongodb # 解析后的日志存储引擎目前支持 [mysql,mongodb] 169 | queue = redis # 队列引擎 此处需要和 inputer 采集端保持一致 170 | queue_name = queue_logger # 队列中 key 或 collection 集合的名称 此处需要和 inputer 采集端保持一致 171 | server_type = nginx # 服务器的类型 172 | worker_process_num = 1 # 指定工作进程数量 根据自己网站流量情况而定，一般4个worker即可 173 | max_batch_insert_db_size = 1 # 最多每次批量写入存储引擎的数量，根据自己应用情况而定，一般5000即可 174 | max_retry_reconnect_time = 200 # 连接存储引擎失败后，最多重试连接次数 175 | 176 | > 大屏监控端 177 | 178 | [web] 179 | env = development # 运行环境 development | production 180 | debug = True # 是否开启 debug 181 | secret_key = xxxx # flask session key 182 | host = 127.0.0.1 # 指定ip 183 | port = 5000 # 指定端口 184 | server_name = 127.0.0.1:5000 # 绑定域名和端口 (不推荐 ,如果是要nginx反代进行访问的话 请不要配置此项.) 185 | data_engine = mysql # 指定读取日志存储数据库引擎 目前内置了 [ mysql , mongodb ] 186 | 187 | -------------------------------------------------------------------------------- /Src/ip2Region.py: -------------------------------------------------------------------------------- 1 | #-*- coding:utf-8 -*- 2 | """ 3 | " ip2region python seacher client module 4 | " 5 | " Author: koma 6 | " Date : 2015-11-06 7 | """ 8 | import struct, io, socket, sys 9 | 10 | class Ip2Region(object): 11 | __INDEX_BLOCK_LENGTH = 12 12 | __TOTAL_HEADER_LENGTH = 8192 13 | 14 | __f = None 15 | __headerSip = [] 16 | __headerPtr = [] 17 | __headerLen = 0 18 | __indexSPtr = 0 19 | __indexLPtr = 0 20 | __indexCount = 0 21 | __dbBinStr = '' 22 | 23 | def __init__(self, dbfile): 24 | self.initDatabase(dbfile) 25 | 26 | def memorySearch(self, ip): 27 | """ 28 | " memory search method 29 | " param: ip 30 | """ 31 | if not ip.isdigit(): ip = self.ip2long(ip) 32 | 33 | if self.__dbBinStr == '': 34 | self.__dbBinStr = self.__f.read() #read all the contents in file 35 | self.__indexSPtr = self.getLong(self.__dbBinStr, 0) 36 | self.__indexLPtr = self.getLong(self.__dbBinStr, 4) 37 | self.__indexCount = int((self.__indexLPtr - self.__indexSPtr)/self.__INDEX_BLOCK_LENGTH)+1 38 | 39 | l, h, dataPtr = (0, self.__indexCount, 0) 40 | while l <= h: 41 | m = int((l+h) >> 1) 42 | p = self.__indexSPtr + m*self.__INDEX_BLOCK_LENGTH 43 | sip = self.getLong(self.__dbBinStr, p) 44 | 45 | if ip < sip: 46 | h = m -1 47 | else: 48 | eip = self.getLong(self.__dbBinStr, p+4) 49 | if ip > eip: 50 | l = m + 1; 51 | else: 52 | dataPtr = self.getLong(self.__dbBinStr, p+8) 53 | break 54 | 55 | if dataPtr == 0: raise Exception("Data pointer not found") 56 | 57 | return self.returnData(dataPtr) 58 | 59 | def binarySearch(self, ip): 60 | """ 61 | " binary search method 62 | " param: ip 63 | """ 64 | if not ip.isdigit(): ip = self.ip2long(ip) 65 | 66 | if self.__indexCount == 0: 67 | self.__f.seek(0) 68 | superBlock = self.__f.read(8) 69 | self.__indexSPtr = self.getLong(superBlock, 0) 70 | self.__indexLPtr = self.getLong(superBlock, 4) 71 | self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) / self.__INDEX_BLOCK_LENGTH) + 1 72 | 73 | l, h, dataPtr = (0, self.__indexCount, 0) 74 | while l <= h: 75 | m = int((l+h) >> 1) 76 | p = m*self.__INDEX_BLOCK_LENGTH 77 | 78 | self.__f.seek(self.__indexSPtr+p) 79 | buffer = self.__f.read(self.__INDEX_BLOCK_LENGTH) 80 | sip = self.getLong(buffer, 0) 81 | if ip < sip: 82 | h = m - 1 83 | else: 84 | eip = self.getLong(buffer, 4) 85 | if ip > eip: 86 | l = m + 1 87 | else: 88 | dataPtr = self.getLong(buffer, 8) 89 | break 90 | 91 | if dataPtr == 0: raise Exception("Data pointer not found") 92 | 93 | return self.returnData(dataPtr) 94 | 95 | def btreeSearch(self, ip): 96 | """ 97 | " b-tree search method 98 | " param: ip 99 | """ 100 | if not ip.isdigit(): ip = self.ip2long(ip) 101 | 102 | if len(self.__headerSip) < 1: 103 | headerLen = 0 104 | #pass the super block 105 | self.__f.seek(8) 106 | #read the header block 107 | b = self.__f.read(self.__TOTAL_HEADER_LENGTH) 108 | #parse the header block 109 | for i in range(0, len(b), 8): 110 | sip = self.getLong(b, i) 111 | ptr = self.getLong(b, i+4) 112 | if ptr == 0: 113 | break 114 | self.__headerSip.append(sip) 115 | self.__headerPtr.append(ptr) 116 | headerLen += 1 117 | self.__headerLen = headerLen 118 | 119 | l, h, sptr, eptr = (0, self.__headerLen, 0, 0) 120 | while l <= h: 121 | m = int((l+h) >> 1) 122 | 123 | if ip == self.__headerSip[m]: 124 | if m > 0: 125 | sptr = self.__headerPtr[m-1] 126 | eptr = self.__headerPtr[m] 127 | else: 128 | sptr = self.__headerPtr[m] 129 | eptr = self.__headerPtr[m+1] 130 | break 131 | 132 | if ip < self.__headerSip[m]: 133 | if m == 0: 134 | sptr = self.__headerPtr[m] 135 | eptr = self.__headerPtr[m+1] 136 | break 137 | elif ip > self.__headerSip[m-1]: 138 | sptr = self.__headerPtr[m-1] 139 | eptr = self.__headerPtr[m] 140 | break 141 | h = m - 1 142 | else: 143 | if m == self.__headerLen - 1: 144 | sptr = self.__headerPtr[m-1] 145 | eptr = self.__headerPtr[m] 146 | break 147 | elif ip <= self.__headerSip[m+1]: 148 | sptr = self.__headerPtr[m] 149 | eptr = self.__headerPtr[m+1] 150 | break 151 | l = m + 1 152 | 153 | if sptr == 0: raise Exception("Index pointer not found") 154 | 155 | indexLen = eptr - sptr 156 | self.__f.seek(sptr) 157 | index = self.__f.read(indexLen + self.__INDEX_BLOCK_LENGTH) 158 | 159 | l, h, dataPrt = (0, int(indexLen/self.__INDEX_BLOCK_LENGTH), 0) 160 | while l <= h: 161 | m = int((l+h) >> 1) 162 | offset = int(m * self.__INDEX_BLOCK_LENGTH) 163 | sip = self.getLong(index, offset) 164 | 165 | if ip < sip: 166 | h = m - 1 167 | else: 168 | eip = self.getLong(index, offset+4) 169 | if ip > eip: 170 | l = m + 1; 171 | else: 172 | dataPrt = self.getLong(index, offset+8) 173 | break 174 | 175 | if dataPrt == 0: raise Exception("Data pointer not found") 176 | 177 | return self.returnData(dataPrt) 178 | 179 | def initDatabase(self, dbfile): 180 | """ 181 | " initialize the database for search 182 | " param: dbFile 183 | """ 184 | try: 185 | self.__f = io.open(dbfile, "rb") 186 | except IOError as e: 187 | print("[Error]: %s" % e) 188 | sys.exit() 189 | 190 | def returnData(self, dataPtr): 191 | """ 192 | " get ip data from db file by data start ptr 193 | " param: dsptr 194 | """ 195 | dataLen = (dataPtr >> 24) & 0xFF 196 | dataPtr = dataPtr & 0x00FFFFFF 197 | 198 | self.__f.seek(dataPtr) 199 | data = self.__f.read(dataLen) 200 | 201 | return { 202 | "city_id": self.getLong(data, 0), 203 | "region" : data[4:] 204 | } 205 | 206 | def ip2long(self, ip): 207 | _ip = socket.inet_aton(ip) 208 | return struct.unpack("!L", _ip)[0] 209 | 210 | def isip(self, ip): 211 | p = ip.split(".") 212 | 213 | if len(p) != 4 : return False 214 | for pp in p: 215 | if not pp.isdigit() : return False 216 | if len(pp) > 3 : return False 217 | if int(pp) > 255 : return False 218 | 219 | return True 220 | 221 | def getLong(self, b, offset): 222 | if len(b[offset:offset+4]) == 4: 223 | return struct.unpack('I', b[offset:offset+4])[0] 224 | return 0 225 | 226 | def close(self): 227 | if self.__f != None: 228 | self.__f.close() 229 | 230 | self.__dbBinStr = None 231 | self.__headerPtr = None 232 | self.__headerSip = None 233 | -------------------------------------------------------------------------------- /StorageAdapter/Mongodb.py: -------------------------------------------------------------------------------- 1 | from StorageAdapter.BaseAdapter import Adapter 2 | from pymongo import MongoClient,errors as pyerrors 3 | from threading import Thread 4 | import time,threading,os,traceback 5 | 6 | 7 | try: 8 | # Python 3.x 9 | from urllib.parse import quote_plus 10 | except ImportError: 11 | # Python 2.x 12 | from urllib import quote_plus 13 | 14 | class StorageAp(Adapter): 15 | 16 | db = None 17 | runner = None 18 | 19 | 20 | @classmethod 21 | def initStorage(cls,runnerObject): 22 | self = cls() 23 | self.runner = runnerObject 24 | self.conf = self.runner.conf 25 | self.logging = self.runner.logging 26 | 27 | if self.conf['mongodb']['username'] and self.conf['mongodb']['password']: 28 | mongo_url = 'mongodb://%s:%s@%s:%s/?authSource=%s' % \ 29 | ( 30 | quote_plus(self.conf['mongodb']['username']), 31 | quote_plus(self.conf['mongodb']['password']), 32 | self.conf['mongodb']['host'], 33 | int(self.conf['mongodb']['port']), 34 | self.conf['mongodb']['db'] 35 | ) 36 | 37 | else: 38 | mongo_url = 'mongodb://%s:%s/?authSource=%s' % \ 39 | ( 40 | self.conf['mongodb']['host'], 41 | int(self.conf['mongodb']['port']), 42 | self.conf['mongodb']['db'] 43 | ) 44 | 45 | mongodb = MongoClient(mongo_url) 46 | self.db = mongodb[self.conf['mongodb']['db']] 47 | 48 | return self 49 | 50 | 51 | def _parseData(self): 52 | 53 | pass 54 | 55 | 56 | def _intoDb(self): 57 | 58 | if 'max_retry_reconnect_time' in self.conf['outputer']: 59 | max_retry_reconnect_time = int(self.conf['outputer']['max_retry_reconnect_time']) 60 | else: 61 | max_retry_reconnect_time = 3 62 | 63 | retry_reconnect_time = 0 64 | 65 | while True: 66 | time.sleep(1) 67 | # self.logging.debug('\n outputerer -------pid: %s tid: %s parseQeueuData _intoDb len: %s' % (os.getpid(), threading.get_ident(), len(self.runner.dqueue) )) 68 | 69 | mongodb_outputer_client = self.db[self.runner.save_engine_conf['collection']] 70 | try: 71 | start_time = time.perf_counter() 72 | 73 | data = [] 74 | for i in range(len(self.runner.dqueue)): 75 | data.append(self.runner.dqueue.pop()) 76 | 77 | if len(data) == 0 : 78 | continue 79 | backup_for_push_back_queue = data 80 | # before_into_storage 81 | self._handle_queue_data_before_into_storage() 82 | 83 | res = mongodb_outputer_client.insert_many(data, ordered=False) 84 | 85 | # after_into_storage 86 | self._handle_queue_data_after_into_storage() 87 | 88 | retry_reconnect_time = 0 89 | 90 | end_time = time.perf_counter() 91 | self.logging.debug("\n outputerer -------pid: %s -- insert into mongodb: %s---- end 耗时: %s \n" % ( 92 | os.getpid(), len(res.inserted_ids), round(end_time - start_time, 3))) 93 | 94 | # 消费完 dqueue 里面的数据后 取出标识 标识已完成 95 | if hasattr(self.runner,'share_worker_list') and self.runner.share_worker_list != None: 96 | self.runner.share_worker_list.pop() 97 | 98 | 99 | except pyerrors.PyMongoError as e: 100 | 101 | time.sleep(1) 102 | retry_reconnect_time = retry_reconnect_time + 1 103 | if retry_reconnect_time >= max_retry_reconnect_time: 104 | self.runner.push_back_to_queue(backup_for_push_back_queue) 105 | self.logging.error('重试重新链接 mongodb 超出最大次数 %s' % max_retry_reconnect_time) 106 | raise pyerrors.PyMongoError('重试重新链接 mongodb 超出最大次数 %s' % max_retry_reconnect_time) 107 | else: 108 | self.logging.warn("\n outputerer -------pid: %s -- retry_reconnect_mongodb at: %s time---- \n" % ( 109 | os.getpid(), retry_reconnect_time)) 110 | continue 111 | 112 | 113 | 114 | def pushDataToStorage(self): 115 | retry_reconnect_time = 0 116 | 117 | while True: 118 | time.sleep(1) 119 | self._getTableName('collection') 120 | 121 | 122 | if retry_reconnect_time == 0: 123 | 124 | # 获取队列数据 125 | queue_data = self.runner.getQueueData() 126 | if len(queue_data) == 0: 127 | self.logging.debug('pid: %s 暂无数据 等待 queue数据 ' % os.getpid()) 128 | time.sleep(1) 129 | continue 130 | 131 | start_time = time.perf_counter() 132 | 133 | # 　错误退回队列 (未解析的原始的数据) 134 | backup_for_push_back_queue = [] 135 | _data = [] 136 | for item in queue_data: 137 | if isinstance(item, bytes): 138 | item = item.decode(encoding='utf-8') 139 | 140 | backup_for_push_back_queue.append(item) 141 | item = self.runner._parse_line_data(item) 142 | if item: 143 | _data.append(item) 144 | 145 | 146 | 147 | end_time = time.perf_counter() 148 | 149 | take_time = round(end_time - start_time, 3) 150 | self.logging.debug( 151 | '\n outputerer ---pid: %s tid: %s reg data len:%s; take time : %s \n' % 152 | (os.getpid(), threading.get_ident(), len(_data), take_time)) 153 | 154 | 155 | if 'max_retry_reconnect_time' in self.conf['outputer']: 156 | max_retry_reconnect_time = int(self.conf['outputer']['max_retry_reconnect_time']) 157 | else: 158 | max_retry_reconnect_time = 3 159 | 160 | mongodb_outputer_client = self.db[self.table] 161 | try: 162 | start_time = time.perf_counter() 163 | 164 | if len(_data) == 0: 165 | continue 166 | 167 | # before_into_storage 168 | self._handle_queue_data_before_into_storage() 169 | 170 | res = mongodb_outputer_client.insert_many(_data, ordered=False) 171 | 172 | # after_into_storage 173 | self._handle_queue_data_after_into_storage() 174 | 175 | # 重置 retry_reconnect_time 176 | retry_reconnect_time = 0 177 | 178 | end_time = time.perf_counter() 179 | self.logging.debug("\n outputerer -------pid: %s -- insert into mongodb: %s---- end 耗时: %s \n" % ( 180 | os.getpid(), len(res.inserted_ids), round(end_time - start_time, 3))) 181 | 182 | except pyerrors.PyMongoError as e: 183 | time.sleep(1) 184 | retry_reconnect_time = retry_reconnect_time + 1 185 | if retry_reconnect_time >= max_retry_reconnect_time: 186 | self.runner.rollBackQueue(backup_for_push_back_queue) 187 | self.logging.error('重试重新链接 mongodb 超出最大次数 %s' % max_retry_reconnect_time) 188 | raise Exception('重试重新链接 mongodb 超出最大次数 %s' % max_retry_reconnect_time) 189 | else: 190 | self.logging.warn("\n outputerer -------pid: %s -- retry_reconnect_mongodb at: %s time---- \n" % ( 191 | os.getpid(), retry_reconnect_time)) 192 | continue 193 | 194 | 195 | # 在持久化存储之前 对 队列中的数据 进行预处理 ,比如 update ,delete 等操作 196 | def _handle_queue_data_before_into_storage(self): 197 | pass 198 | 199 | # 在持久化存储之前 对 队列中的数据 进行预处理 ,比如 update ,delete 等操作 200 | def _handle_queue_data_after_into_storage(self): 201 | # if (hasattr(self.runner, 'queue_data_ids')): 202 | # ids = self.runner.queue_data_ids 203 | # self.db[self.runner.queue_key].update_many( 204 | # {'_id': {'$in': ids}}, 205 | # { 206 | # '$set': {'out_queue': 1}, 207 | # '$currentDate': {'ttl': True} 208 | # }, 209 | # 210 | # ) 211 | pass 212 | 213 | 214 | 215 | 216 | 217 | -------------------------------------------------------------------------------- /ParserAdapter/Apache.py: -------------------------------------------------------------------------------- 1 | from ParserAdapter.BaseAdapter import Adapter,ParseError,ReCompile 2 | import re,os,shutil,json 3 | 4 | 5 | class Handler(Adapter): 6 | 7 | def __init__(self,*args ,**kwargs): 8 | super(Handler,self).__init__(*args,**kwargs) 9 | 10 | 11 | def getLogFormat(self): 12 | # "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %I %O" 13 | return { 14 | ### 非 nginx 日志变量 附加自定义字段 15 | '$node_id': { 16 | 'mysql_field_type': 'varchar(255)', 17 | 'mysql_key_field': True 18 | }, 19 | '$app_name': { 20 | 'mysql_field_type': 'varchar(255)', 21 | }, 22 | ### 非 nginx 日志变量 附加自定义字段 23 | 24 | '%h':{ 25 | 'nickname': 'ip', 26 | 'mysql_field_type': 'varchar(15)', 27 | 'mysql_key_field': [ 28 | '%t.timestamp', 29 | ['%>s', '%r.request_url', '%r.request_method'] 30 | ], 31 | 'extend_field': { 32 | 'isp': { 33 | 'mysql_field_type': 'varchar(30)', 34 | }, 35 | 'city': { 36 | 'mysql_field_type': 'varchar(30)', 37 | 'mysql_key_field': ['%>s'], 38 | }, 39 | 'city_id': { 40 | 'mysql_field_type': 'int(10)', 41 | }, 42 | 'province': { 43 | 'mysql_field_type': 'varchar(30)', 44 | 'mysql_key_field': [ 45 | '%t.timestamp', 46 | '%t.timestamp', 47 | ], 48 | }, 49 | 'country': { 50 | 'mysql_field_type': 'varchar(30)', 51 | } 52 | } 53 | }, 54 | '%l':{ 55 | 'nickname': 'remote_logname', 56 | }, 57 | '%u':{ 58 | 'nickname': 'remote_user' 59 | }, 60 | '%t': { 61 | 'nickname': 'time', 62 | 'extend_field': { 63 | 'time_str': { 64 | 'mysql_field_type': 'datetime', 65 | 'mysql_key_field': True 66 | }, 67 | 'timestamp': { 68 | 'mysql_field_type': 'int(10)', 69 | 'mysql_key_field': True 70 | }, 71 | } 72 | }, 73 | '%r': { 74 | 'nickname': 'request', 75 | 'extend_field': { 76 | 'request_method': { 77 | 'mysql_field_type': 'varchar(10)', 78 | 'mysql_key_field': True, 79 | }, 80 | 'request_url': { 81 | 'mysql_field_type': 'varchar(255)', 82 | 'mysql_key_field': ['%t.timestamp', '%t.timestamp'], 83 | }, 84 | 'args': { 85 | 'mysql_field_type': 'text', 86 | }, 87 | 'server_protocol': { 88 | 'mysql_field_type': 'varchar(10)', 89 | }, 90 | } 91 | }, 92 | '%>s': { 93 | 'nickname': 'status_code', 94 | }, 95 | '%b': { 96 | 'nickname': 'bytes_sent', 97 | }, 98 | '%D':{ 99 | 'nickname': 'request_time', 100 | }, 101 | '%m':{ 102 | 'nickname': 'request_method', 103 | }, 104 | '%{Referer}i': { 105 | 'nickname': 'referer', 106 | }, 107 | '%{User-Agent}i': { 108 | 'nickname': 'ua', 109 | }, 110 | '%I': { 111 | 'nickname': 'bytes_received', 112 | 're': '\d+', 113 | }, 114 | '%O': { 115 | 'nickname': 'bytes_sent_with_header', 116 | 're': '\d+', 117 | }, 118 | } 119 | 120 | 121 | """ 122 | 日志解析 123 | """ 124 | def parse(self,log_format_name='',log_line=''): 125 | 126 | 127 | log_format_list = self.log_line_pattern_dict[log_format_name]['log_format_list'] 128 | log_format_recompile = self.log_line_pattern_dict[log_format_name]['log_format_recompile'] 129 | 130 | res = log_format_recompile.match(log_line) 131 | 132 | 133 | if res == None: 134 | raise ParseError('解析日志失败,请检查client 配置中 日志的 格式名称是否一致 log_format_name') 135 | 136 | matched = list(res.groups()) 137 | 138 | if len(matched) == len(log_format_list): 139 | data = {} 140 | del_key_name = [] 141 | 142 | for i in range(len(list(log_format_list))): 143 | key_name = log_format_list[i] 144 | 145 | # request 参数不支持别名 146 | if 'nickname' in self.getLogFormat()[ log_format_list[i]] : 147 | key_name = self.getLogFormat()[ log_format_list[i]]['nickname'] 148 | 149 | # 解析 %h 成对应的 地理位置信息 isp,city,city_id,province,country,ip, 150 | if key_name == '%h': 151 | ip_data = self.parse_ip_to_area(matched[i]) 152 | data.update(ip_data) 153 | 154 | # 解析 %r 成 request_method ,request_url ,args ,server_protocol , 155 | if key_name == '%r': 156 | request_extend_data = self.parse_request_to_extend(matched[i]) 157 | data.update(request_extend_data) 158 | del_key_name.append(key_name) 159 | 160 | # 解析 %t 成 timestr , timestamp 161 | if key_name == '%t': 162 | time_data = self.parse_time_to_str('time_local',matched[i]) 163 | data.update(time_data) 164 | del_key_name.append(key_name) 165 | 166 | data[key_name] = matched[i] 167 | 168 | 169 | 170 | 171 | # 剔除掉 解析出拓展字符串的 字段 172 | for i in del_key_name: 173 | del data[i] 174 | 175 | 176 | return data 177 | 178 | 179 | 180 | # 从配置获取所有的日志配置项 181 | def getLoggerFormatByServerConf(self,server_conf_path): 182 | 183 | # 根据配置文件 自动获取 log_format 字符串 184 | with open(server_conf_path, 'rb') as fd: 185 | content = fd.read().decode(encoding="utf-8") 186 | 187 | format_list = {} 188 | 189 | log_list = re.findall(r'LogFormat\s?\"([\S\s]*?)\"\s?(\w+)\n' ,content) 190 | 191 | 192 | if len(log_list) == 0: 193 | return format_list 194 | 195 | # 从日志格式字符串中提取日志变量 196 | for i in log_list: 197 | res = re.findall(r'(\%[\>|\{]?[a-zA-Z|\-|\_]+[\}|\^]?\w?)', i[0].strip()) 198 | if len(res): 199 | format_list[i[1]] = { 200 | 'log_format_str':i[0].strip(), 201 | 'log_format_vars':self.LOG_FORMAT_SPLIT_TAG.join(res) 202 | } 203 | 204 | 205 | del content 206 | 207 | 208 | return format_list 209 | 210 | 211 | 212 | # 重启日志进程 213 | def rotatelog(self,server_conf,log_path ,target_file ): 214 | 215 | 216 | try: 217 | 218 | if not os.path.exists(server_conf['apachectl_bin']): 219 | raise FileNotFoundError('apachectl_bin : %s 命令不存在' % server_conf['apachectl_bin']) 220 | 221 | if not os.path.exists(log_path): 222 | raise FileNotFoundError('日志: %s 不存在' % log_path) 223 | 224 | 225 | 226 | # 这里需要注意 日志目录的 权限 是否有www 否则会导致 ngixn 重开日志问件 无法写入的问题 227 | cmd = '%s graceful' % server_conf['apachectl_bin'] 228 | shutil.move(log_path, target_file) 229 | 230 | res = os.popen(cmd) 231 | if len(res.readlines()) > 0: 232 | cmd_res = '' 233 | for i in res.readlines(): 234 | cmd_res += i + '\n' 235 | raise Exception ('reload 服务器进程失败: %s' % cmd_res) 236 | 237 | return True 238 | 239 | except Exception as e: 240 | return '切割日志失败 : %s ; error class : %s error info : %s' % (target_file ,e.__class__, e.args) 241 | 242 | """ 243 | 找到匹配中的日志变量 244 | """ 245 | def __replaceLogVars(self,matched): 246 | 247 | 248 | s = matched.group() 249 | 250 | if s not in self.getLogFormat(): 251 | 252 | raise ValueError('handle 里面不存在日志变量:%s' % s) 253 | 254 | if 're' in self.getLogFormat()[s]: 255 | re_str = self.getLogFormat()[s]['re'] 256 | else: 257 | re_str = '[\s|\S]*?' 258 | 259 | return '(%s)' % re_str 260 | 261 | """ 262 | 根据录入的格式化字符串 返回 parse 所需 log_format 配置 263 | """ 264 | def getLogFormatByConfStr(self ,log_format_str,log_format_vars ,log_format_name ,log_type): 265 | 266 | 267 | # 日志格式不存在 则 预编译 268 | if log_format_name not in self.log_line_pattern_dict: 269 | 270 | # 过滤下 正则中 特殊字符 271 | log_format_str = log_format_str.strip() \ 272 | .replace('[', '\[').replace(']', '\]') \ 273 | .replace('(', '\(').replace(')', '\)') 274 | 275 | 276 | if (log_type == 'string'): 277 | 278 | 279 | # 获取日志变量 280 | log_format_list = log_format_vars.split(self.LOG_FORMAT_SPLIT_TAG) 281 | 282 | # 按照日志格式顺序 替换日志变量成正则 进行预编译 283 | format = re.sub(r'(\%[\>|\{]?[a-zA-Z|\-|\_]+[\}|\^]?\w?)', self.__replaceLogVars, log_format_str).strip() 284 | try: 285 | re_compile = re.compile(format, re.I) 286 | except re.error: 287 | raise Exception('预编译错误,请检查日志字符串中是否包含特殊字符串; 日志:%s' % log_format_str) 288 | 289 | 290 | self.log_line_pattern_dict[log_format_name] = { 291 | 'log_format_list': log_format_list, 292 | 'log_format_recompile':re_compile 293 | } 294 | 295 | 296 | 297 | 298 | -------------------------------------------------------------------------------- /webServer/divers/mongo.py: -------------------------------------------------------------------------------- 1 | # coding=UTF-8 2 | from flask import current_app,request,session 3 | from webServer.customer import Func,ApiCorsResponse 4 | import time,re 5 | 6 | 7 | 8 | 9 | class MongoDb(): 10 | 11 | today = time.strftime('%Y-%m-%d', time.localtime(time.time())) 12 | 13 | @classmethod 14 | def get_total_ip(cls): 15 | res = current_app.db[current_app.db_engine_table].aggregate([ 16 | {'$match': {'time_str': {'$regex': '^%s' % cls.today}}}, 17 | {'$group': {'_id': '$ip'}}, 18 | {'$group': {'_id': '','total_num':{'$sum':1} }}, 19 | {'$project': { '_id': 0}}, 20 | {'$sort': {'total_num': -1}}, 21 | ]) 22 | res = list(res) 23 | if len(res) == 0 : 24 | return ApiCorsResponse.response({}) 25 | return ApiCorsResponse.response(res[0]) 26 | 27 | @classmethod 28 | def get_total_pv(cls): 29 | res = current_app.db[current_app.db_engine_table].find({'time_str': {'$regex': '^%s' % cls.today}}).count() 30 | return ApiCorsResponse.response({'total_num':res}) 31 | 32 | @classmethod 33 | def get_request_num_by_url(cls): 34 | # if request.args.get('type') == 'init': 35 | # # 　一分钟 * 10 10分钟 36 | # limit = 60 * 10 37 | # else: 38 | # limit = 5 39 | 40 | 41 | collection = current_app.db_engine_table 42 | 43 | 44 | # total = current_app.db[collection].find({'time_str': {'$regex': '^%s' % cls.today}}).count() 45 | 46 | res = current_app.db[collection].aggregate([ 47 | {'$match': {'time_str': {'$regex': '^%s' % cls.today}}}, 48 | {'$group': {'_id': '$request_url', 'total_num': {'$sum': 1}}}, 49 | {'$project': { 50 | 'request_url': '$_id', 51 | 'total_num': 1, 52 | '_id': 0, 53 | # 'percent': {'$toDouble': {'$substr': [{'$multiply': [{'$divide': ['$total_num', total]}, 100]}, 0, 4]}} 54 | } 55 | }, 56 | {'$sort': {'total_num': -1}}, 57 | {'$limit': 50} 58 | ]) 59 | 60 | data = list(res) 61 | data.reverse() 62 | 63 | return ApiCorsResponse.response(data) 64 | 65 | @classmethod 66 | def get_request_num_by_ip(cls): 67 | if request.args.get('type') == 'init': 68 | # 　一分钟 * 10 10分钟 69 | limit = 60 * 10 70 | else: 71 | limit = 5 72 | 73 | session['now_timestamp'] = int(time.time()) 74 | 75 | 76 | res = current_app.db[current_app.db_engine_table].aggregate([ 77 | {'$match': {'time_str': {'$regex': '^%s' % cls.today}}}, 78 | {'$group': {'_id': '$ip', 'total_num': {'$sum': 1}}}, 79 | {'$project': { 80 | 'ip':'$_id', 81 | 'total_num': 1, 82 | '_id':0 83 | # 'percent':{ '$toDouble': {'$substr':[ {'$multiply':[ {'$divide':['$total_num' , total]} ,100] } ,0,4 ] } } 84 | } 85 | }, 86 | {'$sort': {'total_num': -1}}, 87 | {'$limit': 50} 88 | ]) 89 | 90 | 91 | 92 | data = list(res) 93 | data.reverse() 94 | return ApiCorsResponse.response(data) 95 | 96 | @classmethod 97 | def get_request_urls_by_ip(cls): 98 | if not request.args.get('ip'): 99 | return ApiCorsResponse.response('缺少ip参数', False) 100 | 101 | session['now_timestamp'] = int(time.time()) 102 | 103 | collection = current_app.db_engine_table 104 | 105 | 106 | 107 | res = current_app.db[collection].aggregate([ 108 | {'$match': {'time_str': {'$regex': '^%s' % cls.today}, 'ip': request.args.get('ip')}}, 109 | {'$group': {'_id': '$request_url', 'total_num': {'$sum': 1}}}, 110 | {'$project': { 111 | 'total_num': 1, 112 | 'request_url': '$_id', 113 | } 114 | }, 115 | {'$sort': {'total_num': -1}}, 116 | {'$limit': 20} 117 | ]) 118 | 119 | 120 | data = list(res) 121 | data.reverse() 122 | return ApiCorsResponse.response(data) 123 | 124 | @classmethod 125 | def get_request_num_by_status(cls): 126 | session['now_timestamp'] = int(time.time()) 127 | 128 | 129 | 130 | res = current_app.db[current_app.db_engine_table].aggregate([ 131 | {'$match': {'time_str': {'$regex': '^%s' % cls.today}, 'status_code': {'$ne': '200'}}}, 132 | {'$group': {'_id': '$status_code', 'total_num': {'$sum': 1}}}, 133 | {'$project': {'status_code': '$_id', 'total_num': 1, '_id': 0}}, 134 | {'$sort': {'total_num': -1}}, 135 | ]) 136 | 137 | data = list(res) 138 | data.reverse() 139 | 140 | return ApiCorsResponse.response(data) 141 | 142 | @classmethod 143 | def get_request_num_by_status_code(cls): 144 | if not request.args.get('code'): 145 | return ApiCorsResponse.response('缺少code参数', False) 146 | 147 | session['now_timestamp'] = int(time.time()) 148 | 149 | 150 | 151 | arg = re.findall('\d+?', request.args.get('code')) 152 | res = current_app.db[current_app.db_engine_table].aggregate([ 153 | {'$match': {'time_str': {'$regex': '^%s' % cls.today}, 'status': ''.join(arg)}}, 154 | {'$group': {'_id': '$request_url', 'total_num': {'$sum': 1}}}, 155 | {'$project': {'request_url': '$_id', 'total_num': 1, '_id': 0}}, 156 | {'$sort': {'total_num': -1}}, 157 | ]) 158 | 159 | data = list(res) 160 | data.reverse() 161 | 162 | return ApiCorsResponse.response(data) 163 | 164 | @classmethod 165 | def get_request_num_by_secends(cls): 166 | if request.args.get('type') == 'init': 167 | # 　一分钟 * 10 10分钟 168 | limit = 60 * 10 169 | else: 170 | limit = 5 171 | 172 | 173 | res = current_app.db[current_app.db_engine_table].aggregate([ 174 | {'$match': {'time_str': {'$regex': '^%s' % cls.today}}}, 175 | {'$group': {'_id': '$timestamp', 'total_num': {'$sum': 1}}}, 176 | {'$project': {'timestamp': '$_id', 'total_request_num': '$total_num', '_id': 0}}, 177 | {'$sort': {'timestamp': -1}}, 178 | {'$limit': limit} 179 | ]) 180 | 181 | data = [] 182 | for i in res: 183 | item = {} 184 | # item['timestamp'] = time.strftime('%H:%M:%S', time.localtime(i['timestamp'])) 185 | # * 1000 for js timestamp 186 | item['timestamp'] = i['timestamp'] * 1000 187 | item['total_request_num'] = i['total_request_num'] 188 | data.append(item) 189 | 190 | data.reverse() 191 | return ApiCorsResponse.response(data) 192 | 193 | @classmethod 194 | def get_network_traffic_by_minute(cls): 195 | current_hour = time.strftime('%Y-%m-%d %H', time.localtime(time.time())) 196 | 197 | res = current_app.db[current_app.db_engine_table].aggregate([ 198 | {'$match': {'time_str': {'$regex': '^%s' % current_hour}}}, 199 | {'$project': { 200 | '_id': 0, 201 | 'time_minute': {'$dateToString': { 202 | 'format': '%Y-%m-%d %H:%M', 'date': {'$dateFromString': {'dateString': '$time_str'}} 203 | } 204 | }, 205 | 'request_length': {'$toInt': '$request_length'}, 206 | 'bytes_sent': {'$toInt': '$bytes_sent'} 207 | }, 208 | }, 209 | {'$group': { 210 | '_id': '$time_minute', 211 | 'in_network_sum': {'$sum': '$request_length'}, 212 | 'out_network_sum': {'$sum': '$bytes_sent'} 213 | } 214 | }, 215 | {'$project': {'_id': 0, 'time_str': '$_id', 'in_network': {'$divide': ['$in_network_sum', 1024]}, 216 | 'out_network': {'$divide': ['$out_network_sum', 1024]}}}, 217 | {'$sort': {'time_str': -1}}, 218 | 219 | ]) 220 | 221 | data = [] 222 | for i in res: 223 | i['time_str'] = int(time.mktime(time.strptime(i['time_str'], '%Y-%m-%d %H:%M'))) 224 | data.append(i) 225 | 226 | data.reverse() 227 | 228 | return ApiCorsResponse.response(data) 229 | 230 | @classmethod 231 | def get_ip_pv_num_by_minute(cls): 232 | 233 | current_hour = time.strftime('%Y-%m-%d %H', time.localtime(time.time())) 234 | 235 | res = current_app.db[current_app.db_engine_table].aggregate([ 236 | {'$match': {'time_str': {'$regex': '^%s' % current_hour}}}, 237 | {'$project': { 238 | '_id': 0, 239 | 'time_minute': { 240 | '$dateToString': { 241 | 'format': '%Y-%m-%d %H:%M', 'date': {'$dateFromString': {'dateString': '$time_str'}} 242 | } 243 | }, 244 | 'ip': 1 245 | }, 246 | }, 247 | {'$group': { 248 | '_id': '$time_minute', 249 | 'pv_num': {'$sum': 1}, 250 | 'ip_arr': {'$addToSet': '$ip'} 251 | } 252 | }, 253 | {'$project': {'_id': 0, 'time_str': '$_id', 'pv_num': 1, 'ip_num': {'$size': '$ip_arr'}}}, 254 | {'$sort': {'time_str': -1}}, 255 | ]) 256 | 257 | data = [] 258 | for i in res: 259 | i['time_str'] = int(time.mktime(time.strptime(i['time_str'], '%Y-%m-%d %H:%M'))) 260 | data.append(i) 261 | 262 | data.reverse() 263 | return ApiCorsResponse.response(data) 264 | 265 | @classmethod 266 | def get_request_num_by_province(cls): 267 | session['now_timestamp'] = int(time.time()) 268 | 269 | res = current_app.db[current_app.db_engine_table].aggregate([ 270 | {'$match': {'time_str': {'$regex': '^%s' % cls.today}}}, 271 | {'$group': {'_id': '$province', 'total_num': {'$sum': 1}}}, 272 | {'$project': {'province': '$_id', 'value': '$total_num', '_id': 0}}, 273 | {'$sort': {'total_num': -1}}, 274 | ]) 275 | 276 | data = list(res) 277 | data.reverse() 278 | 279 | return ApiCorsResponse.response(data) 280 | 281 | @classmethod 282 | def get_spider_by_ua(cls): 283 | res = current_app.db[current_app.db_engine_table].aggregate([ 284 | {'$match': { 285 | 'time_str': {'$regex': '^%s' % cls.today} , 286 | 'ua':{'$regex':'spider'} 287 | } 288 | }, 289 | {'$group': {'_id': '$ua', 'total_num': {'$sum': 1}}}, 290 | {'$project': {'ua': '$_id', 'total_num': 1, '_id': 0}}, 291 | {'$sort': {'total_num': -1}}, 292 | ]) 293 | 294 | data = list(res) 295 | data.reverse() 296 | return ApiCorsResponse.response(data) 297 | -------------------------------------------------------------------------------- /webServer/divers/mysql.py: -------------------------------------------------------------------------------- 1 | # coding=UTF-8 2 | from flask import Response,current_app,request,session 3 | from sqlalchemy import text 4 | from webServer.customer import Func,ApiCorsResponse 5 | import json,time,datetime 6 | 7 | 8 | 9 | # 自定义mysql 数据获取class 10 | class MysqlDb(): 11 | 12 | today = time.strftime('%Y-%m-%d', time.localtime(time.time())) 13 | 14 | @classmethod 15 | def get_total_ip(cls): 16 | with current_app.db.connect() as cursor: 17 | sql = text(""" 18 | select count(DISTINCT ip) as total_num from {0} 19 | where `timestamp` >= UNIX_TIMESTAMP(:today) 20 | """.format(current_app.db_engine_table) 21 | ) 22 | 23 | res = cursor.execute(sql, {'today': cls.today}) 24 | 25 | total = Func.fetchone(res) 26 | return ApiCorsResponse.response(total) 27 | 28 | @classmethod 29 | def get_total_pv(cls): 30 | with current_app.db.connect() as cursor: 31 | sql = text(""" 32 | select count(*) as total_num from {0} FORCE INDEX(timestamp) 33 | where `timestamp` >= UNIX_TIMESTAMP(:today) 34 | """.format(current_app.db_engine_table) 35 | ) 36 | 37 | res = cursor.execute(sql, {'today': cls.today}) 38 | 39 | total = Func.fetchone(res) 40 | return ApiCorsResponse.response(total) 41 | 42 | @classmethod 43 | def get_request_num_by_url(cls): 44 | 45 | 46 | with current_app.db.connect() as cursor: 47 | sql = text(""" 48 | select count(*) as total_num,request_url from {0} 49 | where `timestamp` >= UNIX_TIMESTAMP(:today) 50 | group by request_url 51 | order by total_num desc 52 | limit 10 53 | """.format(current_app.db_engine_table) 54 | ) 55 | 56 | res = cursor.execute(sql,{'today':cls.today}) 57 | data = Func.fetchall(res) 58 | data.reverse() 59 | return ApiCorsResponse.response(data) 60 | 61 | @classmethod 62 | def get_request_num_by_ip(cls): 63 | with current_app.db.connect() as cursor: 64 | sql = text(""" 65 | select count(*) as total_num,ip from {0} 66 | where `timestamp` >= UNIX_TIMESTAMP(:today) 67 | group by ip 68 | order by total_num desc 69 | limit 50 70 | """.format(current_app.db_engine_table) 71 | ) 72 | 73 | res = cursor.execute(sql, {'today': cls.today}) 74 | data = Func.fetchall(res) 75 | data.reverse() 76 | return ApiCorsResponse.response(data) 77 | 78 | @classmethod 79 | def get_request_urls_by_ip(cls): 80 | if not request.args.get('ip'): 81 | return ApiCorsResponse.response('缺少ip参数', False) 82 | 83 | with current_app.db.connect() as cursor: 84 | sql = text(""" 85 | select count(*) as total_num ,request_url from {0} 86 | where FROM_UNIXTIME(`timestamp`,'%Y-%m-%d') = :today and ip = :ip 87 | group by request_url 88 | order by total_num desc 89 | limit 50 90 | """.format(current_app.db_engine_table) 91 | ) 92 | 93 | ip = request.args.get('ip') 94 | 95 | 96 | res = cursor.execute(sql, {'today': cls.today,'ip':ip}) 97 | data = Func.fetchall(res) 98 | data.reverse() 99 | 100 | return ApiCorsResponse.response(data) 101 | 102 | @classmethod 103 | def get_request_num_by_status(cls): 104 | with current_app.db.connect() as cursor: 105 | sql = text(""" 106 | select count(*) as total_num,`status_code` from {0} 107 | where `timestamp` >= UNIX_TIMESTAMP(:today) and `status_code` != 200 108 | group by status_code 109 | order by total_num desc 110 | limit 50 111 | """.format(current_app.db_engine_table) 112 | ) 113 | 114 | 115 | 116 | res = cursor.execute(sql, {'today': cls.today}) 117 | data = Func.fetchall(res) 118 | data.reverse() 119 | 120 | return ApiCorsResponse.response(data) 121 | 122 | @classmethod 123 | def get_request_num_by_status_code(cls): 124 | 125 | if not request.args.get('code'): 126 | return ApiCorsResponse.response('缺少code参数', False) 127 | 128 | with current_app.db.connect() as cursor: 129 | sql = text(""" 130 | select count(*) as total_num,`request_url` from {0} 131 | where FROM_UNIXTIME(`timestamp`,'%Y-%m-%d') = :today AND `status_code` = :status_code 132 | group by request_url 133 | order by total_num desc 134 | limit 30 135 | """.format(current_app.db_engine_table) 136 | ) 137 | 138 | code = request.args.get('code') 139 | 140 | 141 | res = cursor.execute(sql, {'today': cls.today,'status_code':code}) 142 | data = Func.fetchall(res) 143 | data.reverse() 144 | return ApiCorsResponse.response(data) 145 | 146 | @classmethod 147 | def get_request_num_by_secends(cls): 148 | 149 | if request.args.get('type') == 'init': 150 | # 　一分钟 * 10 10分钟 151 | limit = 60 * 10 152 | else: 153 | limit = 5 154 | 155 | with current_app.db.connect() as cursor: 156 | sql = text(""" 157 | select count(*) as total_request_num,`timestamp` from {0} 158 | where FROM_UNIXTIME(`timestamp`,'%Y-%m-%d') = :today 159 | group by `timestamp` 160 | order by `timestamp` desc 161 | limit {1} 162 | """.format(current_app.db_engine_table,limit) 163 | ) 164 | 165 | 166 | res = cursor.execute(sql, {'today': cls.today}) 167 | res = Func.fetchall(res) 168 | 169 | data = [] 170 | for i in res: 171 | 172 | item = {} 173 | # item['timestamp'] = time.strftime('%H:%M:%S', time.localtime(i['timestamp'])) 174 | # * 1000 for js timestamp 175 | item['timestamp'] = i['timestamp'] * 1000 176 | item['total_request_num'] = i['total_request_num'] 177 | data.append(item) 178 | 179 | data.reverse() 180 | 181 | return ApiCorsResponse.response(data) 182 | 183 | @classmethod 184 | def get_network_traffic_by_minute(self): 185 | with current_app.db.connect() as cursor: 186 | current_hour_str = time.strftime('%Y-%m-%d %H', time.localtime(time.time())) 187 | next_hour_str = time.strftime('%Y-%m-%d %H', time.localtime(time.time() + 3600)) 188 | current_hour = int(time.mktime(time.strptime(current_hour_str, '%Y-%m-%d %H'))) 189 | next_hour = int(time.mktime(time.strptime(next_hour_str, '%Y-%m-%d %H'))) 190 | 191 | sql = text(""" 192 | select round((sum(bytes_sent) /1024),2) as out_network, round((sum(request_length) / 1024) ,2) as in_network ,unix_timestamp(STR_TO_DATE(time_str,'%Y-%m-%d %H:%i')) as time_str 193 | from {0} 194 | where `timestamp` >= {1} and `timestamp` < {2} 195 | GROUP BY MINUTE(time_str) 196 | ORDER BY MINUTE(time_str) desc 197 | limit 10 198 | """.format(current_app.db_engine_table, current_hour, next_hour) 199 | ) 200 | 201 | res = cursor.execute(sql) 202 | data = Func.fetchall(res) 203 | data.reverse() 204 | 205 | return ApiCorsResponse.response(data) 206 | 207 | @classmethod 208 | def get_ip_pv_num_by_minute(cls): 209 | with current_app.db.connect() as cursor: 210 | current_hour_str = time.strftime('%Y-%m-%d %H', time.localtime(time.time())) 211 | next_hour_str = time.strftime('%Y-%m-%d %H', time.localtime(time.time() + 3600)) 212 | current_hour = int(time.mktime(time.strptime(current_hour_str, '%Y-%m-%d %H'))) 213 | next_hour = int(time.mktime(time.strptime(next_hour_str, '%Y-%m-%d %H'))) 214 | 215 | sql = text(""" 216 | select count(DISTINCT ip) as ip_num,count(*) as pv_num ,unix_timestamp(STR_TO_DATE(time_str,'%Y-%m-%d %H:%i')) as time_str 217 | from {0} 218 | where `timestamp` >= {1} and `timestamp` < {2} 219 | GROUP BY MINUTE(time_str) 220 | ORDER BY MINUTE(time_str) desc 221 | limit 10 222 | """.format(current_app.db_engine_table ,current_hour ,next_hour ) 223 | ) 224 | 225 | 226 | res = cursor.execute(sql) 227 | data = Func.fetchall(res) 228 | data.reverse() 229 | return ApiCorsResponse.response(data) 230 | 231 | @classmethod 232 | def get_request_num_by_province(cls): 233 | 234 | with current_app.db.connect() as cursor: 235 | sql = text(""" 236 | select count(*) as value,`province` from {0} 237 | where `timestamp` >= UNIX_TIMESTAMP(:today) AND province != '0' 238 | group by `province` 239 | order by `value` desc 240 | """.format(current_app.db_engine_table) 241 | ) 242 | 243 | res = cursor.execute(sql, {'today': cls.today}) 244 | data = Func.fetchall(res) 245 | data.reverse() 246 | 247 | return ApiCorsResponse.response(data) 248 | 249 | @classmethod 250 | def get_spider_by_ua(cls): 251 | with current_app.db.connect() as cursor: 252 | sql = text(""" 253 | select count(*) as total_num ,ua from {0} 254 | where MATCH(`ua`) AGAINST('spider') 255 | GROUP BY ua 256 | ORDER BY total_num desc 257 | """.format(current_app.db_engine_table) 258 | ) 259 | 260 | res = cursor.execute(sql, {'today': cls.today}) 261 | data = Func.fetchall(res) 262 | return ApiCorsResponse.response(data) 263 | 264 | @classmethod 265 | def get_device_type_by_ua(cls): 266 | with current_app.db.connect() as cursor: 267 | sql = text(""" 268 | select count(DISTINCT ua) as pc_num, ( 269 | select count(DISTINCT ua) as mobile_num 270 | from {0} 271 | where match(`ua`) AGAINST('mobile xfb' IN BOOLEAN MODE) 272 | ) as mobile_num 273 | from {1} 274 | where match(`ua`) AGAINST('+gecko -mobile' IN BOOLEAN MODE) 275 | """.format(current_app.db_engine_table,current_app.db_engine_table) 276 | ) 277 | 278 | res = cursor.execute(sql, {'today': cls.today}) 279 | data = Func.fetchall(res) 280 | return ApiCorsResponse.response(data) 281 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /webServer/api_request.http: -------------------------------------------------------------------------------- 1 | ### 获取今日ip 2 | GET http://127.0.0.1:5000/get_total_ip 3 | Accept: */* 4 | Cache-Control: no-cache 5 | Content-Type: application/x-www-form-urlencoded 6 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 7 | Content-Type: application/x-www-form-urlencoded 8 | 9 | ### 获取今日ip 10 | GET http://127.0.0.1:5000/get_total_pv 11 | Accept: */* 12 | Cache-Control: no-cache 13 | Content-Type: application/x-www-form-urlencoded 14 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 15 | Content-Type: application/x-www-form-urlencoded 16 | 17 | 18 | ### 获取每秒钟的请求数量（pv） 19 | GET http://127.0.0.1:5000/get_request_num_by_secends 20 | Accept: */* 21 | Cache-Control: no-cache 22 | Content-Type: application/x-www-form-urlencoded 23 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 24 | Content-Type: application/x-www-form-urlencoded 25 | 26 | ### 获取每分钟的请求数量（pv） 27 | GET http://127.0.0.1:5000/get_pv_num_by_minute 28 | Accept: */* 29 | Cache-Control: no-cache 30 | Content-Type: application/x-www-form-urlencoded 31 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 32 | Content-Type: application/x-www-form-urlencoded 33 | 34 | ### 获取每分钟的IP数量（ip） 35 | GET http://127.0.0.1:5000/get_ip_num_by_minute 36 | Accept: */* 37 | Cache-Control: no-cache 38 | Content-Type: application/x-www-form-urlencoded 39 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 40 | Content-Type: application/x-www-form-urlencoded 41 | 42 | 43 | ### 获取IP请求数量最多的前50 44 | GET http://127.0.0.1:5000/get_request_num_by_ip 45 | Accept: */* 46 | Cache-Control: no-cache 47 | Content-Type: application/x-www-form-urlencoded 48 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 49 | Content-Type: application/x-www-form-urlencoded 50 | 51 | 52 | 53 | 54 | ### 获取请求最多的 url 55 | GET http://127.0.0.1:5000/get_request_num_by_url?type=init 56 | Accept: */* 57 | Cache-Control: no-cache 58 | Content-Type: application/x-www-form-urlencoded 59 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 60 | Content-Type: application/x-www-form-urlencoded 61 | 62 | 63 | 64 | ### 获取某个请求状态码的所有url 65 | GET http://127.0.0.1:5000/get_request_num_by_status_code?code=502 66 | Accept: */* 67 | Cache-Control: no-cache 68 | Content-Type: application/x-www-form-urlencoded 69 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 70 | Content-Type: application/x-www-form-urlencoded 71 | 72 | ### 获取所有请求状态码 73 | GET http://127.0.0.1:5000/get_request_num_by_status 74 | Accept: */* 75 | Cache-Control: no-cache 76 | Content-Type: application/x-www-form-urlencoded 77 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 78 | Content-Type: application/x-www-form-urlencoded 79 | 80 | ### 获取所有中国城市请求分布 81 | GET http://127.0.0.1:5000/get_request_num_by_province 82 | Accept: */* 83 | Cache-Control: no-cache 84 | Content-Type: application/x-www-form-urlencoded 85 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 86 | Content-Type: application/x-www-form-urlencoded 87 | 88 | 89 | ### 获取每秒的请求量 90 | GET http://127.0.0.1:5000/get_request_num_by_secends 91 | Accept: */* 92 | Cache-Control: no-cache 93 | Content-Type: application/x-www-form-urlencoded 94 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 95 | Content-Type: application/x-www-form-urlencoded 96 | 97 | ### 获取前50的请求量最高的ip 98 | GET http://127.0.0.1:5000/get_request_num_by_ip 99 | Accept: */* 100 | Cache-Control: no-cache 101 | Content-Type: application/x-www-form-urlencoded 102 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 103 | Content-Type: application/x-www-form-urlencoded 104 | 105 | 106 | ### 获取某个ip 请求的urls 107 | GET http://127.0.0.1:5000/get_request_urls_by_ip?ip=47.112.167.32 108 | Accept: */* 109 | Cache-Control: no-cache 110 | Content-Type: application/x-www-form-urlencoded 111 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 112 | Content-Type: application/x-www-form-urlencoded 113 | 114 | ### 从ua 获取爬虫信息 115 | GET http://127.0.0.1:5000/get_spider_by_ua 116 | Accept: */* 117 | Cache-Control: no-cache 118 | Content-Type: application/x-www-form-urlencoded 119 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 120 | Content-Type: application/x-www-form-urlencoded 121 | 122 | ### 从ua 获取设备类型 pc端 和 移动端 对比曲线 123 | GET http://127.0.0.1:5000/get_device_type_by_ua 124 | Accept: */* 125 | Cache-Control: no-cache 126 | Content-Type: application/x-www-form-urlencoded 127 | Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJhcGkubS5jb20iLCJpYXQiOjE1OTM0MzkwNzUsInN1YiI6Img1IiwiYXVkIjoiTW96aWxsYVwvNS4wIChMaW51eDsgQW5kcm9pZCA4LjAuMDsgUGl4ZWwgMiBYTCBCdWlsZFwvT1BEMS4xNzA4MTYuMDA0KSBBcHBsZVdlYktpdFwvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lXC84My4wLjQxMDMuMTE2IE1vYmlsZSBTYWZhcmlcLzUzNy4zNiIsImV4cCI6MTYyNDk3NTA3NSwidWlkIjoiNWVmOWYzNjNlYzI3MDAwMDkwMDAzYWIyIiwiYWNjb3VudCI6IjE1OTI2OTAwNjUzIn0.aPgUqu0jnsXVcQvD03xobxk446qBQ58rWX0RwXIGKjVCqPMtTjMhrM1q5OY9VPUh_Ue6HG_wxLh4Gq3zGBcmi4xywq8PrHVJZkvH4Yv0Pvw4_yo1MpTkLGrXEpye1TBNe6W13bh1cjPKGuCvk0hH_3NoqtcbN5ayuJOj0eSZNzo 128 | Content-Type: application/x-www-form-urlencoded 129 | 130 | ### 131 | -------------------------------------------------------------------------------- /ParserAdapter/Nginx.py: -------------------------------------------------------------------------------- 1 | # coding=UTF-8 2 | from ParserAdapter.BaseAdapter import Adapter,ParseError,ReCompile 3 | import os,re,shutil,time 4 | 5 | """ 6 | $remote_addr,$http_x_forwarded_for #记录客户端IP地址 7 | $remote_user #记录客户端用户名称 8 | 9 | $request #记录请求的方法 URL和HTTP协议 (包含 $request_method $request_method http协议 比如: 'GET /api/server/?size=10&page=1 HTTP/1.1') 10 | $request_method 请求方法 11 | $request_uri 请求的URL 12 | $request_length #请求的长度（包括请求行，请求头和请求正文）。 13 | $request_time #请求处理时间，单位为秒，精度毫秒；从读入客户端的第一个字节开始，直到把最后一个字符发送给客户端后进行日志写入位置。 14 | $upstream_response_time 从Nginx向后端（php-cgi)建立连接开始到接受完数据然后关闭连接为止的时间 ($request_time 包括 $upstream_response_time) 所以如果使用nginx的accesslog查看php程序中哪些接口比较慢的话，记得在log_format中加入$upstream_response_time。 15 | 16 | $status #记录请求状态码 17 | $body_bytes_sent #发送给客户端的字节数，不包括响应头的大小；该变量与Apache模块mod_log_config李的“%B”参数兼容 18 | $bytes_sent #发送给客户端的总字节数 19 | $connection #连接到序列号 20 | $connection_requests #当前通过一个链接获得的请求数量 21 | 22 | $msec #日志写入时间，单位为秒精度是毫秒。 23 | $pipe #如果请求是通过HTTP流水线(pipelined)发送，pipe值为“p”,否则为".". 24 | 25 | $http_referer #记录从那个页面链接访问过来的 26 | $http_user_agent #记录客户端浏览器相关信息 27 | 28 | $time_iso8601 ISO8601标准格式下的本地时间 2020-09-11T15:01:38+08:00 29 | $time_local #通用日志格式下的本地时间 11/Sep/2020:15:01:38 +0800 30 | 31 | $upstream_addr 服务端响应的地址 32 | $upstream_http_host 服务端响应的地址 33 | 34 | 35 | """ 36 | 37 | 38 | 39 | class Handler(Adapter): 40 | 41 | def __init__(self,*args ,**kwargs): 42 | super(Handler,self).__init__(*args ,**kwargs) 43 | 44 | 45 | """ 46 | 日志变量 dict map: 47 | nickname 对该变量名称取别名 ; 默认直接使用变量名称 48 | re 指定该变量匹配的正则 ; 默认 [\s|\S]+? 49 | 50 | mysql_field_type 指定该变量值mysql中的 字段类型 【当存储引擎为 mysql 的时候才需配置】 51 | mysql_key_field True (bool) 默认普通索引 ,指定索引 用字符串 UNIQUE ,FULLTEXT (区分大小写) 【当存储引擎为 mysql 的时候才需配置】 52 | extend_field 拓展字段 【拓展字段不支持nickname】 53 | 列表的时候 54 | '$remote_addr': { 55 | 'nickname':'ip' , 56 | 'mysql_field_type':'varchar(15)', #【当存储引擎为 mysql 的时候才需配置】 57 | 'mysql_key_field': [ 58 | '$time_local.timestamp', # 表示当前字段 和 $time_local 里 extend timestamp 联合索引 key ip_timestamp (ip,timestamp) 59 | '$time_iso8601.timestamp', # 表示当前字段 和 $time_iso8601 里 extend timestamp 联合索引 key ip_timestamp (ip,timestamp) 60 | ['$status','$request.request_url','$request.method'] # 表示当前字段 和 $time_iso8601 里 extend timestamp 联合索引 key ip_status_request_url_method (ip,status,request,url,method) 61 | ], 62 | 'extend_field':{ # 拓展字段 由 parse_ip_to_area方法 拓展出来的 63 | 'isp':{ # 字段名称 64 | 'mysql_field_type': 'varchar(30)', # 字段类型及长度 【当存储引擎为 mysql 的时候才需配置】 65 | }, 66 | 'city':{ # 字段名称 67 | 'mysql_field_type': 'varchar(30)', # 字段类型及长度 【当存储引擎为 mysql 的时候才需配置】 68 | 'mysql_key_field': ['$status'], 69 | }, 70 | 'city_id':{ 71 | 'mysql_field_type': 'int(10)', 72 | }, 73 | 'province':{ 74 | 'mysql_field_type': 'varchar(30)', 75 | 'mysql_key_field': [ 76 | '$time_local.timestamp', 77 | '$time_iso8601.timestamp', 78 | ], 79 | }, 80 | 'country':{ 81 | 'mysql_field_type': 'varchar(30)', 82 | } 83 | } 84 | } 85 | 86 | """ 87 | def getLogFormat(self): 88 | 89 | return { 90 | ### 非 nginx 日志变量 附加自定义字段 91 | '$node_id':{ 92 | 'mysql_field_type': 'varchar(255)', 93 | 'mysql_key_field' : True 94 | }, 95 | '$app_name':{ 96 | 'mysql_field_type': 'varchar(255)', 97 | }, 98 | ### 非 nginx 日志变量 附加自定义字段 99 | 100 | 101 | # 客户端IP example 127.0.0.1 102 | '$remote_addr': { 103 | 'nickname':'ip' , 104 | 'mysql_field_type':'varchar(15)', 105 | 'mysql_key_field': [ 106 | '$time_local.timestamp', 107 | '$time_iso8601.timestamp', 108 | ['$status','$request.request_url','$request.request_method'] 109 | ], 110 | 'extend_field':{ 111 | 'isp':{ 112 | 'mysql_field_type': 'varchar(30)', 113 | }, 114 | 'city':{ 115 | 'mysql_field_type': 'varchar(30)', 116 | 'mysql_key_field': ['$status'], 117 | }, 118 | 'city_id':{ 119 | 'mysql_field_type': 'int(10)', 120 | }, 121 | 'province':{ 122 | 'mysql_field_type': 'varchar(30)', 123 | 'mysql_key_field': [ 124 | '$time_local.timestamp', 125 | '$time_iso8601.timestamp', 126 | ], 127 | }, 128 | 'country':{ 129 | 'mysql_field_type': 'varchar(30)', 130 | } 131 | } 132 | }, 133 | # 请求信息 example GET /api/server/?size=50&page=1 HTTP/1.1 134 | '$request': { 135 | 'extend_field': { 136 | 'request_method': { 137 | 'mysql_field_type': 'varchar(10)', 138 | 'mysql_key_field':True, 139 | }, 140 | 'request_url': { 141 | 'mysql_field_type': 'varchar(255)', 142 | 'mysql_key_field': ['$time_local.timestamp', '$time_iso8601.timestamp'], 143 | }, 144 | 'args': { 145 | 'mysql_field_type': 'text', 146 | }, 147 | 'server_protocol': { 148 | 'mysql_field_type': 'varchar(10)', 149 | }, 150 | } 151 | }, 152 | # 记录客户端用户名称 example client_name 153 | '$remote_user': { }, 154 | # 客户端代理IP多个逗号分割 example 203.98.182.163, 203.98.182.169 155 | '$http_x_forwarded_for': { 156 | 'nickname':'proxy_ip', 157 | 'mysql_field_type':'varchar(255)' 158 | }, 159 | # 请求方法 example GET 160 | '$request_method': { 161 | 'mysql_field_type': 'varchar(100)', 162 | }, 163 | # 请求协议 example HTTP/1.1 164 | '$scheme':{ 165 | 'mysql_field_type': 'varchar(255)' 166 | } , 167 | # 服务器的HTTP版本 example “HTTP/1.0” 或 “HTTP/1.1” 168 | '$server_protocol':{ 169 | 'mysql_field_type': 'varchar(10)' 170 | } , 171 | # 请求链接 example /api/server/?size=50&page=1 172 | '$request_uri': { 173 | 'nickname':'request_url', 174 | 'mysql_field_type': 'varchar(255)', 175 | 'mysql_key_field': ['$time_local.timestamp', '$time_iso8601.timestamp'], 176 | }, 177 | # post提交的数据 example name=xxx&age=18 178 | '$request_body': { 179 | 'mysql_field_type': 'mediumtext' 180 | }, 181 | # 请求的字节长度 example 988 182 | '$request_length': { 183 | 're': '\d+?' , 184 | 'type':'int', 185 | 186 | }, 187 | # 请求花费的时间 example 0.018 188 | '$request_time': { 189 | 'type':'float', 190 | 'mysql_field_type':'float(10, 3)', 191 | }, 192 | # 当前的Unix时间戳 nginx 版本需大于 (1.3.9, 1.2.6) 193 | '$msec': { 194 | 'mysql_field_type': 'int(16)', 195 | }, 196 | # nginx交给后端cgi响应的时间(小于$request_time) example 0.215 或者 0.215, 0.838 197 | '$upstream_response_time': { 198 | 'mysql_field_type': 'varchar(50)', 199 | }, 200 | # 请求状态码 example 200 201 | '$status': { 202 | 're': '\d*?', 203 | 'nickname':'status_code', 204 | 'mysql_field_type': 'int(4)', 205 | 'mysql_key_field': ['$time_local.timestamp','$time_iso8601.timestamp'], 206 | }, 207 | # 发送给客户端的总字节数 (包括响应头) example 113 208 | '$bytes_sent':{ 209 | 're': '\d*?' , 210 | 'type':'int', 211 | 'mysql_field_type': 'int(4)', 212 | } , 213 | # 发送给客户端的总字节数(不包括响应头) example 266 214 | '$body_bytes_sent':{ 215 | 're': '\d*?' , 216 | 'type':'int', 217 | 'mysql_field_type': 'int(4)', 218 | } , 219 | # 连接到序列号 example 26 220 | '$connection':{ 221 | 're': '\d*?' , 222 | 'type':'int', 223 | 'mysql_field_type': 'int(10)', 224 | } , 225 | # 每个连接到序列号的请求次数 example 3 226 | '$connection_requests':{ 227 | 're': '\d*?', 228 | 'type':'int', 229 | 'mysql_field_type': 'int(10)', 230 | } , 231 | # 请求头里的host属性,如果没有就返回 server_name example www.baidu.com 232 | '$host':{ 233 | 'mysql_field_type': 'varchar(255)', 234 | } , 235 | # 请求头里的host属性 example www.baidu.com 236 | '$http_host':{ 237 | 'mysql_field_type': 'varchar(255)', 238 | } , 239 | # 请求的来源网址 example www.baidu.com 240 | '$http_referer':{ 241 | 'mysql_field_type': 'varchar(3000)', 242 | 243 | } , 244 | # 客户端的UA信息 example Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36 245 | '$http_user_agent':{ 246 | 'nickname':'ua', 247 | 'mysql_field_type': 'mediumtext', 248 | 'mysql_key_field': 'FULLTEXT' 249 | 250 | } , 251 | # 后端接收请求的服务的ip或地址 example unix:/tmp/php-cgi-71.sock 负载均衡中 是 192.168.0.122:80, 192.168.0.133:80 252 | '$upstream_addr':{ 253 | 'mysql_field_type': 'varchar(255)', 254 | } , 255 | # 服务端响应的地址 unix:/tmp/php-cgi-71.sock 同上 是域名 256 | '$upstream_http_host':{ 257 | 'mysql_field_type': 'varchar(255)', 258 | } , 259 | # iso8601时间格式 example 2020-09-11T15:20:43+08:00 260 | '$time_iso8601':{ 261 | 'extend_field': { 262 | 'time_str': { 263 | 'mysql_field_type': 'datetime', 264 | 'mysql_key_field': True 265 | }, 266 | 'timestamp': { 267 | 'mysql_field_type': 'int(10)', 268 | 'mysql_key_field': True 269 | }, 270 | } 271 | } , 272 | # 本地时间格式 example 11/Sep/2020:15:20:43 +0800 273 | '$time_local':{ 274 | 'extend_field': { 275 | 'time_str': { 276 | 'mysql_field_type': 'datetime', 277 | 'mysql_key_field': True 278 | }, 279 | 'timestamp': { 280 | 'mysql_field_type': 'int(10)', 281 | 'mysql_key_field': True 282 | }, 283 | } 284 | } , 285 | } 286 | 287 | # 日志解析 288 | def parse(self,log_format_name='',log_line=''): 289 | 290 | 291 | log_format_list = self.log_line_pattern_dict[log_format_name]['log_format_list'] 292 | log_format_recompile = self.log_line_pattern_dict[log_format_name]['log_format_recompile'] 293 | 294 | start_time = time.perf_counter() 295 | 296 | res = log_format_recompile.match(log_line) 297 | 298 | 299 | if res == None: 300 | raise ParseError('解析日志失败,请检查client 配置中 日志的 格式名称是否一致 log_format_name') 301 | 302 | matched = list(res.groups()) 303 | 304 | if len(matched) == len(log_format_list): 305 | data = {} 306 | del_key_name = [] 307 | 308 | for i in range(len(list(log_format_list))): 309 | key_name = log_format_list[i].replace('$','') 310 | 311 | # request 参数不支持别名 312 | if 'nickname' in self.getLogFormat()[ log_format_list[i] ] : 313 | key_name = self.getLogFormat()[ log_format_list[i] ]['nickname'] 314 | 315 | 316 | # 解析 ip 对应的 地理位置信息 317 | if log_format_list[i] == '$remote_addr': 318 | ip_data = self.parse_ip_to_area(matched[i]) 319 | data.update(ip_data) 320 | 321 | 322 | # 解析 $request 成 request_method ,request_url ,args ,server_protocol , 323 | if log_format_list[i] == '$request': 324 | request_extend_data = self.parse_request_to_extend(matched[i]) 325 | data.update(request_extend_data) 326 | del_key_name.append(key_name) 327 | 328 | # 解析 time_iso8601 成 timestr , timestamp 329 | if log_format_list[i] == '$time_local': 330 | time_data = self.parse_time_to_str(log_format_list[i].replace('$',''),matched[i]) 331 | data.update(time_data) 332 | del_key_name.append(key_name) 333 | 334 | 335 | data[key_name] = matched[i] 336 | 337 | 338 | 339 | # 剔除掉 解析出拓展字符串的 字段 340 | for i in del_key_name: 341 | del data[i] 342 | 343 | 344 | return data 345 | 346 | 347 | # 根据录入的格式化字符串 返回 parse 所需 log_format 配置以及进行对应的表达式预编译 348 | def getLogFormatByConfStr(self ,log_format_str,log_format_vars,log_format_name ,log_type): 349 | if log_type not in ['string','json']: 350 | raise ValueError('_type 参数类型错误') 351 | 352 | # 日志格式不存在 则 预编译 353 | if log_format_name not in self.log_line_pattern_dict: 354 | 355 | # 过滤下 日志配置字符串 与 正则冲突特殊字符 356 | log_format_str = log_format_str.strip()\ 357 | .replace('[','\[').replace(']','\]')\ 358 | .replace('(','\(').replace(')','\)')\ 359 | .replace('|','\|').replace('-','\-')\ 360 | .replace('+','\+').replace('*','\*')\ 361 | .replace('?','\?') 362 | 363 | 364 | if (log_type == 'string'): 365 | 366 | # 获取到匹配到的 日志格式 367 | 368 | log_format_list = log_format_vars.split(self.LOG_FORMAT_SPLIT_TAG) 369 | 370 | re_str = re.sub(r'(\$\w+)+', self.__replaceLogVars, log_format_str).strip() 371 | 372 | try: 373 | re_compile = re.compile(re_str, re.I) 374 | 375 | except re.error: 376 | raise ReCompile('预编译错误,请检查日志字符串中是否包含特殊正则字符; 日志:%s' % log_format_str) 377 | 378 | self.log_line_pattern_dict[log_format_name] = { 379 | 'log_format_list':log_format_list , 380 | 'log_format_recompile':re_compile 381 | } 382 | 383 | # 找到匹配中的日志变量替换成正则表达式 384 | def __replaceLogVars(self,matched): 385 | 386 | 387 | s = matched.group() 388 | 389 | if s not in self.getLogFormat(): 390 | raise ValueError('handle 里面不存在日志变量:%s' % s) 391 | 392 | if 're' in self.getLogFormat()[s]: 393 | re_str = self.getLogFormat()[s]['re'] 394 | else: 395 | re_str = '[\s|\S]*?' 396 | 397 | return '(%s)' % re_str 398 | 399 | # 获取服务器配置文件中所有的日志配置 400 | def getLoggerFormatByServerConf(self,server_conf_path): 401 | 402 | # 根据配置文件 自动获取 log_format 字符串 403 | with open(server_conf_path,'rb') as fd: 404 | content = fd.read().decode(encoding="utf-8") 405 | 406 | defualt_log_vars = self.LOG_FORMAT_SPLIT_TAG.join( 407 | ['$remote_addr', 408 | '$remote_user', 409 | '$time_local', 410 | '$request', 411 | '$status', 412 | '$body_bytes_sent', 413 | '$http_referer', 414 | '$http_user_agent' 415 | ] 416 | ) 417 | format_list = {} 418 | format_list['defualt'] = { 419 | 'log_format_str':'$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"', 420 | 'log_format_vars' : defualt_log_vars 421 | } 422 | 423 | 424 | res = re.findall(r'log_format\s+(\w+)\s+\'([\s\S]*?\S?)\'\S?\;' ,content) 425 | 426 | if len(res) == 0: 427 | return format_list 428 | 429 | 430 | for i in res: 431 | log_str = i[1].strip() 432 | log_vars = re.findall(r'(\$\w+)+',log_str) 433 | if(len(log_vars)): 434 | format_list[i[0]] = { 435 | 'log_format_str':log_str, 436 | 'log_format_vars':self.LOG_FORMAT_SPLIT_TAG.join(log_vars) 437 | } 438 | 439 | del content 440 | 441 | return format_list 442 | 443 | 444 | # 切割日志 445 | def rotatelog(self,server_conf,log_path ,target_file = None ): 446 | try: 447 | 448 | if not os.path.exists(server_conf['pid_path']): 449 | raise FileNotFoundError(server_conf['pid_path'] + '配置项 server nginx_pid_path 不存在') 450 | 451 | if not os.path.exists(log_path): 452 | raise FileNotFoundError(log_path + ' 不存在') 453 | 454 | 455 | # 这里需要注意 日志目录的 权限 是否有www 否则会导致 ngixn 重开日志问件 无法写入的问题 456 | cmd = 'kill -USR1 `cat %s`' % (server_conf['pid_path']) 457 | shutil.move(log_path, target_file) 458 | 459 | res = os.popen(cmd) 460 | if len(res.readlines()) > 0: 461 | cmd_res = '' 462 | for i in res.readlines(): 463 | cmd_res += i + '\n' 464 | raise Exception ('reload 服务器进程失败: %s' % cmd_res) 465 | 466 | return True 467 | except Exception as e: 468 | return '切割日志失败 : %s ; error class : %s error info : %s' % (target_file ,e.__class__, e.args) 469 | 470 | 471 | 472 | -------------------------------------------------------------------------------- /Src/Core.py: -------------------------------------------------------------------------------- 1 | # coding=UTF-8 2 | from ParserAdapter.BaseAdapter import ParseError,ReCompile 3 | from configparser import ConfigParser 4 | from threading import RLock 5 | from collections import deque 6 | import time,json,os,platform,importlib,logging 7 | 8 | 9 | try: 10 | # Python 3.x 11 | from urllib.parse import quote_plus 12 | except ImportError: 13 | # Python 2.x 14 | from urllib import quote_plus 15 | 16 | 17 | 18 | # 日志解析 19 | class loggerParse(object): 20 | 21 | 22 | def __init__(self ,server_type,server_conf = None): 23 | 24 | self.server_type = server_type 25 | self.__handler = Base.findAdapterHandler('server',server_type)() 26 | self.format = self.__handler.getLogFormat() 27 | if server_conf: 28 | self.logger_format = self.__handler.getLoggerFormatByServerConf(server_conf_path=server_conf) 29 | 30 | 31 | def getLogFormatByConfStr(self,log_format_str,log_format_vars,log_format_name): 32 | 33 | return self.__handler.getLogFormatByConfStr( log_format_str,log_format_vars,log_format_name,'string') 34 | 35 | 36 | def parse(self,log_format_name,log_line): 37 | return self.__handler.parse(log_format_name=log_format_name,log_line=log_line) 38 | 39 | 40 | 41 | class Base(object): 42 | conf = None 43 | CONFIG_FIEL_SUFFIX = '.ini' 44 | config_name = 'config' 45 | 46 | def __init__(self,config_name = None): 47 | self._root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 48 | self.conf = self.__getConfig(config_name) 49 | self.config_name = config_name 50 | self.__initLogging() 51 | 52 | 53 | def __initLogging(self): 54 | LOG_FORMAT = "%(asctime)s %(levelname)s %(pathname)s %(lineno)s %(message)s " 55 | DATE_FORMAT = '%Y-%m-%d %H:%M:%S ' 56 | if self.__class__.__name__ == 'Base':return 57 | 58 | logg_setting_map = { 59 | 'Reader': 'inputer', 60 | 'OutputCustomer': 'outputer', 61 | } 62 | 63 | conf_name = logg_setting_map[self.__class__.__name__] 64 | 65 | if 'log_debug' in self.conf[conf_name] and self.conf[conf_name]['log_debug'] == 'True': 66 | _level = logging.DEBUG 67 | else: 68 | _level = logging.INFO 69 | 70 | logging.basicConfig(level=_level, format=LOG_FORMAT, datefmt=DATE_FORMAT, 71 | filename=r"./%s_%s.log" % (conf_name ,self.config_name) ) 72 | 73 | self.logging = logging 74 | 75 | def __getConfig(self,config_name): 76 | if config_name: 77 | self.config_name = config_name 78 | 79 | if self.config_name.find(self.CONFIG_FIEL_SUFFIX) == -1: 80 | config_path = self._root + '/' + self.config_name + self.CONFIG_FIEL_SUFFIX 81 | else: 82 | config_path = self._root + '/' + self.config_name 83 | 84 | if ( not os.path.exists(config_path) ): 85 | raise FileNotFoundError('config file: %s not found ' % (config_path) ) 86 | 87 | conf = ConfigParser() 88 | conf.read(config_path, encoding="utf-8") 89 | 90 | return conf 91 | 92 | @classmethod 93 | def findAdapterHandler(cls,adapter_type,name): 94 | 95 | if adapter_type not in ['server', 'queue', 'storage','traffic_analysis']: 96 | raise ValueError('%s Adapter 类型不存在' % adapter_type) 97 | 98 | if adapter_type in ['queue', 'storage']: 99 | handler_module = '%sAdapter.%s' % (adapter_type.lower().capitalize(), name.lower().capitalize()) 100 | print(handler_module) 101 | elif adapter_type == 'traffic_analysis': 102 | handler_module = '%sAdapter.%s' % (adapter_type.lower().capitalize(), name.lower().capitalize()) 103 | print(handler_module) 104 | exit() 105 | else: 106 | handler_module = 'ParserAdapter.%s' % name.lower().capitalize() 107 | 108 | if adapter_type == 'queue': 109 | return importlib.import_module(handler_module).QueueAp 110 | if adapter_type == 'storage': 111 | return importlib.import_module(handler_module).StorageAp 112 | if adapter_type == 'traffic_analysis': 113 | return importlib.import_module(handler_module).TrafficAnalysisAp 114 | if adapter_type == 'server': 115 | return importlib.import_module(handler_module).Handler 116 | 117 | 118 | def runMethod(self,method_name): 119 | self.logging.debug('pid:%s , %s ,%s' % (os.getpid() ,method_name ,time.perf_counter())) 120 | getattr(self,method_name)() 121 | 122 | 123 | # 生产者 实时读取日志 && 切割日志 && 上报服务器状况 124 | class Reader(Base): 125 | 126 | event = { 127 | 'cut_file' : 0, 128 | 'stop' : None, 129 | } 130 | 131 | def __init__(self,log_file_conf = None ,config_name = None): 132 | super(Reader, self).__init__(config_name=config_name) 133 | 134 | self.log_path = log_file_conf['file_path'] 135 | 136 | 137 | if 'log_format_name' not in log_file_conf or len(log_file_conf['log_format_name']) == 0: 138 | self.log_format_name = 'defualt' 139 | else: 140 | self.log_format_name = log_file_conf['log_format_name'] 141 | 142 | 143 | self.node_id = self.conf['inputer']['node_id'] 144 | 145 | # 最大写入队列的数据量 146 | if 'max_batch_push_queue_size' in self.conf['inputer']: 147 | self.max_batch_push_queue_size = int(self.conf['inputer']['max_batch_push_queue_size']) 148 | else: 149 | self.max_batch_push_queue_size = 5000 150 | 151 | # 最大重试打开文件次数 152 | if 'max_retry_open_file_time' in self.conf['inputer']: 153 | self.max_retry_open_file_time = int(self.conf['inputer']['max_retry_open_file_time']) 154 | else: 155 | self.max_retry_open_file_time = 10 156 | 157 | # 最大重试链接 queue的次数 158 | if 'max_retry_reconnect_time' in self.conf['inputer']: 159 | self.max_retry_reconnect_time = int(self.conf['inputer']['max_retry_reconnect_time']) 160 | else: 161 | self.max_retry_reconnect_time = 20 162 | 163 | 164 | self.app_name = log_file_conf['app_name'] 165 | self.server_type = log_file_conf['server_type'] 166 | self.read_type = log_file_conf['read_type'] 167 | self.cut_file_type = log_file_conf['cut_file_type'] 168 | self.cut_file_point = log_file_conf['cut_file_point'] 169 | try: 170 | self.cut_file_save_dir = log_file_conf['cut_file_save_dir'] 171 | except KeyError as e: 172 | self.cut_file_save_dir = None 173 | 174 | 175 | log_prev_path = os.path.dirname(log_file_conf['file_path']) 176 | 177 | if platform.system() == 'Linux': 178 | self.newline_char = '\n' 179 | import pwd 180 | """ 181 | 这里需要将 nginx 日志的所属 目录修改为 www 否则在切割日志的时候 kill -USR1 pid 之后 日志文件会被重新打开但是因权限问题不会继续写入文件中 182 | """ 183 | # 检查日志目录所属用户 ; 不是 www 则修改成 www 184 | if pwd.getpwuid(os.stat(log_prev_path).st_uid).pw_name != 'www' and platform.system() == 'Linux': 185 | try: 186 | www_uid = pwd.getpwnam('www').pw_uid 187 | os.chown(log_prev_path, www_uid, www_uid) 188 | except PermissionError as e: 189 | exit('权限不足 : 修改目录: %s 所属用户和用户组 为 www 失败 ' % (log_prev_path)) 190 | 191 | 192 | elif platform.system() == 'Windows': 193 | self.newline_char = '\r\n' 194 | 195 | # 内部队列 196 | self.dqueue = deque() 197 | 198 | self.queue_key = self.conf['inputer']['queue_name'] 199 | 200 | self.server_conf = loggerParse(log_file_conf['server_type'],self.conf[log_file_conf['server_type']]['server_conf']).logger_format 201 | 202 | self.fd = self.__getFileFd() 203 | 204 | # 文件切割中标志 205 | self.cutting_file = False 206 | self.lock = RLock() 207 | 208 | # 外部队列handle 209 | self.queue_handle = self.findAdapterHandler('queue',self.conf['inputer']['queue']).initQueue(self) 210 | self.server_handle = self.findAdapterHandler('server',log_file_conf['server_type'])() 211 | 212 | 213 | def __getFileFd(self): 214 | try: 215 | return open(self.log_path, mode='r+' ,newline=self.newline_char) 216 | 217 | except PermissionError as e: 218 | self.event['stop'] = self.log_path + '没有权限读取文件 请尝试sudo' 219 | return False 220 | 221 | except OSError as e: 222 | self.event['stop'] = self.log_path + ' 文件不存在' 223 | return False 224 | 225 | 226 | def __cutFileHandle(self,server_conf,log_path ,target_path = None ): 227 | 228 | start_time = time.perf_counter() 229 | self.logging.debug("\n start_time -------cutting file start --- %s \n" % ( 230 | start_time)) 231 | 232 | file_suffix = time.strftime('%Y-%m-%d_%H-%M-%S', time.localtime()) 233 | 234 | files_arr = log_path.split('/') 235 | log_name = files_arr.pop().replace('.log', '') 236 | log_path_dir = '/'.join(files_arr) 237 | 238 | 239 | if target_path : 240 | target_dir = target_path + '/' + log_name 241 | else: 242 | target_dir = log_path_dir + '/' + log_name 243 | 244 | if not os.path.exists(target_dir): 245 | try: 246 | os.makedirs(target_dir) 247 | except Exception as e: 248 | self.event['stop'] = '日志切割存储目录创建失败' 249 | return 250 | 251 | target_file = target_dir + '/' + log_name + '_' + file_suffix 252 | 253 | res = self.server_handle.rotatelog(server_conf,log_path,target_file) 254 | if(isinstance(res,str) and res != True): 255 | self.event['stop'] = res 256 | 257 | end_time = time.perf_counter() 258 | self.logging.debug('finnish file_cut take time : %s' % round((start_time - end_time))) 259 | 260 | 261 | def cutFile(self): 262 | 263 | while True: 264 | time.sleep(1) 265 | if self.event['stop']: 266 | self.logging.debug( '%s ; cutFile threading stop pid: %s' % (self.event['stop'] , os.getpid())) 267 | return 268 | 269 | 270 | if self.cut_file_type not in ['filesize', 'time']: 271 | self.event['stop'] = 'cut_file_type 只支持 filesize 文件大小 或者 time 指定每天的时间' 272 | continue 273 | 274 | 275 | self.lock.acquire() 276 | 277 | if self.cut_file_type == 'filesize' : 278 | 279 | try: 280 | now = time.strftime("%H:%M", time.localtime(time.time())) 281 | # self.logging.debug('cut_file_type: filesize ;%s ---pid: %s----thread_id: %s--- %s ---------%s' % ( now, os.getpid(), threading.get_ident(), self.cut_file_point, self.cutting_file)) 282 | 283 | # 文件大小 单位 M 284 | file_size = round(os.path.getsize(self.log_path) / (1024 * 1024)) 285 | if file_size < int(self.cut_file_point): 286 | self.lock.release() 287 | continue 288 | 289 | except FileNotFoundError as e: 290 | self.event['stop'] = self.log_path + '文件不存在' 291 | self.lock.release() 292 | continue 293 | 294 | self.__cutFileHandle( 295 | server_conf = dict(self.conf[self.server_type]) , 296 | log_path= self.log_path , 297 | target_path = self.cut_file_save_dir 298 | ) 299 | 300 | self.event['cut_file'] = 1 301 | 302 | elif self.cut_file_type == 'time': 303 | 304 | now = time.strftime("%H:%M" , time.localtime(time.time()) ) 305 | # self.logging.debug('cut_file_type: time ;%s ---pid: %s----thread_id: %s--- %s ---------%s' % ( 306 | # now,os.getpid(), threading.get_ident(), self.cut_file_point, self.cutting_file)) 307 | 308 | if now == self.cut_file_point and self.cutting_file == False: 309 | self.__cutFileHandle( 310 | server_conf = dict(self.conf[self.server_type]) , 311 | log_path = self.log_path , 312 | target_path = self.cut_file_save_dir 313 | ) 314 | 315 | self.cutting_file = True 316 | self.event['cut_file'] = 1 317 | elif now == self.cut_file_point and self.cutting_file == True and self.event['cut_file'] == 1: 318 | self.event['cut_file'] == 0 319 | 320 | 321 | elif now != self.cut_file_point: 322 | self.cutting_file = False 323 | self.event['cut_file'] = 0 324 | 325 | 326 | self.lock.release() 327 | 328 | 329 | def pushDataToQueue(self): 330 | 331 | self.queue_handle.pushDataToQueue() 332 | 333 | 334 | def readLog(self): 335 | 336 | 337 | position = 0 338 | 339 | if self.read_type not in ['head','tail']: 340 | self.event['stop'] = 'read_type 只支持 head 从头开始 或者 tail 从末尾开始' 341 | 342 | if self.fd == False: 343 | return 344 | 345 | try: 346 | if self.read_type == 'head': 347 | self.fd.seek(position, 0) 348 | elif self.read_type == 'tail': 349 | self.fd.seek(position, 2) 350 | except Exception as e: 351 | self.event['stop'] = self.log_path + ' 文件句柄 seek 错误 %s ; %s' % (e.__class__ , e.args) 352 | 353 | 354 | max_retry_open_file_time = 3 355 | retry_open_file_time = 0 356 | while True: 357 | time.sleep(1) 358 | 359 | if self.event['stop']: 360 | self.logging.debug( '%s ; read threading stop pid: %s' % (self.event['stop'] ,os.getpid())) 361 | return 362 | 363 | start_time = time.perf_counter() 364 | # self.logging.debug("\n start_time -------pid: %s -- read file---queue len: %s---- %s \n" % ( os.getpid(), len(list(self.dqueue)), round(start_time, 2))) 365 | 366 | 367 | for line in self.fd: 368 | # 不是完整的一行继续read 369 | if line.find(self.newline_char) == -1: 370 | continue 371 | 372 | self.dqueue.append(line) 373 | 374 | 375 | end_time = time.perf_counter() 376 | self.logging.debug("\n end_time -------pid: %s -- read file- %s--line len :%s --- 耗时:%s \n" % (os.getpid(),self.log_path, len(list(self.dqueue)), round(end_time - start_time, 2))) 377 | 378 | if self.event['cut_file'] == 1 and self.event['stop'] == None: 379 | # 防止 重启进程服务后 新的日志文件并没有那么快重新打开 380 | time.sleep(1.5) 381 | self.logging.debug('--------------------reopen file--------------------at: %s' % time.time()) 382 | 383 | self.fd.close() 384 | self.fd = self.__getFileFd() 385 | try: 386 | self.fd.seek(0) 387 | except AttributeError as e: 388 | time.sleep(1) 389 | retry_open_file_time = retry_open_file_time + 1 390 | if retry_open_file_time >= max_retry_open_file_time: 391 | self.event['stop'] = '重新打开文件超过最大次数 %s ' % max_retry_open_file_time 392 | continue 393 | 394 | self.event['cut_file'] = 0 395 | 396 | 397 | 398 | # 消费者 解析日志 && 存储日志 399 | class OutputCustomer(Base): 400 | 401 | def __init__(self , config_name = None): 402 | 403 | super(OutputCustomer,self).__init__(config_name) 404 | 405 | self.dqueue = deque() 406 | 407 | self.inputer_queue_type = self.conf['outputer']['queue'] 408 | self.queue_key = self.conf['outputer']['queue_name'] 409 | 410 | 411 | self.save_engine_conf = dict( self.conf[ self.conf['outputer']['save_engine'] ]) 412 | self.save_engine_name = self.conf['outputer']['save_engine'].lower().capitalize() 413 | 414 | 415 | self.server_type = self.conf['outputer']['server_type'] 416 | self.logParse = loggerParse(self.conf['outputer']['server_type'] ,server_conf=None) 417 | 418 | if 'max_batch_insert_db_size' in self.conf['outputer']: 419 | self.max_batch_insert_db_size = int(self.conf['outputer']['max_batch_insert_db_size']) 420 | else: 421 | self.max_batch_insert_db_size = 500 422 | 423 | 424 | # 外部队列handle 425 | self.queue_handle = self.findAdapterHandler('queue',self.conf['outputer']['queue']).initQueue(self) 426 | # 外部 存储引擎 427 | self.storage_handle = self.findAdapterHandler('storage',self.conf['outputer']['save_engine']).initStorage(self) 428 | 429 | # self.traffic_analysis_handel = self.findAdapterHandler('traffic_analysis',self.conf['outputer']['save_engine']).initStorage(self) 430 | 431 | 432 | 433 | def _parse_line_data(self,line): 434 | 435 | if isinstance(line ,str): 436 | line_data = json.loads(line) 437 | else: 438 | line_data = line 439 | 440 | 441 | try: 442 | 443 | # 预编译对应的正则 444 | self.logParse.getLogFormatByConfStr(line_data['log_format_str'],line_data['log_format_vars'], line_data['log_format_name']) 445 | 446 | line_data['line'] = line_data['line'].strip() 447 | 448 | parse_data = self.logParse.parse(line_data['log_format_name'], line_data['line']) 449 | 450 | # 解析数据错误 451 | except ParseError as e: 452 | self.logging.error('\n pid : %s 数据解析错误: %s 数据: %s' % (os.getpid(), e.args, line)) 453 | return False 454 | except ReCompile as e: 455 | unkown_error = '\n pid : %s 预编译错误: %s ,error_class: %s ,数据: %s' % (os.getpid(), e.__class__, e.args, line) 456 | self.logging.error(unkown_error) 457 | raise Exception(unkown_error) 458 | 459 | except Exception as e: 460 | unkown_error = '\n pid : %s 未知错误: %s ,error_class: %s ,数据: %s' % (os.getpid(), e.__class__, e.args, line) 461 | self.logging.error(unkown_error) 462 | raise Exception(unkown_error) 463 | 464 | 465 | del line_data['log_format_name'] 466 | del line_data['log_format_str'] 467 | del line_data['log_format_vars'] 468 | del line_data['line'] 469 | 470 | line_data.update(parse_data) 471 | 472 | 473 | return line_data 474 | 475 | 476 | def _get_queue_count_num(self): 477 | return self.queue_handle.getDataCountNum() 478 | 479 | 480 | #　获取队列数据 481 | def getQueueData(self): 482 | return self.queue_handle.getDataFromQueue() 483 | 484 | # 消费队列 485 | def saveToStorage(self ): 486 | self.storage_handle.pushDataToStorage() 487 | 488 | def watchTraffic(self): 489 | 490 | self.traffic_analysis_handel.start() 491 | 492 | #退回队列 493 | def rollBackQueue(self,data_list): 494 | self.queue_handle.rollBackToQueue(data_list) 495 | 496 | 497 | 498 | 499 | if __name__ == "__main__": 500 | pass 501 | 502 | -------------------------------------------------------------------------------- /StorageAdapter/Mysql.py: -------------------------------------------------------------------------------- 1 | import redis.exceptions 2 | 3 | from StorageAdapter.BaseAdapter import Adapter 4 | import time,threading,os,json,pymysql,re 5 | 6 | 7 | 8 | try: 9 | # Python 3.x 10 | from urllib.parse import quote_plus 11 | except ImportError: 12 | # Python 2.x 13 | from urllib import quote_plus 14 | 15 | class StorageAp(Adapter): 16 | 17 | db = None 18 | runner = None 19 | field_map = None 20 | key_field_map = None 21 | 22 | 23 | @classmethod 24 | def initStorage(cls,runnerObject): 25 | self = cls() 26 | self.runner = runnerObject 27 | self.conf = self.runner.conf 28 | self.logging = self.runner.logging 29 | 30 | pymysql_timeout_secends = 60 31 | try: 32 | self.db = pymysql.connect( 33 | host = self.conf['mysql']['host'], 34 | port = int(self.conf['mysql']['port']), 35 | user = quote_plus(self.conf['mysql']['username']), 36 | password = quote_plus(self.conf['mysql']['password']), 37 | db = quote_plus(self.conf['mysql']['db']), 38 | connect_timeout = pymysql_timeout_secends, 39 | read_timeout = pymysql_timeout_secends, 40 | write_timeout = pymysql_timeout_secends, 41 | 42 | ) 43 | 44 | except pymysql.err.MySQLError: 45 | self.logging.error('Mysql 链接失败,请检查配置文件!') 46 | raise Exception('Mysql 链接失败,请检查配置文件!') 47 | 48 | 49 | return self 50 | 51 | 52 | def pushDataToStorage(self): 53 | retry_reconnect_time = 0 54 | 55 | while True: 56 | time.sleep(0.1) 57 | self._getTableName('table') 58 | 59 | if retry_reconnect_time == 0: 60 | try: 61 | # 获取队列数据 62 | queue_data = self.runner.getQueueData() 63 | except redis.exceptions.RedisError as e: 64 | self.logging.error( 65 | "\n outputerer -------pid: %s -- redis error at: %s time---- Exceptions %s ; %s \n" % ( 66 | os.getpid(), retry_reconnect_time, e.__class__, e.args)) 67 | print("\n outputerer -------pid: %s -- redis error at: %s time---- Exceptions %s ; %s \n" % ( 68 | os.getpid(), retry_reconnect_time, e.__class__, e.args)) 69 | return 70 | 71 | 72 | 73 | 74 | if len(queue_data) == 0: 75 | self.logging.debug('\n outputerer ---pid: %s wait for queue data \n ' % (os.getpid())) 76 | continue 77 | 78 | 79 | 80 | start_time = time.perf_counter() 81 | 82 | # 　错误退回队列 (未解析的原始的数据) 83 | self.backup_for_push_back_queue = [] 84 | 85 | _data = [] 86 | for item in queue_data: 87 | if isinstance(item, bytes): 88 | item = item.decode(encoding='utf-8') 89 | 90 | 91 | self.backup_for_push_back_queue.append(item) 92 | # 解析日志数据 93 | item = self.runner._parse_line_data(item) 94 | 95 | if item: 96 | _data.append(item) 97 | 98 | 99 | end_time = time.perf_counter() 100 | 101 | take_time = round(end_time - start_time, 3) 102 | self.logging.debug( 103 | '\n outputerer ---pid: %s tid: %s reg data len:%s; take time : %s \n ' % 104 | (os.getpid(), threading.get_ident(), len(_data), take_time)) 105 | 106 | 107 | if 'max_retry_reconnect_time' in self.conf['outputer']: 108 | max_retry_reconnect_time = int(self.conf['outputer']['max_retry_reconnect_time']) 109 | else: 110 | max_retry_reconnect_time = 3 111 | 112 | # 解析完成 批量入库 113 | try: 114 | start_time = time.perf_counter() 115 | 116 | if len(_data) == 0: 117 | continue 118 | 119 | # for reconnect 120 | self.db.ping() 121 | 122 | affected_rows = self.__insertToMysql(_data) 123 | 124 | # reset retry_reconnect_time 125 | retry_reconnect_time = 0 126 | 127 | end_time = time.perf_counter() 128 | self.logging.debug("\n outputerer -------pid: %s -- insert into mysql : %s---- end 耗时: %s \n" % ( 129 | os.getpid(), affected_rows, round(end_time - start_time, 3))) 130 | 131 | 132 | except pymysql.err.DataError as e: 133 | 134 | # (1406, "Data too long for column 'isp' at row 2") 135 | if e.args[0] == 1406 : 136 | wrong_field = re.findall(r'\'(\w+)\'' ,e.args[1]) 137 | if len(wrong_field) > 0: 138 | self.__changeFieldTypeReInsert(wrong_field,_data) 139 | 140 | else: 141 | self.__saveWrongData(_data ,e) 142 | continue 143 | 144 | except pymysql.err.MySQLError as e: 145 | time.sleep(2) 146 | retry_reconnect_time = retry_reconnect_time + 1 147 | if retry_reconnect_time >= max_retry_reconnect_time: 148 | self.runner.rollBackQueue(self.backup_for_push_back_queue) 149 | self.logging.error('重试重新链接 mongodb 超出最大次数 %s' % max_retry_reconnect_time) 150 | raise Exception('重试重新链接 mongodb 超出最大次数 %s' % max_retry_reconnect_time) 151 | else: 152 | 153 | self.logging.error("\n outputerer -------pid: %s -- retry_reconnect_mysql at: %s time---- Exceptions %s ; %s \n" % ( 154 | os.getpid(),retry_reconnect_time,e.__class__ ,e.args)) 155 | continue 156 | 157 | 158 | 159 | # 根据数据 和 nginx 解析器中的format 创建 mysql 字段类型的映射 160 | def build_field_map(self,example_data): 161 | field_map = {} 162 | key_field_map = {} 163 | 164 | # 开始组装 mysql字典 165 | data_key = list(example_data.keys()) 166 | 167 | for i in self.runner.logParse.format: 168 | format_key = i.replace('$', '') 169 | 170 | # 检查默认值是否在数据中 171 | if format_key in data_key: 172 | 173 | if 'mysql_key_field' in self.runner.logParse.format[i]: 174 | key_field_map[format_key] = self.runner.logParse.format[i]['mysql_key_field'] 175 | 176 | if 'mysql_field_type' in self.runner.logParse.format[i]: 177 | field_map[format_key] = self.runner.logParse.format[i]['mysql_field_type'] 178 | else: 179 | field_map[format_key] = 'varchar(255)' 180 | 181 | # 检查 nickname 是否在数据中 182 | elif 'nickname' in self.runner.logParse.format[i] \ 183 | and self.runner.logParse.format[i]['nickname'] in data_key: 184 | 185 | if 'mysql_key_field' in self.runner.logParse.format[i]: 186 | key_field_map[ self.runner.logParse.format[i]['nickname'] ] = self.runner.logParse.format[i]['mysql_key_field'] 187 | 188 | if 'mysql_field_type' in self.runner.logParse.format[i]: 189 | field_map[ self.runner.logParse.format[i]['nickname'] ] = self.runner.logParse.format[i]['mysql_field_type'] 190 | else: 191 | field_map[ self.runner.logParse.format[i]['nickname'] ] = 'varchar(255)' 192 | 193 | 194 | # 检查 extend_field 是否在数据中 195 | if 'extend_field' in self.runner.logParse.format[i]: 196 | _intersection = set(data_key).intersection( 197 | set(list(self.runner.logParse.format[i]['extend_field'].keys()))) 198 | 199 | if len(_intersection): 200 | for k in _intersection: 201 | 202 | if 'mysql_key_field' in self.runner.logParse.format[i]['extend_field'][k]: 203 | key_field_map[k] = self.runner.logParse.format[i]['extend_field'][k]['mysql_key_field'] 204 | 205 | if 'mysql_field_type' in self.runner.logParse.format[i]['extend_field'][k]: 206 | field_map[k] = self.runner.logParse.format[i]['extend_field'][k]['mysql_field_type'] 207 | else: 208 | field_map[k] = 'varchar(255)' 209 | 210 | 211 | return field_map , key_field_map 212 | 213 | def __insertToMysql(self,data): 214 | 215 | if self.field_map == None: 216 | self.field_map ,self.key_field_map = self.build_field_map(data[0]) 217 | 218 | try: 219 | fields = None 220 | 221 | _valuelist = [] 222 | for item in data: 223 | 224 | fk = list(item.keys()) 225 | fields = ','.join(fk) 226 | 227 | for i in item: 228 | 229 | field_type = self.field_map[i] 230 | 231 | if (field_type.find('int') > -1 or field_type.find('float') > -1) and str(item[i]) == '': 232 | item[i] = '"0"' 233 | elif str(item[i]) == '': 234 | item[i] = '"-"' 235 | else: 236 | item[i] = '"%s"' % str(item[i]).strip('"') 237 | 238 | values = '(%s)' % ','.join(list(item.values())) 239 | _valuelist.append(values) 240 | except KeyError as e: 241 | self.runner.logging.error('Exception: %s ; 数据写入错误: %s 请检查 ParserAdapter 中的 getLogFormat 配置' % (e.__class__, e.args)) 242 | raise Exception(' Exception: %s ; 数据写入错误: %s ;请检查 ParserAdapter 中的 getLogFormat 配置' % (e.__class__, e.args)) 243 | 244 | 245 | sql = "INSERT INTO %s(%s) VALUES %s" % (self.table,fields,','.join(_valuelist)) 246 | 247 | self.debug_sql = sql 248 | 249 | try: 250 | with self.db.cursor() as cursor: 251 | affected_rows = cursor.execute(sql) 252 | 253 | self.db.commit() 254 | 255 | return affected_rows 256 | # when table not found 257 | except pymysql.err.ProgrammingError as e: 258 | create_table_flag = self._handle_queue_data_before_into_storage(self.backup_for_push_back_queue) 259 | # 创建表的时候 补插入数据 260 | if create_table_flag == True: 261 | 262 | with self.db.cursor() as cursor: 263 | affected_rows = cursor.execute(sql) 264 | self.db.commit() 265 | return affected_rows 266 | # 数据表存在的 其它错误 267 | else: 268 | self.runner.logging.error('Exception: %s ; %s 数据写入错误: %s ;sql: %s' % (e.__class__,e.args , self.debug_sql)) 269 | raise Exception(' Exception: %s ; 数据写入错误: %s ;sql: %s' % (e.__class__,e.args , self.debug_sql)) 270 | 271 | 272 | def __changeFieldTypeReInsert(self,wrong_field,data): 273 | 274 | 275 | 276 | for f in wrong_field: 277 | data_len_arg = [] 278 | for item in data: 279 | data_len_arg.append( len(item[f]) ) 280 | 281 | 282 | try: 283 | # varchar 长度 取最大的一个 284 | sql = 'ALTER TABLE `%s`.`%s` MODIFY COLUMN `%s` varchar(%s) NOT NULL' % \ 285 | (self.conf['mysql']['db'], self.table, f, sorted(data_len_arg)[-1]) 286 | with self.db.cursor() as cursor: 287 | cursor.execute(sql) 288 | 289 | except pymysql.err.OperationalError as e: 290 | # 字段太长导致无法加索引 Specified key was too long; max key length is 3072 bytes 291 | # 字段太长导致无法存储 Column length too big for column 'request_url' (max = 16383); use BLOB or TEXT instead 292 | if e.args[0] in [1071,1074] : 293 | if e.args[0] == 1071: 294 | ftype = 'text' 295 | if e.args[0] == 1074: 296 | ftype = 'mediumtext' 297 | 298 | # 该字段有索引 则删除 299 | table_keys = self.__getTableKeys() 300 | for k in table_keys: 301 | if k.find(f) > -1: 302 | key_name = k.split(' ')[1].strip('`') 303 | drop_index = 'ALTER TABLE `%s`.`%s` DROP INDEX `%s`' % (self.conf['mysql']['db'], self.table ,key_name) 304 | try: 305 | with self.db.cursor() as cursor: 306 | cursor.execute(drop_index) 307 | except pymysql.err.OperationalError: 308 | continue 309 | 310 | 311 | # 重建字段类型 text 312 | sql = 'ALTER TABLE `%s`.`%s` MODIFY COLUMN `%s` %s NOT NULL' % (self.conf['mysql']['db'], self.table, f , ftype) 313 | with self.db.cursor() as cursor: 314 | cursor.execute(sql) 315 | 316 | 317 | 318 | 319 | 320 | try: 321 | self.__insertToMysql(data) 322 | except pymysql.err.DataError as e: 323 | self.__saveWrongData(data ,e) 324 | 325 | 326 | 327 | def __saveWrongData(self,data ,e): 328 | # 写入错误的数据 输出成json文件以供分析 329 | error_json_file_dir = self.runner._root + '/error_insert_data/%s' % self.runner.config_name.replace( 330 | '.ini', '') 331 | error_json_file = error_json_file_dir + '/%s_pid_%s.json' % ( 332 | time.strftime('%Y_%m_%d_%H:%M:%S.%s', time.localtime()), os.getpid()) 333 | error_msg = "\n outputerer -------pid: %s -- pymysql.err.DataError 数据类型错误 请检查 field_map 配置---- Exceptions: %s ;异常数据已保存在 %s \n" % ( 334 | os.getpid(), e.args, error_json_file) 335 | self.logging.error(error_msg) 336 | 337 | if not os.path.exists(error_json_file_dir): 338 | os.makedirs(error_json_file_dir) 339 | 340 | if not os.path.exists(error_json_file): 341 | with open(error_json_file, 'w+') as fd: 342 | json.dump(data, fd) 343 | fd.close() 344 | 345 | 346 | 347 | def getKeyFieldStrForCreateTableFromList(self,key_field_needed ,i): 348 | 349 | def func(vars,i,re_field = False): 350 | _list = [] 351 | _key_field_name = [] 352 | 353 | if vars.find('.') > -1: 354 | _key = vars.split('.') 355 | if _key[1] in self.runner.logParse.format[_key[0]]['extend_field']: 356 | _list.append('KEY `{0}_{1}` (`{0}`,`{1}`)'.format(i, _key[1])) 357 | _key_field_name = _key[1] 358 | else: 359 | error_str = 'self.runner.logParse.format[%s] 不存在 "%s" 属性; 请检查解析中的logformat 配置' % (_key[0], _key[1]) 360 | self.runner.logging.error(error_str) 361 | raise KeyError(error_str) 362 | else: 363 | if 'nickname' in self.runner.logParse.format[vars]: 364 | field_name = self.runner.logParse.format[vars]['nickname'] 365 | else: 366 | field_name = vars.replace('$', '') 367 | 368 | _key_field_name = field_name 369 | _list.append('KEY `{0}_{1}` (`{0}`,`{1}`)'.format(i, field_name)) 370 | 371 | if re_field: 372 | return _key_field_name 373 | 374 | return _list 375 | 376 | karg = [] 377 | 378 | for args in key_field_needed[i]: 379 | 380 | if isinstance(args, str): 381 | 382 | karg = karg + func(args,i) 383 | 384 | elif isinstance(args, list): 385 | key_str = [i] 386 | 387 | for g in args: 388 | key_str.append(func(g,i,True)) 389 | 390 | karg.append('KEY %s (%s)' % ('_'.join(key_str), ','.join(key_str))) 391 | 392 | 393 | return karg 394 | 395 | def __getTableKeys(self): 396 | # 从字段中获取需要创建索引的 字段 397 | key_field_needed = self.key_field_map 398 | 399 | karg = [] 400 | if len(key_field_needed): 401 | 402 | for i in key_field_needed: 403 | 404 | if isinstance(key_field_needed[i], str): # 字符串 405 | karg.append('{0} `{1}` (`{1}`)'.format(key_field_needed[i].upper(), i)) 406 | elif isinstance(key_field_needed[i], bool): 407 | karg.append('KEY `{0}` (`{0}`)'.format(i)) 408 | elif isinstance(key_field_needed[i], list): 409 | karg.append('KEY `{0}` (`{0}`)'.format(i)) 410 | karg = karg + self.getKeyFieldStrForCreateTableFromList(key_field_needed, i) 411 | # 去重 412 | return list(set(karg)) 413 | 414 | def __createTable(self,org_data): 415 | 416 | if len(org_data) > 0: 417 | 418 | fields = [] 419 | for i in self.field_map: 420 | _str = "`%s` %s NOT NULL " % (i ,self.field_map[i] ) 421 | fields.append(_str) 422 | 423 | # 从字段中获取需要创建索引的 字段 424 | key_field_needed = self.key_field_map 425 | 426 | key_str = '' 427 | keys_arr = self.__getTableKeys() 428 | if len(keys_arr): 429 | key_str = ',' + ','.join(keys_arr) 430 | 431 | 432 | sql = """ 433 | CREATE TABLE IF NOT EXISTS `%s`.`%s` ( 434 | `id` int(11) NOT NULL AUTO_INCREMENT, 435 | %s , 436 | PRIMARY KEY (`id`) 437 | %s 438 | ) 439 | """ % (self.conf['mysql']['db'], self.table ,','.join(fields),key_str) 440 | 441 | 442 | try: 443 | with self.db.cursor() as cursor: 444 | cursor.execute(sql) 445 | except pymysql.MySQLError as e: 446 | 447 | self.logging.error('数据表 %s.%s 创建失败 ;Exception: %s ; SQL:%s' % (self.conf['mysql']['db'], self.conf['mysql']['table'] , e.args ,sql)) 448 | raise Exception('数据表 %s.%s 创建失败 ;Exception: %s ; SQL:%s' % (self.conf['mysql']['db'], self.conf['mysql']['table'], e.args ,sql)) 449 | 450 | 451 | def __checkTableExist(self): 452 | sql = "SELECT table_name FROM information_schema.TABLES WHERE table_name ='%s'" % self.table; 453 | with self.db.cursor() as cursor: 454 | cursor.execute(sql) 455 | res = cursor.fetchone() 456 | 457 | return res 458 | 459 | # 检查table　是否存在 460 | def _handle_queue_data_before_into_storage(self ,org_data): 461 | 462 | res = self.__checkTableExist() 463 | 464 | if not res: 465 | self.logging.warn('没有发现数据表,开始创建数据表') 466 | self.__createTable(org_data) 467 | return True 468 | 469 | 470 | return False 471 | 472 | # 在持久化存储之前 对 队列中的数据 进行预处理 ,比如 update ,delete 等操作 473 | def _handle_queue_data_after_into_storage(self): 474 | pass 475 | 476 | 477 | 478 | 479 | # 流量分析 480 | class TrafficAnalysisAp(StorageAp): 481 | 482 | def analysisTraffic(self): 483 | table_exist = self.__checkTableExist() 484 | 485 | print(table_exist) 486 | pass -------------------------------------------------------------------------------- /webServer/static/js/index.js: -------------------------------------------------------------------------------- 1 | /*********************left start********************************/ 2 | 3 | // 请求量最高的IP TOP50 ------------------------------- start ------------------------------- 4 | var top_ip_chart = echarts.init(document.querySelector(".bar .chart")); 5 | window.addEventListener("resize", function () { 6 | top_ip_chart.resize(); 7 | }); 8 | window.chart_load_func['top_ip_chart'] = function () { 9 | 10 | $.ajax({ 11 | url: host + '/get_request_num_by_ip', 12 | type:'GET', 13 | async:true, 14 | headers: { 'Authorization':Authorization,}, 15 | success:function(msg){ 16 | data = msg.data 17 | var xdata = [] 18 | var ydata = [] 19 | for (var i=0; i< data.length ;i++){ 20 | xdata.push(data[i]['ip']) 21 | ydata.push(data[i]['total_num']) 22 | } 23 | // 指定配置和数据 24 | var option = { 25 | color: ["#2f89cf"], 26 | tooltip: { 27 | trigger: "axis", 28 | axisPointer: { 29 | // 坐标轴指示器，坐标轴触发有效 30 | type: "line" // 默认为直线，可选为：'line' | 'shadow' 31 | } 32 | }, 33 | grid: { 34 | left: "0%", 35 | top: "10px", 36 | right: "0%", 37 | bottom: "4%", 38 | containLabel: true 39 | }, 40 | xAxis: [ 41 | { 42 | type: "category", 43 | data: xdata, 44 | axisTick: { 45 | alignWithLabel: true 46 | }, 47 | axisLabel: { 48 | textStyle: { 49 | color: "rgba(255,255,255,.6)", 50 | fontSize: "12" 51 | } 52 | }, 53 | axisLine: { 54 | show: true 55 | } 56 | } 57 | ], 58 | yAxis: [ 59 | { 60 | type: "value", 61 | axisLabel: { 62 | textStyle: { 63 | color: "rgba(255,255,255,.6)", 64 | fontSize: "12" 65 | } 66 | }, 67 | axisLine: { 68 | lineStyle: { 69 | color: "rgba(255,255,255,.1)" 70 | // width: 1, 71 | // type: "solid" 72 | } 73 | }, 74 | splitLine: { 75 | lineStyle: { 76 | color: "rgba(255,255,255,.1)" 77 | } 78 | } 79 | } 80 | ], 81 | series: [ 82 | { 83 | name: "IP", 84 | type: "bar", 85 | barWidth: "35%", 86 | // data: [200, 300, 300, 900, 1500, 1200, 600], 87 | data: ydata, 88 | itemStyle: { 89 | barBorderRadius: 5 90 | } 91 | } 92 | ] 93 | }; 94 | 95 | // 把配置给实例对象 96 | top_ip_chart.setOption(option); 97 | } 98 | }) 99 | 100 | 101 | 102 | } 103 | // 请求量最高的IP TOP50 ------------------------------- end ------------------------------- 104 | 105 | 106 | // 最近10分钟每分钟流量 ------------------------------- start ------------------------------- 107 | var network_traffic_by_minute = echarts.init(document.querySelector(".line .chart")); 108 | window.addEventListener("resize", function () { 109 | network_traffic_by_minute.resize(); 110 | }); 111 | window.chart_load_func['network_traffic_by_minute'] = function (type = null) { 112 | if(type == 'init'){ 113 | __url = host + '/get_network_traffic_by_minute?type=init' 114 | }else { 115 | __url = host + '/get_network_traffic_by_minute' 116 | } 117 | $.ajax({ 118 | url:__url, 119 | type:'GET', 120 | async:true, 121 | headers: { 'Authorization':Authorization,}, 122 | success:function(msg){ 123 | data = msg.data 124 | 125 | secends_values = { 126 | 'in_network':[], 127 | 'out_network':[], 128 | 'xAxis':[] 129 | } 130 | secends_xAxis = [] 131 | $.each(data,function(k,v){ 132 | secends_values['in_network'].push(Math.round(v['in_network'],0)) 133 | secends_values['out_network'].push(Math.round(v['out_network'],0)) 134 | secends_values['xAxis'].push(timestampToTime(v['time_str'])) 135 | }) 136 | 137 | // 2. 指定配置和数据 138 | var option = { 139 | color: ["#00f2f1", "#ed3f35"], 140 | tooltip: { 141 | // 通过坐标轴来触发 142 | trigger: "axis" 143 | }, 144 | legend: { 145 | // 距离容器10% 146 | right: "10%", 147 | // 修饰图例文字的颜色 148 | textStyle: { 149 | color: "#4c9bfd" 150 | } 151 | // 如果series 里面设置了name，此时图例组件的data可以省略 152 | // data: ["邮件营销", "联盟广告"] 153 | }, 154 | grid: { 155 | top: "20%", 156 | left: "3%", 157 | right: "4%", 158 | bottom: "3%", 159 | show: true, 160 | borderColor: "#012f4a", 161 | containLabel: true 162 | }, 163 | 164 | xAxis: { 165 | type: "category", 166 | boundaryGap: false, 167 | data: secends_values['xAxis'], 168 | // 去除刻度 169 | axisTick: { 170 | show: false 171 | }, 172 | // 修饰刻度标签的颜色 173 | axisLabel: { 174 | color: "rgba(255,255,255,.7)" 175 | }, 176 | // 去除x坐标轴的颜色 177 | axisLine: { 178 | show: false 179 | } 180 | }, 181 | yAxis: { 182 | type: "value", 183 | // 去除刻度 184 | axisTick: { 185 | show: false 186 | }, 187 | // 修饰刻度标签的颜色 188 | axisLabel: { 189 | color: "rgba(255,255,255,.7)" 190 | }, 191 | // 修改y轴分割线的颜色 192 | splitLine: { 193 | lineStyle: { 194 | color: "#012f4a" 195 | } 196 | } 197 | }, 198 | series: [ 199 | { 200 | name:'入网 KB', 201 | type: "line", 202 | // stack: "总量", 203 | smooth: true, 204 | data: secends_values['in_network'] 205 | }, 206 | { 207 | name:'出网 KB', 208 | type: "line", 209 | // stack: "总量", 210 | smooth: true, 211 | data: secends_values['out_network'] 212 | } 213 | ] 214 | }; 215 | // 3. 把配置和数据给实例对象 216 | network_traffic_by_minute.setOption(option); 217 | } 218 | 219 | 220 | }) 221 | 222 | 223 | 224 | } 225 | // 最近10分钟pv ------------------------------- end ------------------------------- 226 | 227 | 228 | // 非200状态码 二级分类 ------------------------------- start ------------------------------- 229 | var status_code_chart = echarts.init(document.querySelector(".bar1 .chart")); 230 | window.addEventListener("resize", function () { 231 | status_code_chart.resize(); 232 | }); 233 | window.chart_load_func['status_code_chart'] = function () { 234 | $.ajax({ 235 | url: host + '/get_request_num_by_status', 236 | type:'GET', 237 | async:true, 238 | headers: { 'Authorization':Authorization,}, 239 | success:function(msg){ 240 | var data = []; 241 | var titlename = []; 242 | var valdata = []; 243 | var maxRang = [] 244 | var total = 0 245 | $.each(msg.data ,function(k,v){ 246 | total = total + v.total_num 247 | valdata.push(v.total_num) 248 | titlename.push('状态码：' + v.status_code.toString()) 249 | }) 250 | 251 | $.each(msg.data ,function(k,v){ 252 | percent = (v.total_num / total) * 100 253 | data.push(percent.toFixed(2)) 254 | maxRang.push(100) 255 | }) 256 | 257 | 258 | 259 | var myColor = ["#1089E7", "#F57474", "#56D0E3", "#F8B448", "#8B78F6"]; 260 | option = { 261 | //图标位置 262 | grid: { 263 | top: "10%", 264 | left: "22%", 265 | bottom: "10%" 266 | }, 267 | xAxis: { 268 | show: false 269 | }, 270 | yAxis: [ 271 | { 272 | show: true, 273 | data: titlename, 274 | inverse: true, 275 | axisLine: { 276 | show: false 277 | }, 278 | splitLine: { 279 | show: false 280 | }, 281 | axisTick: { 282 | show: false 283 | }, 284 | axisLabel: { 285 | color: "#fff", 286 | rich: { 287 | lg: { 288 | backgroundColor: "#339911", 289 | color: "#fff", 290 | borderRadius: 15, 291 | // padding: 5, 292 | align: "center", 293 | width: 15, 294 | height: 15 295 | } 296 | } 297 | } 298 | }, 299 | { 300 | show: true, 301 | inverse: true, 302 | data: valdata, 303 | axisLabel: { 304 | textStyle: { 305 | fontSize: 12, 306 | color: "#fff" 307 | } 308 | } 309 | } 310 | ], 311 | series: [ 312 | { 313 | name: "条", 314 | type: "bar", 315 | yAxisIndex: 0, 316 | data: data, 317 | barCategoryGap: 50, 318 | barWidth: 10, 319 | itemStyle: { 320 | normal: { 321 | barBorderRadius: 20, 322 | color: function (params) { 323 | var num = myColor.length; 324 | return myColor[params.dataIndex % num]; 325 | } 326 | } 327 | }, 328 | label: { 329 | normal: { 330 | show: true, 331 | position: "right", 332 | formatter: "{c}%" 333 | } 334 | } 335 | }, 336 | { 337 | name: "框", 338 | type: "bar", 339 | yAxisIndex: 1, 340 | barCategoryGap: 50, 341 | data: maxRang, 342 | barWidth: 15, 343 | itemStyle: { 344 | normal: { 345 | color: "none", 346 | borderColor: "#00c1de", 347 | borderWidth: 3, 348 | barBorderRadius: 15 349 | } 350 | } 351 | } 352 | ] 353 | }; 354 | 355 | // 使用刚指定的配置项和数据显示图表。 356 | status_code_chart.setOption(option); 357 | 358 | 359 | } 360 | 361 | }) 362 | } 363 | // 非200状态码 二级分类 ------------------------------- end ------------------------------- 364 | 365 | 366 | // 最近10分钟IP / PV ------------------------------- start ------------------------------- 367 | var request_ip_pv_by_minute = echarts.init(document.querySelector(".member .chart")); 368 | window.addEventListener("resize", function () { 369 | request_ip_pv_by_minute.resize(); 370 | }); 371 | window.chart_load_func['request_ip_pv_by_minute'] = function () { 372 | $.ajax({ 373 | url: host + '/get_ip_pv_num_by_minute', 374 | type:'GET', 375 | async:true, 376 | headers: { 'Authorization':Authorization,}, 377 | success:function(msg){ 378 | data = msg.data 379 | 380 | 381 | nums = { 382 | 'ip_num':[], 383 | 'pv_num':[] 384 | } 385 | timestamp = [] 386 | $.each(data,function(k,v){ 387 | 388 | nums['ip_num'].push(v['ip_num']) 389 | nums['pv_num'].push(v['pv_num']) 390 | timestamp.push(timestampToTime(v['time_str'])) 391 | }) 392 | 393 | 394 | 395 | option = { 396 | tooltip: { 397 | trigger: "axis", 398 | axisPointer: { 399 | lineStyle: { 400 | color: "#dddc6b" 401 | } 402 | } 403 | }, 404 | legend: { 405 | top: "0%", 406 | textStyle: { 407 | color: "rgba(255,255,255,.5)", 408 | fontSize: "12" 409 | } 410 | }, 411 | grid: { 412 | left: "10", 413 | top: "30", 414 | right: "10", 415 | bottom: "10", 416 | containLabel: true 417 | }, 418 | 419 | xAxis: [ 420 | { 421 | type: "category", 422 | boundaryGap: false, 423 | axisLabel: { 424 | textStyle: { 425 | color: "rgba(255,255,255,.6)", 426 | fontSize: 12 427 | } 428 | }, 429 | axisLine: { 430 | lineStyle: { 431 | color: "rgba(255,255,255,.2)" 432 | } 433 | }, 434 | 435 | data: timestamp 436 | }, 437 | { 438 | axisPointer: { show: false }, 439 | axisLine: { show: false }, 440 | position: "bottom", 441 | offset: 20 442 | } 443 | ], 444 | yAxis: [ 445 | { 446 | type: "value", 447 | axisTick: { show: false }, 448 | axisLine: { 449 | lineStyle: { 450 | color: "rgba(255,255,255,.1)" 451 | } 452 | }, 453 | axisLabel: { 454 | textStyle: { 455 | color: "rgba(255,255,255,.6)", 456 | fontSize: 12 457 | } 458 | }, 459 | 460 | splitLine: { 461 | lineStyle: { 462 | color: "rgba(255,255,255,.1)" 463 | } 464 | } 465 | } 466 | ], 467 | series: [ 468 | { 469 | name: "每分钟IP数", 470 | type: "line", 471 | smooth: true, 472 | symbol: "circle", 473 | symbolSize: 5, 474 | showSymbol: true, 475 | lineStyle: { 476 | normal: { 477 | color: "#00d887", 478 | width: 2 479 | } 480 | }, 481 | areaStyle: { 482 | normal: { 483 | color: new echarts.graphic.LinearGradient( 484 | 0, 485 | 0, 486 | 0, 487 | 1, 488 | [ 489 | { 490 | offset: 0, 491 | color: "rgba(0, 216, 135, 0.4)" 492 | }, 493 | { 494 | offset: 0.8, 495 | color: "rgba(0, 216, 135, 0.1)" 496 | } 497 | ], 498 | false 499 | ), 500 | shadowColor: "rgba(0, 0, 0, 0.1)" 501 | } 502 | }, 503 | itemStyle: { 504 | normal: { 505 | color: "#00d887", 506 | borderColor: "rgba(221, 220, 107, .1)", 507 | borderWidth: 12 508 | } 509 | }, 510 | data: nums['ip_num'] 511 | }, 512 | { 513 | name: "每分钟PV数", 514 | type: "line", 515 | smooth: true, 516 | symbol: "circle", 517 | symbolSize: 5, 518 | showSymbol: true, 519 | lineStyle: { 520 | normal: { 521 | color: "#006cff", 522 | width: 2 523 | } 524 | }, 525 | areaStyle: { 526 | normal: { 527 | color: new echarts.graphic.LinearGradient( 528 | 0, 529 | 0, 530 | 0, 531 | 1, 532 | [ 533 | { 534 | offset: 0, 535 | color: "rgba(0, 216, 135, 0.4)" 536 | }, 537 | { 538 | offset: 0.8, 539 | color: "rgba(0, 216, 135, 0.1)" 540 | } 541 | ], 542 | false 543 | ), 544 | shadowColor: "rgba(0, 0, 0, 0.1)" 545 | } 546 | }, 547 | itemStyle: { 548 | normal: { 549 | color: "#006cff", 550 | borderColor: "rgba(221, 220, 107, .1)", 551 | borderWidth: 12 552 | } 553 | }, 554 | data: nums['pv_num'] 555 | } 556 | ] 557 | }; 558 | // 使用刚指定的配置项和数据显示图表。 559 | request_ip_pv_by_minute.setOption(option); 560 | } 561 | 562 | }) 563 | } 564 | // 最近10分钟IP ------------------------------- end ------------------------------- 565 | 566 | 567 | // 热门接口URL请求TOP 10分布 ------------------------------- start ------------------------------- 568 | var request_num_by_url = echarts.init(document.querySelector(".pie .chart")); 569 | window.addEventListener("resize", function () { 570 | request_num_by_url.resize(); 571 | }); 572 | window.chart_load_func['request_num_by_url'] = function () { 573 | $.ajax({ 574 | url: host + '/get_request_num_by_url', 575 | type:'GET', 576 | async:true, 577 | headers: { 'Authorization':Authorization,}, 578 | success:function(msg){ 579 | data = msg.data 580 | range_keys = [] 581 | range_values = [] 582 | $.each(data ,function(k,v){ 583 | range_keys.push(v['request_url']) 584 | range_values.push({value:v['total_num'] ,name:v['request_url']}) 585 | }) 586 | option = { 587 | tooltip: { 588 | trigger: "item", 589 | formatter: "{a}
{b}: {c} ({d}%)", 590 | position: function (p) { 591 | //其中p为当前鼠标的位置 592 | return [p[0] + 10, p[1] - 10]; 593 | } 594 | }, 595 | legend: { 596 | top: "90%", 597 | itemWidth: 10, 598 | itemHeight: 10, 599 | data: range_keys, 600 | textStyle: { 601 | color: "rgba(255,255,255,.5)", 602 | fontSize: "12" 603 | } 604 | }, 605 | series: [ 606 | { 607 | name: "接口分布", 608 | type: "pie", 609 | center: ["50%", "42%"], 610 | radius: ["40%", "60%"], 611 | color: [ 612 | "#006cff", 613 | "#60cda0", 614 | "#ed8884", 615 | "#ff9f7f", 616 | "#0096ff", 617 | "#9fe6b8", 618 | "#32c5e9", 619 | "#1d9dff" 620 | ], 621 | label: { show: true }, 622 | labelLine: { show: true }, 623 | data:range_values 624 | } 625 | ] 626 | }; 627 | 628 | // 使用刚指定的配置项和数据显示图表。 629 | request_num_by_url.setOption(option); 630 | 631 | 632 | } 633 | 634 | }) 635 | } 636 | // 热门接口URL请求TOP 10 ------------------------------- end ------------------------------- 637 | 638 | 639 | // 搜索引擎蜘蛛占比 ------------------------------- start ------------------------------- 640 | var spider_by_ua = echarts.init(document.querySelector(".pie1 .chart")); 641 | window.addEventListener("resize", function () { 642 | spider_by_ua.resize(); 643 | }); 644 | window.chart_load_func['spider_by_ua'] = function () { 645 | $.ajax({ 646 | url: host + '/get_spider_by_ua', 647 | type:'GET', 648 | async:true, 649 | headers: { 'Authorization':Authorization,}, 650 | success:function(msg){ 651 | data = msg.data 652 | 653 | company_data = [ ] 654 | 655 | $.each(data,function(k,v){ 656 | company_data.push({value:v['total_num'] , name:v['ua']}) 657 | }) 658 | 659 | var option = { 660 | legend: { 661 | top: "90%", 662 | itemWidth: 10, 663 | itemHeight: 10, 664 | textStyle: { 665 | color: "rgba(255,255,255,.5)", 666 | fontSize: "12" 667 | } 668 | }, 669 | tooltip: { 670 | trigger: "item", 671 | formatter: "{a}
{b} : {c} ({d}%)" 672 | }, 673 | // 注意颜色写的位置 674 | color: [ 675 | "#006cff", 676 | "#60cda0", 677 | "#ed8884", 678 | "#ff9f7f", 679 | "#0096ff", 680 | "#9fe6b8", 681 | "#32c5e9", 682 | "#1d9dff" 683 | ], 684 | series: [ 685 | { 686 | name: "前十占比", 687 | type: "pie", 688 | // 如果radius是百分比则必须加引号 689 | radius: ["10%", "70%"], 690 | center: ["50%", "42%"], 691 | // roseType: "radius", 692 | data: company_data, 693 | // 修饰饼形图文字相关的样式 label对象 694 | label: { 695 | fontSize: 10 696 | }, 697 | // 修饰引导线样式 698 | labelLine: { 699 | // 连接到图形的线长度 700 | length: 10, 701 | // 连接到文字的线长度 702 | length2: 10 703 | } 704 | } 705 | ] 706 | }; 707 | // 3. 配置项和数据给我们的实例化对象 708 | spider_by_ua.setOption(option); 709 | } 710 | }) 711 | 712 | } 713 | // 搜索引擎蜘蛛占比 ------------------------------- end ------------------------------- --------------------------------------------------------------------------------