├── README.md ├── activemq └── activemq_mon.py ├── canal-monitor ├── canal-monitor.py ├── falcon.py └── zookeeper.py ├── cost-monitor └── cost_monitor.sh ├── ip-ping ├── bin │ ├── ip-ping.sh │ └── ping-check.sh └── conf │ └── list-inner ├── keepalived └── check_activemq.sh ├── logstash-conf └── mysql-slow-query.conf ├── msgget-monitor ├── msgget-monitor-client │ ├── config.go │ ├── config.yaml │ ├── control │ ├── falcon.go │ ├── main.go │ ├── re.go │ ├── redis.go │ ├── set.go │ └── tail.go └── msgget-monitor-server │ ├── config.go │ ├── config.yaml │ ├── control │ ├── falcon.go │ ├── main.go │ └── redis.go ├── mysqlmon ├── falcon.go ├── instance.go ├── instance.yaml └── main.go ├── oom-checker ├── checker.sh ├── collector.awk └── collector.sh ├── order-count ├── order_count.conf └── order_count.sh ├── proc-killer └── proc-killer.sh ├── redismon ├── check.go ├── config.go ├── config.json ├── database.go ├── falcon.go └── main.go ├── request ├── request.json └── request.py └── zkmon └── zkmon.sh /README.md: -------------------------------------------------------------------------------- 1 | # Falcon Plugins Collection 2 | 这个项目收集了一些日常工作中写的监控插件和脚本 3 | 4 | - activemq: ActiveMQ监控插件，在官方代码上稍作修改 5 | - cost-monitor: 业务日志监控，提取日志中访问耗时，计算最大最小平均值并上报 6 | - ip-ping: Ping IP 监控脚本，监控网络状况 7 | - keepalived: Keepalived 对业务模块健康状况监控脚本 8 | - logstash-conf: logstash MySQL慢查询配置 9 | - msgget-monitor: nginx 日志请求和ID监控 10 | - mysqlmon: mysql 端口探活监控 11 | - oom-checker: Java业务模块OOM日志监控重启脚本 12 | - order-count: MySQL 数据定时查询上报脚本 13 | - proc-kiiler: 业务CPU使用率监控和停止脚本 14 | - request: Python实现对业务HTTP 健康检查脚本 15 | - zkmon: ZooKeeper 健康检查脚本 16 | - canal-monitor: Canal Server同步状态监控 17 | - redismon: redis状态监控 18 | -------------------------------------------------------------------------------- /activemq/activemq_mon.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | #-*- coding:utf-8 -*- 3 | # 4 | # Author: yezhiqin@hualala.com 5 | # Date : 2017/01/10 6 | 7 | import os 8 | import requests 9 | import xml.dom.minidom 10 | import time 11 | import json 12 | 13 | class ActivemqMonitor: 14 | def __init__(self): 15 | self.data = [] 16 | self.endpoint = os.uname()[1] 17 | self.metric = "activemq.consumerCount" 18 | self.step = 60 19 | self.counterType = "GAUGE" 20 | self.reqURL = "http://127.0.0.1:8161/admin/xml/queues.jsp" 21 | self.xmlString = "" 22 | self.xmlFile = "queues.xml" 23 | self.agentURL = "http://127.0.0.1:1988/v1/push" 24 | 25 | # function to get xml. 26 | def getXML(self, url, xmlFile, method="get"): 27 | try: 28 | response = requests.request(method, url) 29 | code = response.status_code 30 | if code == 200: 31 | self.xmlString = response.text 32 | f = open(xmlFile, 'w') 33 | f.write(response.text) 34 | f.close() 35 | return 0 36 | else: 37 | return 1 38 | except: 39 | return -1 40 | 41 | # TODO:: function to parse queues xml file. 42 | def processXML(self, xmlString): 43 | domTree = xml.dom.minidom.parseString(xmlString) 44 | queues = domTree.getElementsByTagName( "queue" ) 45 | for queue in queues: 46 | queueName = queue.getAttribute('name') 47 | consumerCount = int(queue.getElementsByTagName("stats")[0].getAttribute('consumerCount')) 48 | self.addPoint(queueName, consumerCount) 49 | 50 | def addPoint(self, queueName, consumerCount): 51 | point = { 52 | 'endpoint': self.endpoint, 53 | 'metric': self.metric, 54 | 'timestamp': int(time.time()), 55 | 'step': self.step, 56 | 'value': consumerCount, 57 | 'counterType': self.counterType, 58 | 'tags': "queuename=%s" % queueName 59 | } 60 | self.data.append(point) 61 | 62 | def push(self): 63 | response = requests.post(self.agentURL, json.dumps(self.data)) 64 | print(json.dumps(self.data)) 65 | 66 | def run(self): 67 | self.getXML(self.reqURL, self.xmlFile) 68 | self.processXML(self.xmlString) 69 | self.push() 70 | 71 | # start to check 72 | if __name__ == '__main__': 73 | monitor = ActivemqMonitor() 74 | monitor.run(); 75 | -------------------------------------------------------------------------------- /canal-monitor/canal-monitor.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | #-*- coding:utf-8 -*- 3 | # 4 | # Author: ye.zhiqin@outlook.com 5 | # Date : 2017/11/6 6 | 7 | import json 8 | import logging 9 | import shelve 10 | import time 11 | from falcon import Falcon 12 | from zookeeper import ZooKeeper 13 | 14 | logging.basicConfig(level=logging.INFO, filename='canal-monitor.log', filemode='a') 15 | 16 | now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) 17 | logging.info("=====Started: %s=====" % now) 18 | 19 | # init falcon object 20 | falcon = Falcon() 21 | 22 | # current timestamp 23 | tsNow = int(round(time.time() * 1000)) 24 | 25 | # read last timestamp from dump file 26 | tsRec = shelve.open('canal-monitor.db', writeback=True) 27 | 28 | # read data from zookeeper 29 | endpoint="127.0.0.1:2181" 30 | zookeeper = ZooKeeper(endpoint) 31 | zookeeper.connect() 32 | 33 | instances = zookeeper.listInstances(node='/otter/canal/destinations') 34 | for instance in instances: 35 | logging.info("-----%s-----" % instance) 36 | node = "/otter/canal/destinations/%s/1001/cursor" % instance 37 | cursor, stat = zookeeper.getCursor(node) 38 | if cursor is None or cursor == '': 39 | # status-3: no data in zookeeper 40 | logging.info("%s: no data in zookeeper" % instance) 41 | tags = "from=127.0.0.1,instance=%s" % str(instance) 42 | falcon.add(value=3, tags=tags) 43 | continue 44 | # dump cursor data 45 | cursorObj = json.loads(cursor) 46 | ts = cursorObj['postion']['timestamp'] 47 | if not tsRec.has_key(str(instance)): 48 | # status-1: first time to keep timestamp 49 | logging.info("%s: first time to keeper timestamp %d" % (instance, ts)) 50 | tsRec[str(instance)] = ts 51 | tags = "from=127.0.0.1,instance=%s" % str(instance) 52 | falcon.add(value=1, tags=tags) 53 | else: 54 | lastTs = tsRec[str(instance)] 55 | logging.info("Last TS:%d <---> This TS:%d" % (lastTs, ts)) 56 | if ts <= lastTs: 57 | # status-2: timestamp has no change 58 | logging.info("timestamp has no change") 59 | tags = "from=127.0.0.1,instance=%s" % str(instance) 60 | falcon.add(value=2, tags=tags) 61 | elif tsNow - ts > 3600000: 62 | # status-4: before one hour 63 | logging.info("before one hour") 64 | tags = "from=127.0.0.1,instance=%s" % str(instance) 65 | falcon.add(value=4, tags=tags) 66 | else: 67 | # status-0: normal 68 | logging.info("OK") 69 | tsRec[str(instance)] = ts 70 | tags = "from=127.0.0.1,instance=%s" % str(instance) 71 | falcon.add(value=0, tags=tags) 72 | 73 | # close zookeeper connection 74 | zookeeper.close() 75 | 76 | # close and write back dump file 77 | tsRec.close() 78 | 79 | # push monitor data to falcon 80 | falcon.push() 81 | -------------------------------------------------------------------------------- /canal-monitor/falcon.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | #-*- coding:utf-8 -*- 3 | # 4 | # Author: ye.zhiqin@outlook.com 5 | # Date : 2017/11/6 6 | 7 | import json 8 | import logging 9 | import requests 10 | import time 11 | 12 | class Falcon: 13 | def __init__(self, url="http://127.0.0.1:1988/v1/push"): 14 | logging.basicConfig() 15 | self.url=url 16 | self.points = [] 17 | 18 | def add(self, value, tags, metric="canal.status", endpoint="canal.app", step=900, counterType="GAUGE"): 19 | point = { 20 | 'endpoint': endpoint, 21 | 'metric': metric, 22 | 'timestamp': int(time.time()), 23 | 'step': step, 24 | 'value': value, 25 | 'counterType': counterType, 26 | 'tags': tags 27 | } 28 | self.points.append(point) 29 | 30 | def push(self): 31 | logging.info("push data to falcon: %s" % json.dumps(self.points)) 32 | response = requests.post(self.url, json.dumps(self.points), timeout=5) 33 | code = response.status_code 34 | text = response.text 35 | logging.info("push data to falcon, status code is %d and response text is %s" % (code,text)) 36 | -------------------------------------------------------------------------------- /canal-monitor/zookeeper.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | #-*- coding:utf-8 -*- 3 | # 4 | # Author: ye.zhiqin@outlook.com 5 | # Date : 2017/11/6 6 | 7 | import logging 8 | from kazoo.client import KazooClient 9 | from kazoo.retry import KazooRetry 10 | 11 | class ZooKeeper: 12 | def __init__(self, endpoint): 13 | logging.basicConfig() 14 | self.endpoint=endpoint 15 | self.zk=None 16 | 17 | def connect(self): 18 | retry = KazooRetry(max_tries=-1, max_delay=5) 19 | self.zk = KazooClient(hosts=self.endpoint, connection_retry=retry) 20 | self.zk.start() 21 | 22 | def close(self): 23 | self.zk.stop() 24 | 25 | def listInstances(self, node='/otter/canal/destinations'): 26 | if self.zk.exists(node): 27 | instances = self.zk.get_children(node) 28 | return instances 29 | else: 30 | return None 31 | 32 | def getCursor(self, node): 33 | if self.zk.exists(node): 34 | return self.zk.get(node) 35 | else: 36 | return (None, None) 37 | 38 | # __main__ 39 | if __name__=='__main__': 40 | endpoint="127.0.0.1:2181" 41 | zookeeper = ZooKeeper(endpoint) 42 | zookeeper.connect() 43 | instances = zookeeper.listInstances(node='/otter/canal/destinations') 44 | print instances 45 | cursor, stat = zookeeper.getCursor(node='/otter/canal/destinations/db_order/1001/cursor') 46 | print cursor 47 | print stat 48 | zookeeper.close() 49 | -------------------------------------------------------------------------------- /cost-monitor/cost_monitor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Author: yezhiqin@hualala.com 4 | # Date: 2017/3/13 5 | # Brief: 6 | # Get monitor data from log file. 7 | # Globals: 8 | # TS 9 | # HOSTNAME 10 | # TIMESTAMP 11 | # PATTERN 12 | # APP_LOG 13 | # APP_TMP_LOG 14 | # MAX 15 | # MIN 16 | # AVG 17 | # TIME 18 | # Arguments: 19 | # None 20 | # Return: 21 | # None 22 | 23 | # set 24 | 25 | # global variables 26 | readonly TS=$(date +%s -d "1 minute ago") 27 | readonly HOSTNAME=$(hostname) 28 | 29 | readonly TIMESTAMP=$(date '+%Y-%m-%d %H:%M:' -d '1 minute ago') 30 | readonly PATTERN="rpc cost [[0-9]*]ms" 31 | 32 | readonly APP_LOG=/path/to/app.log 33 | readonly APP_TMP_LOG=app.1m.log 34 | 35 | declare TIME=0 36 | 37 | # usage 38 | function usage() { 39 | echo "Usage: cost_monitor.sh" 40 | } 41 | 42 | ################################################# 43 | # Brief: 44 | # new day judgement. 45 | # Globals: 46 | # TIMESTAMP 47 | # Arguments: 48 | # None 49 | # Returns: 50 | # 0: date not changed 51 | # 1: date changed 52 | ################################################# 53 | function change_date() { 54 | if [[ "${TIMESTAMP}" =~ " 23:59:" ]];then 55 | return 1 56 | else 57 | return 0 58 | fi 59 | } 60 | 61 | ################################################# 62 | # Brief: 63 | # pick 1 minute log from log file. 64 | # Globals: 65 | # TIMESTAMP 66 | # URL 67 | # Arguments: 68 | # $1: in_file 69 | # $2: out_file 70 | # Returns: 71 | # None 72 | ################################################# 73 | function pick_log() { 74 | local in_file 75 | local out_file 76 | 77 | if [[ $# -ne 2 ]];then 78 | echo "Need 2 parameters!" 79 | exit 1 80 | else 81 | in_file="$1" 82 | out_file="$2" 83 | fi 84 | 85 | if [[ ! -f "${in_file}" ]];then 86 | echo "No such target log: ${in_file}" 87 | > "${out_file}" 88 | return 2 89 | fi 90 | 91 | grep "${TIMESTAMP}" "${in_file}" | grep -oE "${PATTERN}" > "${out_file}" 92 | } 93 | 94 | ################################################# 95 | # Brief: 96 | # calculate client cost. 97 | # Globals: 98 | # MAX 99 | # MIN 100 | # AVG 101 | # Arguments: 102 | # $1: log_file 103 | # Returns: 104 | # None 105 | ################################################# 106 | function calculate() { 107 | local log_file 108 | local result 109 | 110 | if [[ $# -ne 1 ]];then 111 | echo "Need 1 parameters!" 112 | exit 1 113 | else 114 | log_file="$1" 115 | fi 116 | 117 | if [[ -f ${log_file} && -s ${log_file} ]];then 118 | result=$(grep -oE "[0-9]*" ${log_file} | awk 'BEGIN{max=0;min=99999;avg=0;count=0;sum=0;} {count+=1;sum+=$1;if(max<$1)max=$1;if(min>$1)min=$1;} END{if(count!=0) avg=sum/count;print max" "min" "avg" "count" "sum}') 119 | MAX=$(echo "${result}" | awk '{print $1}') 120 | MIN=$(echo "${result}" | awk '{print $2}') 121 | AVG=$(echo "${result}" | awk '{print $3}') 122 | else 123 | MAX=0 124 | MIN=0 125 | AVG=0 126 | fi 127 | } 128 | 129 | ################################################# 130 | # Brief: 131 | # push data to falcon 132 | # Globals: 133 | # HOSTNAME 134 | # MAX 135 | # MIN 136 | # AVG 137 | # TS 138 | # TIME 139 | # Returns: 140 | # None 141 | ################################################# 142 | function push() { 143 | if [[ -z "${MAX}" ]];then 144 | echo "Max: None" 145 | else 146 | echo "Max: ${MAX}" 147 | curl -X POST -d "[{\"metric\": \"client.cost.max\", \"endpoint\": \"$HOSTNAME\", \"timestamp\": $TS,\"step\": 60,\"value\": $MAX,\"counterType\": \"GAUGE\",\"tags\": \"module=app,container=1\"}]" http://127.0.0.1:1988/v1/push 148 | fi 149 | 150 | if [[ -z "${MIN}" ]];then 151 | echo "Min: None" 152 | else 153 | echo "Min: ${MIN}" 154 | curl -X POST -d "[{\"metric\": \"client.cost.min\", \"endpoint\": \"$HOSTNAME\", \"timestamp\": $TS,\"step\": 60,\"value\": $MIN,\"counterType\": \"GAUGE\",\"tags\": \"module=app,container=1\"}]" http://127.0.0.1:1988/v1/push 155 | fi 156 | 157 | if [[ -z "${AVG}" ]];then 158 | echo "Avg: None" 159 | else 160 | echo "Avg: ${AVG}" 161 | curl -X POST -d "[{\"metric\": \"client.cost.avg\", \"endpoint\": \"$HOSTNAME\", \"timestamp\": $TS,\"step\": 60,\"value\": $AVG,\"counterType\": \"GAUGE\",\"tags\": \"module=app,container=1\"}]" http://127.0.0.1:1988/v1/push 162 | fi 163 | } 164 | 165 | # main 166 | function main() { 167 | local start 168 | local end 169 | local is_new_day 170 | 171 | start=$(date +%s) 172 | 173 | # new date judgement 174 | change_date 175 | is_new_day=$? 176 | 177 | if [[ 1 -eq "${is_new_day}" ]];then 178 | MAX=0 179 | MIN=0 180 | AVG=0 181 | else 182 | pick_log "${APP_LOG}" "${APP_TMP_LOG}" 183 | calculate "${APP_TMP_LOG}" 184 | fi 185 | 186 | end=$(date +%s) 187 | TIME=$((end-start)) 188 | 189 | # node push data to redis and falcon 190 | push 191 | } 192 | 193 | main "$@" 194 | -------------------------------------------------------------------------------- /ip-ping/bin/ip-ping.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # global variables 4 | readonly TS=$(date +%s) 5 | readonly HOSTNAME=$(hostname) 6 | 7 | if [[ $# -ne 2 ]];then 8 | echo "Parameter Error!" 9 | exit 1 10 | else 11 | ip=$1 12 | group=$2 13 | fi 14 | 15 | loss=$(ping -c 30 -w 30 -W 30 -q "${ip}" | grep -oE "[0-9]*% packet loss" | awk -F'%' '{print $1}') 16 | 17 | echo "${ip} loss percentage: ${loss}%" 18 | curl -X POST -d "[{\"metric\": \"ip.loss\", \"endpoint\": \"$HOSTNAME\", \"timestamp\": $TS,\"step\": 60,\"value\": $loss,\"counterType\": \"GAUGE\",\"tags\": \"group=$group,ip=$ip\"}]" http://127.0.0.1:1988/v1/push 19 | -------------------------------------------------------------------------------- /ip-ping/bin/ping-check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BIN_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | CONF_PATH=${BIN_PATH}/../conf 5 | LOG_PATH=${BIN_PATH}/../log 6 | 7 | echo "=====Start to Ping=====" 8 | 9 | for file in $(ls ${CONF_PATH});do 10 | group=${file##list-} 11 | echo "*****file:${file} group:${group}*****" 12 | 13 | while read ip || [[ -n "${ip}" ]];do 14 | echo "ping ${ip} ......" 15 | sh -x ${BIN_PATH}/ip-ping.sh "${ip}" "${group}" &> ${LOG_PATH}/${ip}.log & 16 | done < ${CONF_PATH}/${file} 17 | done 18 | 19 | echo ">>>>>End of Ping<<<<<" 20 | -------------------------------------------------------------------------------- /ip-ping/conf/list-inner: -------------------------------------------------------------------------------- 1 | 192.168.1.1 2 | 192.168.1.2 3 | -------------------------------------------------------------------------------- /keepalived/check_activemq.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # check 1st time. 4 | if [ $(ps aux | grep -v "grep" | grep "apache-activemq" | wc -l) -eq 0 ]; then 5 | sleep 5 6 | else 7 | exit 0 8 | fi 9 | 10 | # check 2nd time. 11 | if [ $(ps aux | grep -v "grep" | grep "apache-activemq" | wc -l) -eq 0 ]; then 12 | sleep 5 13 | else 14 | exit 0 15 | fi 16 | 17 | # check 3rd time. 18 | if [ $(ps aux | grep -v "grep" | grep "apache-activemq" | wc -l) -eq 0 ]; then 19 | systemctl stop keepalived 20 | exit 1 21 | else 22 | exit 0 23 | fi 24 | -------------------------------------------------------------------------------- /logstash-conf/mysql-slow-query.conf: -------------------------------------------------------------------------------- 1 | input { 2 | file { 3 | type => "string" 4 | path =>["/path/to/mysql/log/mysql-slow.log"] 5 | codec => multiline { 6 | pattern => "^#User@Host:" 7 | negate => true 8 | what => "previous" 9 | } 10 | } 11 | } 12 | 13 | filter { 14 | grok { 15 | match => { 16 | "message" => "SELECT SLEEP" 17 | } 18 | add_tag => ["sleep_drop"] 19 | tag_on_failure => [] 20 | } 21 | if "sleep_drop" in [tags] { 22 | drop {} 23 | } 24 | grok { 25 | match => ["message","(?m)^# User@Host: %{USER:user}\[[^\]]+\] @ (?:(?\S*) )?\[(?:%{IP:clientip})?\]\s*# Query_time: %{NUMBER:query_time:float}\s+Lock_time: %{NUMBER:lock_time:float}\s+Rows_sent: %{NUMBER:rows_sent:int}\s+Rows_examined: %{NUMBER:rows_examined:int}\s*(?:use %{DATA:database};\s*)?SET timestamp=%{NUMBER:timestamp};\s*(?(?\w+)\s+.*)\n# Time:.*$"] 26 | } 27 | date { 28 | match => ["timestamp","UNIX"] 29 | remove_field => ["timestamp"] 30 | } 31 | } 32 | 33 | output { 34 | redis { 35 | host => "127.0.0.1:6379" 36 | data_type => "list" 37 | key => "logstash:mysqlSlowQuery" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-client/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * config.go - the structure of configuration and related functions 3 | * 4 | * history 5 | * -------------------- 6 | * 2018/1/11, by Ye Zhiqin, create 7 | * 8 | */ 9 | package main 10 | 11 | import ( 12 | "gopkg.in/yaml.v2" 13 | "io/ioutil" 14 | "log" 15 | ) 16 | 17 | type Config struct { 18 | RedisServer string `yaml:"redis"` 19 | Path string `yaml:"path"` 20 | Delimiter string `yaml:"delimiter"` 21 | Inotify bool `yaml:"inotify"` 22 | SelfCheck bool `yaml:"selfCheck"` 23 | FalconAgent string `yaml:"agent"` 24 | Endpoint string `yaml:"endpoint"` 25 | Tags string `yaml:"tags"` 26 | CounterType string `yaml:"counterType"` 27 | Step int64 `yaml:"step"` 28 | } 29 | 30 | var config *Config 31 | 32 | /* 33 | * LoadConfig - load configuration file to Config struct 34 | * 35 | * PARAMS: 36 | * No paramter 37 | * 38 | * RETURNS: 39 | * nil, if error ocurred 40 | * *Config, if succeed 41 | */ 42 | func LoadConfig() *Config { 43 | cfg := new(Config) 44 | buf, err := ioutil.ReadFile("config.yaml") 45 | if err != nil { 46 | log.Printf("configuration file reading FAIL: %s", err.Error()) 47 | return nil 48 | } 49 | if err := yaml.Unmarshal(buf, cfg); err != nil { 50 | log.Printf("yaml file unmarshal FAIL: %s", err.Error()) 51 | return nil 52 | } 53 | log.Printf("config: %v", cfg) 54 | 55 | // check configuration 56 | if cfg.RedisServer == "" { 57 | log.Printf("redis server address should not EMPTY!") 58 | return nil 59 | } 60 | 61 | if cfg.Path == "" { 62 | log.Printf("log file path should not EMPTY!") 63 | return nil 64 | } 65 | 66 | if cfg.FalconAgent == "" { 67 | log.Printf("falcon agent address should not EMPTY!") 68 | return nil 69 | } 70 | 71 | return cfg 72 | } 73 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-client/config.yaml: -------------------------------------------------------------------------------- 1 | redis: "127.0.0.1:6379" 2 | path: "/path/to/log" 3 | delimiter: "\n" 4 | inotify: false 5 | selfCheck: true 6 | agent: "http://127.0.0.1:1988/v1/push" 7 | endpoint: "endpoint.name" 8 | tags: "tag1=t1,tag2=t2" 9 | counterType: "GAUGE" 10 | step: 60 11 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-client/control: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | WORKSPACE=$(cd $(dirname $0)/; pwd) 4 | cd ${WORKSPACE} 5 | 6 | app=msgget-monitor-client 7 | pidfile=${app}.pid 8 | logfile=${app}.log 9 | 10 | function check_pid() { 11 | if [[ -f ${pidfile} ]];then 12 | pid=$(cat ${pidfile}) 13 | if [[ -n ${pid} ]]; then 14 | running=$(ps -p $pid|grep -v "PID TTY" |wc -l) 15 | return ${running} 16 | fi 17 | fi 18 | return 0 19 | } 20 | 21 | function start() { 22 | check_pid 23 | running=$? 24 | if [[ ${running} -gt 0 ]];then 25 | echo -n "${app} now is running already, pid=" 26 | cat ${pidfile} 27 | return 1 28 | fi 29 | 30 | nohup ./${app} &>${logfile} & 31 | echo $! > ${pidfile} 32 | echo "${app} started..., pid=$!" 33 | } 34 | 35 | function stop() { 36 | pid=$(cat ${pidfile}) 37 | kill ${pid} 38 | rm ${pidfile} 39 | echo "${app} stoped..." 40 | } 41 | 42 | function restart() { 43 | stop 44 | sleep 1 45 | start 46 | } 47 | 48 | function help() { 49 | echo "$0 pid|start|stop|restart" 50 | } 51 | 52 | function pid() { 53 | cat ${pidfile} 54 | } 55 | 56 | if [ "$1" == "" ]; then 57 | help 58 | elif [ "$1" == "stop" ];then 59 | stop 60 | elif [ "$1" == "start" ];then 61 | start 62 | elif [ "$1" == "restart" ];then 63 | restart 64 | elif [ "$1" == "pid" ];then 65 | pid 66 | else 67 | help 68 | fi 69 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-client/falcon.go: -------------------------------------------------------------------------------- 1 | /* 2 | * falcon.go - the data structure of open falcon and related functions 3 | * 4 | * history 5 | * -------------------- 6 | * 2017/8/18, by Ye Zhiqin, create 7 | * 8 | * DESCRIPTION 9 | * This file contains the definition of open falcon data structure 10 | * and the function to push data to open falcon 11 | */ 12 | 13 | package main 14 | 15 | import ( 16 | "bytes" 17 | "encoding/json" 18 | "fmt" 19 | "io/ioutil" 20 | "log" 21 | "net/http" 22 | "os" 23 | "time" 24 | ) 25 | 26 | type FalconData struct { 27 | Endpoint string `json:"endpoint"` 28 | Metric string `json:"metric"` 29 | Tags string `json:"tags"` 30 | CounterType string `json:"counterType"` 31 | Step int64 `json:"step"` 32 | Timestamp int64 `json:"timestamp"` 33 | Value interface{} `json:"value"` 34 | } 35 | 36 | /* 37 | * SetValue - set FalconData value 38 | * 39 | * RECEIVER: *FalconData 40 | * 41 | * PARAMS: 42 | * - v: value 43 | * 44 | * RETURNS: 45 | * No return value 46 | */ 47 | func (data *FalconData) SetValue(v interface{}) { 48 | data.Value = v 49 | } 50 | 51 | /* 52 | * String - generate a new FalconData 53 | * 54 | * RECEIVER: *FalconData 55 | * 56 | * PARAMS: 57 | * No paramter 58 | * 59 | * RETURNS: 60 | * - string: string to display 61 | */ 62 | func (data *FalconData) String() string { 63 | s := fmt.Sprintf("FalconData Endpoint:%s Metric:%s Tags:%s CounterType:%s Step:%d Timestamp:%d Value:%v", 64 | data.Endpoint, data.Metric, data.Tags, data.CounterType, data.Step, data.Timestamp, data.Value) 65 | return s 66 | } 67 | 68 | /* 69 | * NewFalconData - generate a new FalconData 70 | * 71 | * PARAMS: 72 | * - metric 73 | * - endpoint 74 | * - value 75 | * - counterType 76 | * - timestamp 77 | * - step 78 | * 79 | * RETURNS: 80 | * - *FalconData 81 | */ 82 | func NewFalconData(endpoint string, metric string, tags string, counterType string, step int64, timestamp int64, value interface{}) *FalconData { 83 | point := &FalconData{ 84 | Endpoint: GetEndpoint(endpoint), 85 | Metric: metric, 86 | Tags: tags, 87 | CounterType: counterType, 88 | Step: step, 89 | Timestamp: GetTimestamp(timestamp), 90 | } 91 | point.SetValue(value) 92 | return point 93 | } 94 | 95 | /* 96 | * GetEndpoint - generate endpoint value 97 | * 98 | * PARAMS: 99 | * - endpoint 100 | * 101 | * RETURNS: 102 | * - endpoint: if endpoint is avaliable 103 | * - hostname: if endpoint is empty 104 | * - localhost: if endpoint is empty and can't get hostname 105 | */ 106 | func GetEndpoint(endpoint string) string { 107 | if endpoint != "" { 108 | return endpoint 109 | } 110 | 111 | hostname, err := os.Hostname() 112 | if err != nil { 113 | hostname = "localhost" 114 | } 115 | return hostname 116 | } 117 | 118 | /* 119 | * GetTimestamp - generate timestamp value 120 | * 121 | * PARAMS: 122 | * - timestamp 123 | * 124 | * RETURNS: 125 | * - timestamp: if timestamp > 0 126 | * - now: if timestamp <= 0 127 | */ 128 | func GetTimestamp(timestamp int64) int64 { 129 | if timestamp > 0 { 130 | return timestamp 131 | } else { 132 | return time.Now().Unix() 133 | } 134 | } 135 | 136 | /* 137 | * PushData - push data to open falcon 138 | * 139 | * PARAMS: 140 | * - api: url of agent or transfer 141 | * - data: an array of FalconData 142 | * 143 | * RETURNS: 144 | * - []byte, nil: if succeed 145 | * - nil, error: if fail 146 | */ 147 | func PushData(api string, data []*FalconData) ([]byte, error) { 148 | points, err := json.Marshal(data) 149 | if err != nil { 150 | log.Printf("data marshaling FAIL: %v", err) 151 | return nil, err 152 | } 153 | 154 | response, err := http.Post(api, "Content-Type: application/json", bytes.NewBuffer(points)) 155 | if err != nil { 156 | log.Printf("api call FAIL: %v", err) 157 | return nil, err 158 | } 159 | defer response.Body.Close() 160 | 161 | content, err := ioutil.ReadAll(response.Body) 162 | if err != nil { 163 | return nil, err 164 | } 165 | 166 | return content, nil 167 | } 168 | 169 | /* 170 | * Push2Falcon - push file agent data 171 | * 172 | * PARAMS: 173 | * - api: falcon agent address 174 | * - data: AgentData 175 | */ 176 | func Push2Falcon(api string, data *AgentData) { 177 | var d []*FalconData 178 | 179 | // error.count 180 | point := NewFalconData(data.Endpoint, "error.count", data.Tags, data.CounterType, data.Step, data.TsEnd, data.ErrorCnt) 181 | d = append(d, point) 182 | 183 | // request.count 184 | point = NewFalconData(data.Endpoint, "request.count", data.Tags, data.CounterType, data.Step, data.TsEnd, data.ReqCnt) 185 | d = append(d, point) 186 | 187 | // request.200.count 188 | point = NewFalconData(data.Endpoint, "request.200.count", data.Tags, data.CounterType, data.Step, data.TsEnd, data.Req200Cnt) 189 | d = append(d, point) 190 | 191 | // request.499.count 192 | point = NewFalconData(data.Endpoint, "request.499.count", data.Tags, data.CounterType, data.Step, data.TsEnd, data.Req499Cnt) 193 | d = append(d, point) 194 | 195 | // request.500.count 196 | point = NewFalconData(data.Endpoint, "request.500.count", data.Tags, data.CounterType, data.Step, data.TsEnd, data.Req500Cnt) 197 | d = append(d, point) 198 | 199 | // request.502.count 200 | point = NewFalconData(data.Endpoint, "request.502.count", data.Tags, data.CounterType, data.Step, data.TsEnd, data.Req502Cnt) 201 | d = append(d, point) 202 | 203 | // request.504.count 204 | point = NewFalconData(data.Endpoint, "request.504.count", data.Tags, data.CounterType, data.Step, data.TsEnd, data.Req504Cnt) 205 | d = append(d, point) 206 | 207 | // ip.count 208 | point = NewFalconData(data.Endpoint, "ip.count", data.Tags, data.CounterType, data.Step, data.TsEnd, data.IpSet.Len()) 209 | d = append(d, point) 210 | 211 | // shopid.count 212 | point = NewFalconData(data.Endpoint, "shopid.count", data.Tags, data.CounterType, data.Step, data.TsEnd, data.IdSet.Len()) 213 | d = append(d, point) 214 | 215 | //log.Printf("falcon points: %v", d) 216 | response, err := PushData(api, d) 217 | if err != nil { 218 | log.Printf("push data to falcon FAIL: %s", err.Error()) 219 | } 220 | log.Printf("push data to falcon succeed: %s", string(response)) 221 | return 222 | } 223 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-client/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * main.go - the entry of program 3 | * 4 | * history 5 | * -------------------- 6 | * 2018/1/11, by Ye Zhiqin, create 7 | * 8 | */ 9 | 10 | package main 11 | 12 | import ( 13 | //"bytes" 14 | "log" 15 | "os" 16 | "os/signal" 17 | "sync" 18 | "syscall" 19 | "time" 20 | ) 21 | 22 | const ( 23 | MAX_UNCHANGED_TIME = 5 24 | REQUEST_URI = "/msg/get.htm" 25 | ) 26 | 27 | type Record struct { 28 | Finish chan bool 29 | Agent *FileAgent 30 | } 31 | 32 | var record *Record 33 | var wg sync.WaitGroup 34 | 35 | // main 36 | func main() { 37 | sysCh := make(chan os.Signal, 1) 38 | signal.Notify(sysCh, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) 39 | defer close(sysCh) 40 | 41 | // set log format 42 | log.SetFlags(log.LstdFlags | log.Lshortfile) 43 | 44 | // load configuration 45 | config = LoadConfig() 46 | if config == nil { 47 | log.Printf("configuration loading FAIL, please check the config.yaml") 48 | os.Exit(-1) 49 | } 50 | 51 | StartAgent() 52 | 53 | MAIN: 54 | for { 55 | select { 56 | case <-sysCh: 57 | log.Printf("system signal: %s", sysCh) 58 | StopAgent() 59 | break MAIN 60 | default: 61 | time.Sleep(time.Second) 62 | } 63 | } 64 | 65 | wg.Wait() 66 | log.Printf("msgget-monitor-client exit...") 67 | } 68 | 69 | /* 70 | * StartAgent - generate the file agent by the configuration 71 | * 72 | * PARAMS: 73 | * No paramter 74 | * 75 | * RETURNS: 76 | * No return value 77 | */ 78 | func StartAgent() { 79 | // initialize agent data 80 | data := new(AgentData) 81 | data.Endpoint = config.Endpoint 82 | data.Tags = config.Tags 83 | data.CounterType = config.CounterType 84 | data.Step = config.Step 85 | data.Pattern = REQUEST_URI 86 | 87 | data.TsStart = 0 88 | data.TsEnd = 0 89 | data.TsUpdate = 0 90 | 91 | data.ErrorCnt = 0 92 | data.ReqCnt = 0 93 | data.Req200Cnt = 0 94 | data.Req499Cnt = 0 95 | data.Req500Cnt = 0 96 | data.Req502Cnt = 0 97 | data.Req504Cnt = 0 98 | data.IpSet = nil 99 | data.IdSet = nil 100 | 101 | // initialize agent 102 | agent := new(FileAgent) 103 | agent.Filename = config.Path 104 | agent.Delimiter = config.Delimiter 105 | agent.Inotify = config.Inotify 106 | agent.SelfCheck = config.SelfCheck 107 | 108 | agent.File = nil 109 | agent.FileInfo = nil 110 | agent.LastOffset = 0 111 | agent.UnchangeTime = 0 112 | 113 | agent.Data = data 114 | 115 | // initialize agent record 116 | record := new(Record) 117 | ch := make(chan bool, 1) 118 | record.Finish = ch 119 | record.Agent = agent 120 | 121 | // launch file agent 122 | wg.Add(1) 123 | log.Printf("wg: %v", wg) 124 | if record.Agent.Inotify { 125 | go TailWithInotify(record.Agent, record.Finish) 126 | } else { 127 | go TailWithCheck(record.Agent, record.Finish) 128 | } 129 | } 130 | 131 | /* 132 | * StopAgent - recall the file agent when program exit or configuration changed 133 | * 134 | * PARAMS: 135 | * No paramter 136 | * 137 | * RETURNS: 138 | * No return value 139 | */ 140 | func StopAgent() { 141 | record.Finish <- true 142 | close(record.Finish) 143 | 144 | record = nil 145 | } 146 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-client/re.go: -------------------------------------------------------------------------------- 1 | /* 2 | * re.go - functions related to text processing 3 | * 4 | * history 5 | * -------------------- 6 | * 2018/1/12, by Ye Zhiqin, create 7 | * 8 | */ 9 | 10 | package main 11 | 12 | import ( 13 | "strings" 14 | ) 15 | 16 | /* 17 | * MatchRequest - match the request uri 18 | * 19 | * PARAMS: 20 | * - request: request in log 21 | * - pattern: certain uri 22 | * 23 | * RETURNS: 24 | * - true: if match 25 | * - false: if not match 26 | */ 27 | func MatchRequest(request, pattern string) bool { 28 | ok := strings.Contains(request, pattern) 29 | return ok 30 | } 31 | 32 | /* 33 | * GetID - get shop ID in request body 34 | * 35 | * PARAMS: 36 | * - body: request body in log 37 | * 38 | * RETURNS: 39 | * - true, id, nil: if found 40 | * - false, "": if not found 41 | */ 42 | func GetID(body string) (bool, string) { 43 | fields := strings.Split(body, "&") 44 | for _, field := range fields { 45 | kv := strings.Split(field, "=") 46 | if len(kv) != 2 { 47 | continue 48 | } 49 | k := kv[0] 50 | if k != "shopID" { 51 | continue 52 | } 53 | v := kv[1] 54 | return true, v 55 | } 56 | return false, "" 57 | } 58 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-client/redis.go: -------------------------------------------------------------------------------- 1 | /* 2 | * redis.go - functions to push data to redis 3 | * 4 | * history 5 | * -------------------- 6 | * 2017/1/11, by Ye Zhiqin, create 7 | * 8 | */ 9 | 10 | package main 11 | 12 | import ( 13 | "log" 14 | "time" 15 | 16 | "github.com/garyburd/redigo/redis" 17 | ) 18 | 19 | func IncrBy(endpoint string, key string, value int64) { 20 | conn, err := redis.Dial("tcp", endpoint) 21 | if err != nil { 22 | log.Printf("connect to redis %s FAiL", endpoint) 23 | return 24 | } 25 | defer conn.Close() 26 | 27 | result, err := redis.Int(conn.Do("INCRBY", key, value)) 28 | if err != nil { 29 | log.Printf("Execute: INCRBY %s %d FAiL", key, value) 30 | return 31 | } 32 | log.Printf("Result: %d, Execute: INCRBY %s %d", result, key, value) 33 | return 34 | } 35 | 36 | func Sadd(endpoint string, key string, value []string) { 37 | conn, err := redis.Dial("tcp", endpoint) 38 | if err != nil { 39 | log.Printf("connect to redis %s FAiL", endpoint) 40 | return 41 | } 42 | defer conn.Close() 43 | 44 | c := 0 45 | for _, id := range value { 46 | result, err := redis.Int(conn.Do("SADD", key, id)) 47 | if err != nil { 48 | log.Printf("Execute: SADD %s %s FAiL", key, id) 49 | } 50 | c += result 51 | } 52 | log.Printf("Result: %d, Execute: SADD %s ...", c, key) 53 | return 54 | } 55 | 56 | func Push2Redis(endpoint string, data *AgentData) { 57 | start := time.Unix(data.TsStart, 0) 58 | prefix := start.Format("200601021504") 59 | 60 | key := "request-" + prefix 61 | IncrBy(endpoint, key, data.ReqCnt) 62 | 63 | key = "request200-" + prefix 64 | IncrBy(endpoint, key, data.Req200Cnt) 65 | 66 | key = "request499-" + prefix 67 | IncrBy(endpoint, key, data.Req499Cnt) 68 | 69 | key = "request500-" + prefix 70 | IncrBy(endpoint, key, data.Req500Cnt) 71 | 72 | key = "request502-" + prefix 73 | IncrBy(endpoint, key, data.Req502Cnt) 74 | 75 | key = "request504-" + prefix 76 | IncrBy(endpoint, key, data.Req504Cnt) 77 | 78 | key = "ip-" + prefix 79 | Sadd(endpoint, key, data.IpSet.ToSlice()) 80 | 81 | key = "shopid-" + prefix 82 | Sadd(endpoint, key, data.IdSet.ToSlice()) 83 | } 84 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-client/set.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | type StringSet struct { 8 | S map[string]struct{} 9 | } 10 | 11 | func NewStringSet() *StringSet { 12 | return &StringSet{ 13 | S: make(map[string]struct{}), 14 | } 15 | } 16 | 17 | func (this *StringSet) Add(element string) *StringSet { 18 | this.S[element] = struct{}{} 19 | return this 20 | } 21 | 22 | func (this *StringSet) Exists(element string) bool { 23 | _, exists := this.S[element] 24 | return exists 25 | } 26 | 27 | func (this *StringSet) Delete(element string) { 28 | delete(this.S, element) 29 | } 30 | 31 | func (this *StringSet) Clear() { 32 | this.S = make(map[string]struct{}) 33 | } 34 | 35 | func (this *StringSet) Len() int { 36 | return len(this.S) 37 | } 38 | 39 | func (this *StringSet) ToValueString() string { 40 | count := len(this.S) 41 | if count == 0 { 42 | return "" 43 | } 44 | 45 | result := "" 46 | for element := range this.S { 47 | result = result + " " + element 48 | } 49 | return strings.Trim(result, " ") 50 | } 51 | 52 | func (this *StringSet) ToSlice() []string { 53 | count := len(this.S) 54 | if count == 0 { 55 | return []string{} 56 | } 57 | 58 | r := make([]string, count) 59 | 60 | i := 0 61 | for element := range this.S { 62 | r[i] = element 63 | i++ 64 | } 65 | 66 | return r 67 | } 68 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-client/tail.go: -------------------------------------------------------------------------------- 1 | /* 2 | * tail.go - file agent data structure and funtions to tail file 3 | * 4 | * history 5 | * -------------------- 6 | * 2017/8/18, by Ye Zhiqin, create 7 | * 2017/9/30, by Ye Zhiqin, modify 8 | * 2018/1/3, by Ye Zhiqin, modify 9 | * 2018/1/11, by Ye Zhiqin, modify 10 | * 11 | * DESCRIPTION 12 | * This file contains the definition of file agent 13 | * and the functions to tail log file 14 | */ 15 | 16 | package main 17 | 18 | import ( 19 | "bytes" 20 | "fmt" 21 | "io" 22 | "io/ioutil" 23 | "log" 24 | "os" 25 | "path" 26 | "time" 27 | 28 | "github.com/bitly/go-simplejson" 29 | "github.com/fsnotify/fsnotify" 30 | ) 31 | 32 | type FileAgent struct { 33 | Filename string 34 | Delimiter string 35 | Inotify bool 36 | SelfCheck bool 37 | 38 | File *os.File 39 | FileInfo os.FileInfo 40 | LastOffset int64 41 | UnchangeTime int 42 | 43 | Data *AgentData 44 | } 45 | 46 | type AgentData struct { 47 | Endpoint string 48 | Tags string 49 | CounterType string 50 | Step int64 51 | Pattern string 52 | 53 | TsStart int64 54 | TsEnd int64 55 | TsUpdate int64 56 | 57 | ErrorCnt int64 58 | ReqCnt int64 59 | Req200Cnt int64 60 | Req499Cnt int64 61 | Req500Cnt int64 62 | Req502Cnt int64 63 | Req504Cnt int64 64 | 65 | IpSet *StringSet 66 | IdSet *StringSet 67 | } 68 | 69 | /* 70 | * Report - push and update data after a period passed 71 | * 72 | * RECEIVER: *FileAgent 73 | * 74 | * PARAMS: 75 | * - ts: timestamp 76 | * 77 | * RETURNS: 78 | * No paramter 79 | */ 80 | func (fa *FileAgent) Report(ts time.Time) { 81 | if fa.SelfCheck { 82 | Push2Falcon(config.FalconAgent, fa.Data) 83 | } 84 | 85 | Push2Redis(config.RedisServer, fa.Data) 86 | 87 | log.Printf("=====Report[%d]=====\nError:%d Total:%d 200:%d 499:%d 500:%d 502:%d 504:%d ip:%d id:%d", 88 | fa.Data.TsStart, 89 | fa.Data.ErrorCnt, 90 | fa.Data.ReqCnt, 91 | fa.Data.Req200Cnt, 92 | fa.Data.Req499Cnt, 93 | fa.Data.Req500Cnt, 94 | fa.Data.Req502Cnt, 95 | fa.Data.Req504Cnt, 96 | fa.Data.IpSet.Len(), 97 | fa.Data.IdSet.Len()) 98 | 99 | // update value 100 | fa.Data.ErrorCnt = 0 101 | fa.Data.ReqCnt = 0 102 | fa.Data.Req200Cnt = 0 103 | fa.Data.Req499Cnt = 0 104 | fa.Data.Req500Cnt = 0 105 | fa.Data.Req502Cnt = 0 106 | fa.Data.Req504Cnt = 0 107 | fa.Data.IpSet.Clear() 108 | fa.Data.IdSet.Clear() 109 | 110 | //update timestamp 111 | fa.Data.TsStart += fa.Data.Step 112 | fa.Data.TsEnd += fa.Data.Step 113 | fa.Data.TsUpdate = ts.Unix() 114 | } 115 | 116 | /* 117 | * Timeup - the process after a period passed 118 | * 119 | * RECEIVER: *FileAgent 120 | * 121 | * PARAMS: 122 | * No paramter 123 | * 124 | * RETURNS: 125 | * No paramter 126 | */ 127 | func (fa *FileAgent) Timeup() { 128 | ts := time.Now() 129 | if ts.Unix()-fa.Data.TsStart >= fa.Data.Step { 130 | fa.Report(ts) 131 | } 132 | } 133 | 134 | /* 135 | * MatchLine - process each line of log 136 | * 137 | * RECEIVER: *FileAgent 138 | * 139 | * PARAMS: 140 | * - line: a line of log file 141 | * 142 | * RETURNS: 143 | * No paramter 144 | */ 145 | func (fa *FileAgent) MatchLine(line []byte) { 146 | // convert one request log to a json struct 147 | contents, err := ioutil.ReadAll(bytes.NewReader(line)) 148 | if err != nil { 149 | log.Printf("convert line <%s> to *Reader FAIL: %s", line, err.Error()) 150 | fa.Data.ErrorCnt += 1 151 | return 152 | } 153 | 154 | json, err := simplejson.NewJson(contents) 155 | if err != nil { 156 | log.Printf("convert json string FAIL: %s", err.Error()) 157 | fa.Data.ErrorCnt += 1 158 | return 159 | } 160 | 161 | // get request field, return if uri is not /msg/get.htm 162 | request, err := json.Get("request").String() 163 | if err != nil { 164 | fa.Data.ErrorCnt += 1 165 | log.Printf("get request field in line FAIL: %s", err.Error()) 166 | return 167 | } 168 | 169 | if ok := MatchRequest(request, fa.Data.Pattern); !ok { 170 | return 171 | } 172 | 173 | // process status, ip, shopID info 174 | status, err := json.Get("status").String() 175 | if err != nil { 176 | fa.Data.ErrorCnt += 1 177 | log.Printf("get status field in line FAIL: %s", err.Error()) 178 | return 179 | } 180 | switch status { 181 | case "200": 182 | fa.Data.Req200Cnt++ 183 | fa.Data.ReqCnt++ 184 | case "499": 185 | fa.Data.Req499Cnt++ 186 | fa.Data.ReqCnt++ 187 | case "500": 188 | fa.Data.Req500Cnt++ 189 | fa.Data.ReqCnt++ 190 | case "502": 191 | fa.Data.Req502Cnt++ 192 | fa.Data.ReqCnt++ 193 | case "504": 194 | fa.Data.Req504Cnt++ 195 | fa.Data.ReqCnt++ 196 | default: 197 | fa.Data.ReqCnt++ 198 | } 199 | 200 | ip, err := json.Get("remote_addr").String() 201 | if err != nil { 202 | fa.Data.ErrorCnt += 1 203 | log.Printf("get remote_addr field in line FAIL: %s", err.Error()) 204 | return 205 | } 206 | fa.Data.IpSet.Add(ip) 207 | 208 | body, err := json.Get("request_body").String() 209 | if err != nil { 210 | fa.Data.ErrorCnt += 1 211 | log.Printf("get request_body field in line FAIL: %s", err.Error()) 212 | return 213 | } 214 | ok, id := GetID(body) 215 | if !ok { 216 | fa.Data.ErrorCnt += 1 217 | log.Printf("get shopID from request body FAIL") 218 | return 219 | } 220 | fa.Data.IdSet.Add(id) 221 | 222 | return 223 | } 224 | 225 | /* 226 | * ReadRemainder - reading new bytes of log file 227 | * 228 | * RECEIVER: *FileAgent 229 | * 230 | * PARAMS: 231 | * No paramter 232 | * 233 | * RETURNS: 234 | * nil: succeed 235 | * error: fail 236 | */ 237 | func (fa *FileAgent) ReadRemainder() error { 238 | tailable := fa.FileInfo.Mode().IsRegular() 239 | size := fa.FileInfo.Size() 240 | 241 | // LastOffset less then new size. Maybe the file has been truncated. 242 | if tailable && fa.LastOffset > size { 243 | // seek the cursor to the header of new file 244 | offset, err := fa.File.Seek(0, os.SEEK_SET) 245 | if err != nil { 246 | log.Printf("file %s seek FAIL: %s", fa.Filename, err.Error()) 247 | return err 248 | } 249 | if offset != 0 { 250 | log.Printf("offset is not equal 0") 251 | } 252 | fa.LastOffset = 0 253 | 254 | return nil 255 | } 256 | 257 | bufsize := size - fa.LastOffset 258 | if bufsize == 0 { 259 | return nil 260 | } 261 | data := make([]byte, bufsize) 262 | readsize, err := fa.File.Read(data) 263 | 264 | if err != nil && err != io.EOF { 265 | log.Printf("file %s read FAIL: %s", err.Error()) 266 | return err 267 | } 268 | if readsize == 0 { 269 | log.Printf("file %s read 0 data", fa.Filename) 270 | return nil 271 | } 272 | 273 | if fa.Delimiter == "" { 274 | fa.Delimiter = "\n" 275 | } 276 | sep := []byte(fa.Delimiter) 277 | lines := bytes.SplitAfter(data, sep) 278 | length := len(lines) 279 | 280 | for idx, line := range lines { 281 | // just process entire line with the delimiter 282 | if idx == length-1 { 283 | backsize := len(line) 284 | movesize := readsize - backsize 285 | 286 | _, err := fa.File.Seek(-int64(backsize), os.SEEK_CUR) 287 | if err != nil { 288 | log.Printf("seek file %s FAIL: %s", fa.Filename, err.Error()) 289 | return err 290 | } 291 | fa.LastOffset += int64(movesize) 292 | 293 | break 294 | } 295 | fa.MatchLine(line) 296 | } 297 | return nil 298 | } 299 | 300 | /* 301 | * TailWithCheck - tail log file in a loop 302 | * 303 | * PARAMS: 304 | * - fa: file agent 305 | * - finish: a channel to receiver stop signal 306 | * 307 | * RETURNS: 308 | * No return value 309 | */ 310 | func TailWithCheck(fa *FileAgent, finish <-chan bool) { 311 | log.Printf("agent for %s is launching...", fa.Filename) 312 | 313 | // create one second ticker 314 | ticker := time.NewTicker(time.Second) 315 | defer ticker.Stop() 316 | 317 | TAIL: 318 | for { 319 | select { 320 | case <-finish: 321 | if fa.File != nil { 322 | if err := fa.File.Close(); err != nil { 323 | log.Printf("file closing FAIL: %s", err.Error()) 324 | } 325 | } 326 | break TAIL 327 | case <-ticker.C: 328 | fa.Timeup() 329 | default: 330 | fa.TryReading() 331 | time.Sleep(time.Millisecond * 250) 332 | } 333 | } 334 | 335 | wg.Done() 336 | log.Printf("wg: %v", wg) 337 | log.Printf("agent for %s is exiting...", fa.Filename) 338 | } 339 | 340 | /* 341 | * TryReading - reading log file 342 | * 343 | * RECEIVER: *FileAgent 344 | * 345 | * PARAMS: 346 | * No paramter 347 | * 348 | * RETURNS: 349 | * No return value 350 | */ 351 | func (fa *FileAgent) TryReading() { 352 | if fa.File == nil { 353 | log.Printf("file %s is nil", fa.Filename) 354 | if err := fa.FileRecheck(); err != nil { 355 | log.Printf("file recheck FAIL: %s", err.Error()) 356 | } 357 | return 358 | } 359 | 360 | if !fa.IsChanged() { 361 | if fa.UnchangeTime >= MAX_UNCHANGED_TIME { 362 | fa.FileRecheck() 363 | } 364 | return 365 | } 366 | 367 | fa.ReadRemainder() 368 | } 369 | 370 | /* 371 | * TailWithInotify - trace log file 372 | * 373 | * PARAMS: 374 | * - fa: file agent 375 | * - finish: a channel to receiver stop signal 376 | * 377 | * RETURNS: 378 | * No return value 379 | */ 380 | func TailWithInotify(fa *FileAgent, chanFinish <-chan bool) { 381 | fmt.Printf("agent for %s is starting...\n", fa.Filename) 382 | 383 | // create one second ticker 384 | ticker := time.NewTicker(time.Second) 385 | defer ticker.Stop() 386 | 387 | // craete file watcher 388 | watcher, err := fsnotify.NewWatcher() 389 | if err != nil { 390 | log.Printf("watcher creating FAIL: %s", err.Error()) 391 | wg.Done() 392 | log.Printf("agent for %s is exiting...", fa.Filename) 393 | return 394 | } 395 | defer watcher.Close() 396 | 397 | // add parent directory to watcher 398 | dir := path.Dir(fa.Filename) 399 | if err := watcher.Add(dir); err != nil { 400 | log.Printf("add %s to watcher FAIL: %s", dir, err.Error()) 401 | wg.Done() 402 | log.Printf("agent for %s is exiting...", fa.Filename) 403 | return 404 | } 405 | 406 | // open file and initialize file agent 407 | if err := fa.FileOpen(); err != nil { 408 | log.Printf("file $s open FAIL when agent initializing: %s", fa.Filename, err.Error()) 409 | } 410 | 411 | // trace file in this loop 412 | TRACE: 413 | for { 414 | select { 415 | case <-chanFinish: 416 | if err := watcher.Remove(dir); err != nil { 417 | log.Printf("watcher file removing FAIL: %s", err.Error()) 418 | } 419 | if fa.File != nil { 420 | if err := fa.File.Close(); err != nil { 421 | log.Printf("file closing FAIL: %s", err.Error()) 422 | } 423 | } 424 | break TRACE 425 | case event := <-watcher.Events: 426 | if event.Name == fa.Filename { 427 | // WRITE event 428 | if 2 == event.Op { 429 | if err := fa.FileInfoUpdate(); err != nil { 430 | log.Printf("file %s stat FAIL", fa.Filename) 431 | if fa.UnchangeTime > MAX_UNCHANGED_TIME { 432 | fa.FileReopen() 433 | } 434 | continue 435 | } 436 | 437 | if err := fa.ReadRemainder(); err != nil { 438 | log.Printf("file %s reading FAIL, recheck it", fa.Filename) 439 | fa.FileReopen() 440 | } 441 | } 442 | // CREATE event 443 | if 1 == event.Op { 444 | fmt.Printf("fa %s, watch %s receive event CREATE\n", fa.Filename, event.Name) 445 | fa.FileReopen() 446 | } 447 | // REMOVE/RENAME event 448 | if 4 == event.Op || 8 == event.Op { 449 | fmt.Printf("fa %s, watch %s receive event REMOVE|RENAME\n", fa.Filename, event.Name) 450 | fa.FileClose() 451 | } 452 | // CHMOD event 453 | if 16 == event.Op { 454 | fmt.Printf("fa %s, watch %s receive event CHMOD\n", fa.Filename, event.Name) 455 | } 456 | } 457 | case err := <-watcher.Errors: 458 | log.Printf("%s receive error %s", fa.Filename, err.Error()) 459 | case <-ticker.C: 460 | fa.Timeup() 461 | default: 462 | time.Sleep(time.Millisecond * 250) 463 | } 464 | } 465 | 466 | wg.Done() 467 | 468 | fmt.Printf("agent for %s is exiting...\n", fa.Filename) 469 | } 470 | 471 | /* 472 | * FileOpen - open file while file agent initializing 473 | * 474 | * RECEIVER: *FileAgent 475 | * 476 | * PARAMS: 477 | * No paramter 478 | * 479 | * RETURNS: 480 | * nil, if succeed 481 | * error, if fail 482 | */ 483 | func (fa *FileAgent) FileOpen() error { 484 | // close old file 485 | if fa.File != nil { 486 | if err := fa.File.Close(); err != nil { 487 | log.Printf("file closing FAIL: %s", err.Error()) 488 | } 489 | } 490 | 491 | fa.File = nil 492 | fa.FileInfo = nil 493 | 494 | // open new file 495 | filename := fa.Filename 496 | 497 | file, err := os.Open(filename) 498 | if err != nil { 499 | log.Printf("file %s open FAIL: %s", fa.Filename, err.Error()) 500 | return err 501 | } 502 | 503 | fileinfo, err := file.Stat() 504 | if err != nil { 505 | log.Printf("file %s stat FAIL: %s", fa.Filename, err.Error()) 506 | return err 507 | } 508 | 509 | fmt.Printf("file %s is open\n", fa.Filename) 510 | 511 | fa.File = file 512 | fa.FileInfo = fileinfo 513 | fa.LastOffset = 0 514 | fa.UnchangeTime = 0 515 | 516 | // seek the cursor to the end of new file 517 | _, err = fa.File.Seek(fa.FileInfo.Size(), os.SEEK_SET) 518 | if err != nil { 519 | log.Printf("seek file %s FAIL: %s", fa.Filename, err.Error()) 520 | } 521 | fa.LastOffset += fa.FileInfo.Size() 522 | 523 | // initialize agent data 524 | now := time.Now() 525 | minute := now.Format("200601021504") 526 | 527 | tsNow := now.Unix() 528 | tsStart := tsNow 529 | 530 | start, err := time.ParseInLocation("20060102150405", minute+"00", now.Location()) 531 | if err != nil { 532 | log.Printf("timestamp setting FAIL: %s", err.Error()) 533 | } else { 534 | tsStart = start.Unix() 535 | } 536 | 537 | fa.Data.TsStart = tsStart 538 | fa.Data.TsEnd = tsStart + fa.Data.Step - 1 539 | fa.Data.TsUpdate = tsNow 540 | fa.Data.ErrorCnt = 0 541 | fa.Data.ReqCnt = 0 542 | fa.Data.Req200Cnt = 0 543 | fa.Data.Req499Cnt = 0 544 | fa.Data.Req500Cnt = 0 545 | fa.Data.Req502Cnt = 0 546 | fa.Data.Req504Cnt = 0 547 | fa.Data.IpSet = NewStringSet() 548 | fa.Data.IdSet = NewStringSet() 549 | 550 | return nil 551 | } 552 | 553 | /* 554 | * FileInfoUpdate - stat file when WRITE 555 | * 556 | * RECEIVER: *FileAgent 557 | * 558 | * PARAMS: 559 | * No paramter 560 | * 561 | * RETURNS: 562 | * nil, if succeed 563 | * error, if fail 564 | */ 565 | func (fa *FileAgent) FileInfoUpdate() error { 566 | fileinfo, err := fa.File.Stat() 567 | if err != nil { 568 | log.Printf("file %s stat FAIL: %s", fa.Filename, err.Error()) 569 | fa.UnchangeTime += 1 570 | return err 571 | } 572 | 573 | fa.FileInfo = fileinfo 574 | fa.UnchangeTime = 0 575 | return nil 576 | } 577 | 578 | /* 579 | * FileClose - close file when REMOVE/RENAME 580 | * 581 | * RECEIVER: *FileAgent 582 | * 583 | * PARAMS: 584 | * No paramter 585 | * 586 | * RETURNS: 587 | * nil, if succeed 588 | * error, if fail 589 | */ 590 | func (fa *FileAgent) FileClose() error { 591 | // close old file 592 | if fa.File != nil { 593 | if err := fa.File.Close(); err != nil { 594 | log.Printf("file closing FAIL: %s", err.Error()) 595 | return err 596 | } 597 | } 598 | 599 | // clear file info 600 | fa.File = nil 601 | fa.FileInfo = nil 602 | fa.LastOffset = 0 603 | fa.UnchangeTime = 0 604 | 605 | // clear data 606 | fa.Data.TsStart = 0 607 | fa.Data.TsEnd = 0 608 | fa.Data.TsUpdate = 0 609 | fa.Data.ErrorCnt = 0 610 | fa.Data.ReqCnt = 0 611 | fa.Data.Req200Cnt = 0 612 | fa.Data.Req499Cnt = 0 613 | fa.Data.Req500Cnt = 0 614 | fa.Data.Req502Cnt = 0 615 | fa.Data.Req504Cnt = 0 616 | fa.Data.IpSet = nil 617 | fa.Data.IdSet = nil 618 | 619 | return nil 620 | } 621 | 622 | /* 623 | * FileReopen - reopen file when CREATE/ERROR 624 | * 625 | * RECEIVER: *FileAgent 626 | * 627 | * PARAMS: 628 | * No paramter 629 | * 630 | * RETURNS: 631 | * nil: succeed 632 | * error: fail 633 | */ 634 | func (fa *FileAgent) FileReopen() error { 635 | // close old file 636 | if fa.File != nil { 637 | if err := fa.File.Close(); err != nil { 638 | log.Printf("file closing FAIL: %s", err.Error()) 639 | } 640 | } 641 | 642 | fa.File = nil 643 | fa.FileInfo = nil 644 | 645 | // open new file 646 | filename := fa.Filename 647 | 648 | file, err := os.Open(filename) 649 | if err != nil { 650 | log.Printf("file %s opening FAIL: %s", fa.Filename, err.Error()) 651 | return err 652 | } 653 | 654 | fileinfo, err := file.Stat() 655 | if err != nil { 656 | log.Printf("file %s stat FAIL: %s", fa.Filename, err.Error()) 657 | return err 658 | } 659 | 660 | log.Printf("file %s recheck ok, it is a new file", fa.Filename) 661 | 662 | fa.File = file 663 | fa.FileInfo = fileinfo 664 | fa.LastOffset = 0 665 | fa.UnchangeTime = 0 666 | 667 | // seek the cursor to the start of new file 668 | _, err = fa.File.Seek(0, os.SEEK_SET) 669 | if err != nil { 670 | log.Printf("seek file %s FAIL: %s", fa.Filename, err.Error()) 671 | } 672 | 673 | // initialize agent data 674 | now := time.Now() 675 | minute := now.Format("200601021504") 676 | 677 | tsNow := now.Unix() 678 | tsStart := tsNow 679 | 680 | start, err := time.ParseInLocation("20060102150405", minute+"00", now.Location()) 681 | if err != nil { 682 | log.Printf("timestamp setting FAIL: %s", err.Error()) 683 | } else { 684 | tsStart = start.Unix() 685 | } 686 | 687 | fa.Data.TsStart = tsStart 688 | fa.Data.TsEnd = tsStart + fa.Data.Step - 1 689 | fa.Data.TsUpdate = tsNow 690 | fa.Data.ErrorCnt = 0 691 | fa.Data.ReqCnt = 0 692 | fa.Data.Req200Cnt = 0 693 | fa.Data.Req499Cnt = 0 694 | fa.Data.Req500Cnt = 0 695 | fa.Data.Req502Cnt = 0 696 | fa.Data.Req504Cnt = 0 697 | fa.Data.IpSet = NewStringSet() 698 | fa.Data.IdSet = NewStringSet() 699 | 700 | return nil 701 | } 702 | 703 | /* 704 | * FileRecheck - recheck the file for file agent 705 | * 706 | * RECEIVER: *FileAgent 707 | * 708 | * PARAMS: 709 | * No paramter 710 | * 711 | * RETURNS: 712 | * nil, if succeed 713 | * error, if fail 714 | */ 715 | func (fa *FileAgent) FileRecheck() error { 716 | filename := fa.Filename 717 | 718 | file, err := os.Open(filename) 719 | if err != nil { 720 | log.Printf("file %s opening FAIL: %s", fa.Filename, err.Error()) 721 | fa.UnchangeTime = 0 722 | return err 723 | } 724 | 725 | fileinfo, err := file.Stat() 726 | if err != nil { 727 | log.Printf("file %s stat FAIL: %s", fa.Filename, err.Error()) 728 | fa.UnchangeTime = 0 729 | return err 730 | } 731 | 732 | isSameFile := os.SameFile(fa.FileInfo, fileinfo) 733 | if !isSameFile { 734 | log.Printf("file %s recheck, it is a new file", fa.Filename) 735 | if fa.File != nil { 736 | if err := fa.File.Close(); err != nil { 737 | log.Printf("old file closing FAIL: %s", err.Error()) 738 | } 739 | } 740 | 741 | fa.File = file 742 | fa.FileInfo = fileinfo 743 | fa.LastOffset = 0 744 | fa.UnchangeTime = 0 745 | 746 | // seek the cursor to the end of new file 747 | offset, err := fa.File.Seek(fa.FileInfo.Size(), os.SEEK_SET) 748 | if err != nil { 749 | log.Printf("seek file %s FAIL: %s", fa.Filename, err.Error()) 750 | } 751 | log.Printf("seek file %s to %d", fa.Filename, offset) 752 | fa.LastOffset += fa.FileInfo.Size() 753 | 754 | // initialize agent data 755 | now := time.Now() 756 | minute := now.Format("200601021504") 757 | 758 | tsNow := now.Unix() 759 | tsStart := tsNow 760 | 761 | start, err := time.ParseInLocation("20060102150405", minute+"00", now.Location()) 762 | if err != nil { 763 | log.Printf("timestamp setting FAIL: %s", err.Error()) 764 | } else { 765 | tsStart = start.Unix() 766 | } 767 | 768 | fa.Data.TsStart = tsStart 769 | fa.Data.TsEnd = tsStart + fa.Data.Step - 1 770 | fa.Data.TsUpdate = tsNow 771 | fa.Data.ErrorCnt = 0 772 | fa.Data.ReqCnt = 0 773 | fa.Data.Req200Cnt = 0 774 | fa.Data.Req499Cnt = 0 775 | fa.Data.Req500Cnt = 0 776 | fa.Data.Req502Cnt = 0 777 | fa.Data.Req504Cnt = 0 778 | fa.Data.IpSet = NewStringSet() 779 | fa.Data.IdSet = NewStringSet() 780 | 781 | return nil 782 | } else { 783 | fa.UnchangeTime = 0 784 | return nil 785 | } 786 | } 787 | 788 | /* 789 | * IsChanged - check the change of log file 790 | * 791 | * RECEIVER: *FileAgent 792 | * 793 | * PARAMS: 794 | * No paramter 795 | * 796 | * RETURNS: 797 | * - true: if change 798 | * - false: if not change 799 | */ 800 | func (fa *FileAgent) IsChanged() bool { 801 | lastMode := fa.FileInfo.Mode() 802 | lastSize := fa.FileInfo.Size() 803 | lastModTime := fa.FileInfo.ModTime().Unix() 804 | 805 | fileinfo, err := fa.File.Stat() 806 | if err != nil { 807 | log.Printf("file %s stat FAIL: %v", err) 808 | fa.UnchangeTime += 1 809 | return false 810 | } 811 | 812 | thisMode := fileinfo.Mode() 813 | thisSize := fileinfo.Size() 814 | thisModTime := fileinfo.ModTime().Unix() 815 | thisTailable := fileinfo.Mode().IsRegular() 816 | 817 | if lastMode == thisMode && 818 | (!thisTailable || lastSize == thisSize) && 819 | lastModTime == thisModTime { 820 | 821 | fa.UnchangeTime += 1 822 | return false 823 | } 824 | 825 | // replace the FileInfo for reading the new content 826 | fa.UnchangeTime = 0 827 | fa.FileInfo = fileinfo 828 | return true 829 | } 830 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-server/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * config.go - the structure of configuration and related functions 3 | * 4 | * history 5 | * -------------------- 6 | * 2018/1/11, by Ye Zhiqin, create 7 | * 8 | */ 9 | package main 10 | 11 | import ( 12 | "gopkg.in/yaml.v2" 13 | "io/ioutil" 14 | "log" 15 | ) 16 | 17 | type Config struct { 18 | RedisServer string `yaml:"redis"` 19 | FalconAgent string `yaml:"agent"` 20 | Endpoint string `yaml:"endpoint"` 21 | Tags string `yaml:"tags"` 22 | CounterType string `yaml:"counterType"` 23 | Step int64 `yaml:"step"` 24 | } 25 | 26 | var config *Config 27 | 28 | /* 29 | * LoadConfig - load configuration file to Config struct 30 | * 31 | * PARAMS: 32 | * No paramter 33 | * 34 | * RETURNS: 35 | * nil, if error ocurred 36 | * *Config, if succeed 37 | */ 38 | func LoadConfig() *Config { 39 | cfg := new(Config) 40 | buf, err := ioutil.ReadFile("config.yaml") 41 | if err != nil { 42 | log.Printf("configuration file reading FAIL: %s", err.Error()) 43 | return nil 44 | } 45 | if err := yaml.Unmarshal(buf, cfg); err != nil { 46 | log.Printf("yaml file unmarshal FAIL: %s", err.Error()) 47 | return nil 48 | } 49 | log.Printf("config: %v", cfg) 50 | 51 | // check configuration 52 | if cfg.RedisServer == "" { 53 | log.Printf("redis server address should not EMPTY!") 54 | return nil 55 | } 56 | 57 | if cfg.FalconAgent == "" { 58 | log.Printf("falcon agent address should not EMPTY!") 59 | return nil 60 | } 61 | 62 | return cfg 63 | } 64 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-server/config.yaml: -------------------------------------------------------------------------------- 1 | redis: "127.0.0.1:6379" 2 | agent: "http://127.0.0.1:1988/v1/push" 3 | endpoint: "endpoint.name" 4 | tags: "tag1=t1,tag2=t2" 5 | counterType: "GAUGE" 6 | step: 60 7 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-server/control: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | WORKSPACE=$(cd $(dirname $0)/; pwd) 4 | cd ${WORKSPACE} 5 | 6 | app=msgget-monitor-server 7 | pidfile=${app}.pid 8 | logfile=${app}.log 9 | 10 | function check_pid() { 11 | if [[ -f ${pidfile} ]];then 12 | pid=$(cat ${pidfile}) 13 | if [[ -n ${pid} ]]; then 14 | running=$(ps -p $pid|grep -v "PID TTY" |wc -l) 15 | return ${running} 16 | fi 17 | fi 18 | return 0 19 | } 20 | 21 | function start() { 22 | check_pid 23 | running=$? 24 | if [[ ${running} -gt 0 ]];then 25 | echo -n "${app} now is running already, pid=" 26 | cat ${pidfile} 27 | return 1 28 | fi 29 | 30 | nohup ./${app} &>${logfile} & 31 | echo $! > ${pidfile} 32 | echo "${app} started..., pid=$!" 33 | } 34 | 35 | function stop() { 36 | pid=$(cat ${pidfile}) 37 | kill ${pid} 38 | rm ${pidfile} 39 | echo "${app} stoped..." 40 | } 41 | 42 | function restart() { 43 | stop 44 | sleep 1 45 | start 46 | } 47 | 48 | function help() { 49 | echo "$0 pid|start|stop|restart" 50 | } 51 | 52 | function pid() { 53 | cat ${pidfile} 54 | } 55 | 56 | if [ "$1" == "" ]; then 57 | help 58 | elif [ "$1" == "stop" ];then 59 | stop 60 | elif [ "$1" == "start" ];then 61 | start 62 | elif [ "$1" == "restart" ];then 63 | restart 64 | elif [ "$1" == "pid" ];then 65 | pid 66 | else 67 | help 68 | fi 69 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-server/falcon.go: -------------------------------------------------------------------------------- 1 | /* 2 | * falcon.go - the data structure of open falcon and related functions 3 | * 4 | * history 5 | * -------------------- 6 | * 2017/8/18, by Ye Zhiqin, create 7 | * 8 | * DESCRIPTION 9 | * This file contains the definition of open falcon data structure 10 | * and the function to push data to open falcon 11 | */ 12 | 13 | package main 14 | 15 | import ( 16 | "bytes" 17 | "encoding/json" 18 | "fmt" 19 | "io/ioutil" 20 | "log" 21 | "net/http" 22 | "os" 23 | "time" 24 | ) 25 | 26 | type FalconData struct { 27 | Endpoint string `json:"endpoint"` 28 | Metric string `json:"metric"` 29 | Tags string `json:"tags"` 30 | CounterType string `json:"counterType"` 31 | Step int64 `json:"step"` 32 | Timestamp int64 `json:"timestamp"` 33 | Value interface{} `json:"value"` 34 | } 35 | 36 | /* 37 | * SetValue - set FalconData value 38 | * 39 | * RECEIVER: *FalconData 40 | * 41 | * PARAMS: 42 | * - v: value 43 | * 44 | * RETURNS: 45 | * No return value 46 | */ 47 | func (data *FalconData) SetValue(v interface{}) { 48 | data.Value = v 49 | } 50 | 51 | /* 52 | * String - generate a new FalconData 53 | * 54 | * RECEIVER: *FalconData 55 | * 56 | * PARAMS: 57 | * No paramter 58 | * 59 | * RETURNS: 60 | * - string: string to display 61 | */ 62 | func (data *FalconData) String() string { 63 | s := fmt.Sprintf("FalconData Endpoint:%s Metric:%s Tags:%s CounterType:%s Step:%d Timestamp:%d Value:%v", 64 | data.Endpoint, data.Metric, data.Tags, data.CounterType, data.Step, data.Timestamp, data.Value) 65 | return s 66 | } 67 | 68 | /* 69 | * NewFalconData - generate a new FalconData 70 | * 71 | * PARAMS: 72 | * - metric 73 | * - endpoint 74 | * - value 75 | * - counterType 76 | * - timestamp 77 | * - step 78 | * 79 | * RETURNS: 80 | * - *FalconData 81 | */ 82 | func NewFalconData(endpoint string, metric string, tags string, counterType string, step int64, timestamp int64, value interface{}) *FalconData { 83 | point := &FalconData{ 84 | Endpoint: GetEndpoint(endpoint), 85 | Metric: metric, 86 | Tags: tags, 87 | CounterType: counterType, 88 | Step: step, 89 | Timestamp: GetTimestamp(timestamp), 90 | } 91 | point.SetValue(value) 92 | return point 93 | } 94 | 95 | /* 96 | * GetEndpoint - generate endpoint value 97 | * 98 | * PARAMS: 99 | * - endpoint 100 | * 101 | * RETURNS: 102 | * - endpoint: if endpoint is avaliable 103 | * - hostname: if endpoint is empty 104 | * - localhost: if endpoint is empty and can't get hostname 105 | */ 106 | func GetEndpoint(endpoint string) string { 107 | if endpoint != "" { 108 | return endpoint 109 | } 110 | 111 | hostname, err := os.Hostname() 112 | if err != nil { 113 | hostname = "localhost" 114 | } 115 | return hostname 116 | } 117 | 118 | /* 119 | * GetTimestamp - generate timestamp value 120 | * 121 | * PARAMS: 122 | * - timestamp 123 | * 124 | * RETURNS: 125 | * - timestamp: if timestamp > 0 126 | * - now: if timestamp <= 0 127 | */ 128 | func GetTimestamp(timestamp int64) int64 { 129 | if timestamp > 0 { 130 | return timestamp 131 | } else { 132 | return time.Now().Unix() 133 | } 134 | } 135 | 136 | /* 137 | * PushData - push data to open falcon 138 | * 139 | * PARAMS: 140 | * - api: url of agent or transfer 141 | * - data: an array of FalconData 142 | * 143 | * RETURNS: 144 | * - []byte, nil: if succeed 145 | * - nil, error: if fail 146 | */ 147 | func PushData(api string, data []*FalconData) ([]byte, error) { 148 | points, err := json.Marshal(data) 149 | if err != nil { 150 | log.Printf("data marshaling FAIL: %v", err) 151 | return nil, err 152 | } 153 | 154 | response, err := http.Post(api, "Content-Type: application/json", bytes.NewBuffer(points)) 155 | if err != nil { 156 | log.Printf("api call FAIL: %v", err) 157 | return nil, err 158 | } 159 | defer response.Body.Close() 160 | 161 | content, err := ioutil.ReadAll(response.Body) 162 | if err != nil { 163 | return nil, err 164 | } 165 | 166 | return content, nil 167 | } 168 | 169 | /* 170 | * Push2Falcon - push file agent data 171 | * 172 | * PARAMS: 173 | * - api: falcon agent address 174 | * - data: AggrData 175 | */ 176 | func Push2Falcon(api string, data *AggrData) { 177 | var d []*FalconData 178 | 179 | // request.count.sum 180 | point := NewFalconData(data.Endpoint, "request.count.sum", data.Tags, data.CounterType, data.Step, data.Timestamp, data.ReqSum) 181 | d = append(d, point) 182 | 183 | // request.200.count.sum 184 | point = NewFalconData(data.Endpoint, "request.200.count.sum", data.Tags, data.CounterType, data.Step, data.Timestamp, data.Req200Sum) 185 | d = append(d, point) 186 | 187 | // request.499.count.sum 188 | point = NewFalconData(data.Endpoint, "request.499.count.sum", data.Tags, data.CounterType, data.Step, data.Timestamp, data.Req499Sum) 189 | d = append(d, point) 190 | 191 | // request.500.count.sum 192 | point = NewFalconData(data.Endpoint, "request.500.count.sum", data.Tags, data.CounterType, data.Step, data.Timestamp, data.Req500Sum) 193 | d = append(d, point) 194 | 195 | // request.502.count.sum 196 | point = NewFalconData(data.Endpoint, "request.502.count.sum", data.Tags, data.CounterType, data.Step, data.Timestamp, data.Req502Sum) 197 | d = append(d, point) 198 | 199 | // request.504.count.sum 200 | point = NewFalconData(data.Endpoint, "request.504.count.sum", data.Tags, data.CounterType, data.Step, data.Timestamp, data.Req504Sum) 201 | d = append(d, point) 202 | 203 | // ip.count.sum 204 | point = NewFalconData(data.Endpoint, "ip.count.sum", data.Tags, data.CounterType, data.Step, data.Timestamp, data.IpSum) 205 | d = append(d, point) 206 | 207 | // shopid.count.sum 208 | point = NewFalconData(data.Endpoint, "shopid.count.sum", data.Tags, data.CounterType, data.Step, data.Timestamp, data.IdSum) 209 | d = append(d, point) 210 | 211 | //log.Printf("falcon points: %v", d) 212 | response, err := PushData(api, d) 213 | if err != nil { 214 | log.Printf("push data to falcon FAIL: %s", err.Error()) 215 | } 216 | log.Printf("push data to falcon succeed: %s", string(response)) 217 | return 218 | } 219 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-server/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * main.go - the entry of program 3 | * 4 | * history 5 | * -------------------- 6 | * 2018/1/12, by Ye Zhiqin, create 7 | * 8 | */ 9 | 10 | package main 11 | 12 | import ( 13 | //"bytes" 14 | "log" 15 | "os" 16 | "os/signal" 17 | "syscall" 18 | "time" 19 | ) 20 | 21 | type AggrData struct { 22 | Endpoint string 23 | Tags string 24 | CounterType string 25 | Step int64 26 | 27 | Timestamp int64 28 | 29 | ReqSum int64 30 | Req200Sum int64 31 | Req499Sum int64 32 | Req500Sum int64 33 | Req502Sum int64 34 | Req504Sum int64 35 | IpSum int64 36 | IdSum int64 37 | } 38 | 39 | func Timeup(data *AggrData, t time.Time) { 40 | d2m, err := time.ParseDuration("-2m") 41 | if err != nil { 42 | log.Printf("calculate -2m duration FAIL: %s", err.Error()) 43 | return 44 | } 45 | 46 | d62m, err := time.ParseDuration("-62m") 47 | if err != nil { 48 | log.Printf("calculate -62m duration FAIL: %s", err.Error()) 49 | return 50 | } 51 | 52 | t2m := t.Add(d2m) 53 | t62m := t.Add(d62m) 54 | 55 | dp := t62m.Format("200601021504") 56 | rp := t2m.Format("200601021504") 57 | data.Timestamp = t2m.Unix() 58 | 59 | DelFromRedis(config.RedisServer, dp) 60 | ReadFromRedis(config.RedisServer, rp, data) 61 | Push2Falcon(config.FalconAgent, data) 62 | 63 | data.ReqSum = 0 64 | data.Req200Sum = 0 65 | data.Req499Sum = 0 66 | data.Req500Sum = 0 67 | data.Req502Sum = 0 68 | data.Req504Sum = 0 69 | data.IpSum = 0 70 | data.IdSum = 0 71 | } 72 | 73 | // main 74 | func main() { 75 | sysCh := make(chan os.Signal, 1) 76 | signal.Notify(sysCh, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) 77 | defer close(sysCh) 78 | 79 | // set log format 80 | log.SetFlags(log.LstdFlags | log.Lshortfile) 81 | 82 | // load configuration 83 | config = LoadConfig() 84 | if config == nil { 85 | log.Printf("configuration loading FAIL, please check the config.yaml") 86 | os.Exit(-1) 87 | } 88 | 89 | // create one minute ticker 90 | ticker := time.NewTicker(time.Minute) 91 | defer ticker.Stop() 92 | 93 | data := new(AggrData) 94 | data.Endpoint = config.Endpoint 95 | data.Tags = config.Tags 96 | data.CounterType = config.CounterType 97 | data.Step = config.Step 98 | 99 | MAIN: 100 | for { 101 | select { 102 | case <-sysCh: 103 | log.Printf("system signal: %s", sysCh) 104 | break MAIN 105 | case t := <-ticker.C: 106 | Timeup(data, t) 107 | default: 108 | time.Sleep(time.Second) 109 | } 110 | } 111 | 112 | log.Printf("msgger-monitor-server exit...") 113 | } 114 | -------------------------------------------------------------------------------- /msgget-monitor/msgget-monitor-server/redis.go: -------------------------------------------------------------------------------- 1 | /* 2 | * redis.go - functions to push data to redis 3 | * 4 | * history 5 | * -------------------- 6 | * 2017/1/11, by Ye Zhiqin, create 7 | * 8 | */ 9 | 10 | package main 11 | 12 | import ( 13 | "log" 14 | 15 | "github.com/garyburd/redigo/redis" 16 | ) 17 | 18 | func Get(endpoint string, key string) int64 { 19 | conn, err := redis.Dial("tcp", endpoint) 20 | if err != nil { 21 | log.Printf("connect to redis %s FAiL", endpoint) 22 | return -1 23 | } 24 | defer conn.Close() 25 | 26 | result, err := redis.Int64(conn.Do("GET", key)) 27 | if err != nil { 28 | log.Printf("Execute: GET %s FAiL", key) 29 | return -1 30 | } 31 | log.Printf("Result: %d, Execute: GET %s", result, key) 32 | return result 33 | } 34 | 35 | func Scard(endpoint string, key string) int64 { 36 | conn, err := redis.Dial("tcp", endpoint) 37 | if err != nil { 38 | log.Printf("connect to redis %s FAiL", endpoint) 39 | return -1 40 | } 41 | defer conn.Close() 42 | 43 | result, err := redis.Int64(conn.Do("SCARD", key)) 44 | if err != nil { 45 | log.Printf("Execute: SCARD %s FAiL", key) 46 | return -1 47 | } 48 | log.Printf("Result: %d, Execute: SCARD %s", result, key) 49 | return result 50 | } 51 | 52 | func Del(endpoint string, key string) { 53 | conn, err := redis.Dial("tcp", endpoint) 54 | if err != nil { 55 | log.Printf("connect to redis %s FAiL", endpoint) 56 | return 57 | } 58 | defer conn.Close() 59 | 60 | result, err := redis.Int(conn.Do("DEL", key)) 61 | if err != nil { 62 | log.Printf("Execute: DEL %s FAiL", key) 63 | return 64 | } 65 | log.Printf("Result: %d, Execute: DEL %s", result, key) 66 | return 67 | } 68 | 69 | func DelFromRedis(endpoint string, prefix string) { 70 | key := "request-" + prefix 71 | Del(endpoint, key) 72 | 73 | key = "request200-" + prefix 74 | Del(endpoint, key) 75 | 76 | key = "request499-" + prefix 77 | Del(endpoint, key) 78 | 79 | key = "request500-" + prefix 80 | Del(endpoint, key) 81 | 82 | key = "request502-" + prefix 83 | Del(endpoint, key) 84 | 85 | key = "request504-" + prefix 86 | Del(endpoint, key) 87 | 88 | key = "ip-" + prefix 89 | Del(endpoint, key) 90 | 91 | key = "shopid-" + prefix 92 | Del(endpoint, key) 93 | } 94 | 95 | func ReadFromRedis(endpoint string, prefix string, data *AggrData) { 96 | key := "request-" + prefix 97 | data.ReqSum = Get(endpoint, key) 98 | 99 | key = "request200-" + prefix 100 | data.Req200Sum = Get(endpoint, key) 101 | 102 | key = "request499-" + prefix 103 | data.Req499Sum = Get(endpoint, key) 104 | 105 | key = "request500-" + prefix 106 | data.Req500Sum = Get(endpoint, key) 107 | 108 | key = "request502-" + prefix 109 | data.Req502Sum = Get(endpoint, key) 110 | 111 | key = "request504-" + prefix 112 | data.Req504Sum = Get(endpoint, key) 113 | 114 | key = "ip-" + prefix 115 | data.IpSum = Scard(endpoint, key) 116 | 117 | key = "shopid-" + prefix 118 | data.IdSum = Scard(endpoint, key) 119 | } 120 | -------------------------------------------------------------------------------- /mysqlmon/falcon.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "os" 11 | "time" 12 | ) 13 | 14 | type FalconData struct { 15 | Metric string `json:"metric"` 16 | Endpoint string `json:"endpoint"` 17 | Value interface{} `json:"value"` 18 | CounterType string `json:"counterType"` 19 | Tags string `json:"tags"` 20 | Timestamp int64 `json:"timestamp"` 21 | Step int64 `json:"step"` 22 | } 23 | 24 | func (data *FalconData) SetValue(v interface{}) { 25 | data.Value = v 26 | } 27 | 28 | func (data *FalconData) String() string { 29 | s := fmt.Sprintf("FalconData Metric:%s Endpoint:%s Value:%v CounterType:%s Tags:%s Timestamp:%d Step:%d", 30 | data.Metric, data.Endpoint, data.Value, data.CounterType, data.Tags, data.Timestamp, data.Step) 31 | return s 32 | } 33 | 34 | func NewFalconData(metric string, endpoint string, value interface{}, counterType string, tags string, timestamp int64, step int64) *FalconData { 35 | point := &FalconData{ 36 | Metric: metric, 37 | Endpoint: GetEndpoint(endpoint), 38 | CounterType: counterType, 39 | Tags: tags, 40 | Timestamp: GetTimestamp(timestamp), 41 | Step: step, 42 | } 43 | point.SetValue(value) 44 | return point 45 | } 46 | 47 | func GetEndpoint(endpoint string) string { 48 | if endpoint != "" { 49 | return endpoint 50 | } 51 | 52 | hostname, err := os.Hostname() 53 | if err != nil { 54 | hostname = "localhost" 55 | } 56 | return hostname 57 | } 58 | 59 | func GetTimestamp(timestamp int64) int64 { 60 | if timestamp > 0 { 61 | return timestamp 62 | } else { 63 | return time.Now().Unix() 64 | } 65 | } 66 | 67 | func PushToFalcon(api string, data []*FalconData) ([]byte, error) { 68 | points, err := json.Marshal(data) 69 | if err != nil { 70 | log.Printf("fail to convert data to json: %s", err.Error()) 71 | return nil, err 72 | } 73 | 74 | response, err := http.Post(api, "Content-Type: application/json", bytes.NewBuffer(points)) 75 | if err != nil { 76 | log.Printf("fail to call falcon API: %s", err.Error()) 77 | return nil, err 78 | } 79 | defer response.Body.Close() 80 | 81 | content, err := ioutil.ReadAll(response.Body) 82 | if err != nil { 83 | return nil, err 84 | } 85 | 86 | return content, nil 87 | } 88 | -------------------------------------------------------------------------------- /mysqlmon/instance.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | 7 | "gopkg.in/yaml.v2" 8 | ) 9 | 10 | type MySQLs struct { 11 | Instances []Instance `yaml:"instances"` 12 | } 13 | 14 | type Instance struct { 15 | Name string `yaml:"name"` 16 | IP string `yaml:"ip"` 17 | Port int `yaml:"port"` 18 | Enabled bool `yaml:"enabled"` 19 | } 20 | 21 | var DBList *MySQLs 22 | 23 | func LoadDBList() *MySQLs { 24 | dblist := new(MySQLs) 25 | 26 | buf, err := ioutil.ReadFile("instance.yaml") 27 | if err != nil { 28 | log.Printf("fail to read instance.yaml: %s", err.Error()) 29 | return nil 30 | } 31 | if err := yaml.Unmarshal(buf, dblist); err != nil { 32 | log.Printf("fail to unmarshal dblist: %s", err.Error()) 33 | return nil 34 | } 35 | log.Printf("load dblist successfully: %v", dblist) 36 | 37 | return dblist 38 | } 39 | -------------------------------------------------------------------------------- /mysqlmon/instance.yaml: -------------------------------------------------------------------------------- 1 | instances: 2 | - name: "192.168.1.2:3306" 3 | ip: "192.168.1.2" 4 | port: 3306 5 | enabled: true 6 | - name: "192.168.1.3:3306" 7 | ip: "192.168.1.3" 8 | port: 3306 9 | enabled: false 10 | -------------------------------------------------------------------------------- /mysqlmon/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net" 7 | "time" 8 | ) 9 | 10 | func main() { 11 | log.SetFlags(log.LstdFlags | log.Lshortfile) 12 | 13 | fmt.Println("=====mysqlmon=====") 14 | fmt.Printf("Start:%s\n", time.Now().Format("200601021504")) 15 | 16 | DBList = LoadDBList() 17 | if DBList == nil { 18 | panic("fail to load mysql instance configuration") 19 | } 20 | 21 | api := "http://127.0.0.1:1988/v1/push" 22 | from := GetEndpoint("") 23 | endpoint := "mysqlmon" 24 | metric := "mysql.alive" 25 | ts := time.Now().Unix() 26 | 27 | var data []*FalconData 28 | 29 | for _, instance := range DBList.Instances { 30 | if instance.Enabled { 31 | tags := fmt.Sprintf("from=%s,ip=%s,port=%d", from, instance.IP, instance.Port) 32 | if ok := DialMySQL(instance.Name); ok { 33 | fmt.Printf("MySQL %s is ok.\n", instance.Name) 34 | point := NewFalconData(metric, endpoint, 1, "GAUGE", tags, ts, 60) 35 | data = append(data, point) 36 | } else { 37 | fmt.Printf("MySQL %s is DOWN!!!\n", instance.Name) 38 | point := NewFalconData(metric, endpoint, 0, "GAUGE", tags, ts, 60) 39 | data = append(data, point) 40 | } 41 | } 42 | } 43 | 44 | response, err := PushToFalcon(api, data) 45 | if err != nil { 46 | log.Printf("fail to push falcon data: %s", err.Error) 47 | } 48 | fmt.Printf("push data to falcon successfully: %s\n", string(response)) 49 | 50 | fmt.Printf("End:%s\n", time.Now().Format("200601021504")) 51 | return 52 | } 53 | 54 | func DialMySQL(address string) bool { 55 | conn, err := net.DialTimeout("tcp", address, time.Second*5) 56 | if err != nil { 57 | log.Printf("fail to connect mysql: %s", err.Error()) 58 | return false 59 | } 60 | fmt.Printf("connect to mysql successfully: %s\n", conn.RemoteAddr().String()) 61 | conn.Close() 62 | return true 63 | } 64 | -------------------------------------------------------------------------------- /oom-checker/checker.sh: -------------------------------------------------------------------------------- 1 | set -x 2 | 3 | WORKSPACE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | cd ${WORKSPACE} 5 | 6 | HOST=127.0.0.1 7 | 8 | REDIS_CLI=./redis-cli 9 | REDIS_IP=127.0.0.1 10 | REDIS_PORT=6379 11 | 12 | OOM=$(${REDIS_CLI} -h ${REDIS_IP} -p ${REDIS_PORT} "GET" "oom.key") 13 | 14 | if [[ -z ${OOM} ]];then 15 | OOM=0 16 | fi 17 | 18 | if [ "${OOM}" -gt "0" ];then 19 | echo "OOM=${OOM} Restart APP!" 20 | docker restart container-name 21 | fi 22 | 23 | ${REDIS_CLI} -h ${REDIS_IP} -p ${REDIS_PORT} "DEL" "oom.key" 24 | -------------------------------------------------------------------------------- /oom-checker/collector.awk: -------------------------------------------------------------------------------- 1 | /OutOfMemoryError/ { 2 | print "INCR oom.key" 3 | } 4 | -------------------------------------------------------------------------------- /oom-checker/collector.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | LOG=/path/to/app.log 4 | 5 | PATTERNS="java.lang.OutOfMemoryError" 6 | 7 | REDIS_CLI=./redis-cli 8 | REDIS_IP=127.0.0.1 9 | REDIS_PORT=6379 10 | 11 | stdbuf -oL tail -F ${LOG} |stdbuf -oL grep -oE "${PATTERNS}" | stdbuf -oL awk -f collector.awk | "${REDIS_CLI}" -h "${REDIS_IP}" -p "${REDIS_PORT}" 12 | -------------------------------------------------------------------------------- /order-count/order_count.conf: -------------------------------------------------------------------------------- 1 | # mysql 2 | mysql_client=/path/to/mysql/bin/mysql 3 | 4 | # db 5 | db_ip=127.0.0.1 6 | db_port=3306 7 | db_username=username 8 | db_password=password 9 | db_database=dbname 10 | db_table=tablename 11 | 12 | # query 13 | end=$(date +%Y%m%d%H%M%S) 14 | start=$(date +%Y%m%d%H%M%S -d "1 minute ago") 15 | sql="select ...;" 16 | 17 | # falcon 18 | falcon_ip=127.0.0.1 19 | falcon_port=1988 20 | falcon_api=http://${falcon_ip}:${falcon_port}/v1/push 21 | 22 | endpoint=$(hostname) 23 | metric='metric' 24 | timestamp=$(date +%s) 25 | step=60 26 | counterType=GAUGE 27 | tags="module=mysql" 28 | -------------------------------------------------------------------------------- /order-count/order_count.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BIN_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | 5 | source ${BIN_PATH}/order_count.conf 6 | 7 | # query from mysql 8 | result=$(${mysql_client} -h${db_ip} -P${db_port} -u${db_username} -p${db_password} -e"use ${db_database}; ${sql}" | grep -v "-" | grep -v "+" | grep -v "in set" | grep -v "orderCount" | grep -v "orderTotalAmount") 9 | 10 | orderCount=$(echo ${result} | awk '{print $1}') 11 | if [[ -z "${orderCount}" || 0 -eq "${orderCount}" ]];then 12 | orderTotalAmount=0 13 | else 14 | orderTotalAmount=$(echo ${result} | awk '{print $2}') 15 | fi 16 | 17 | # push to falcon 18 | curl -X POST -d "[{\"metric\": \"order.count\", \"endpoint\": \"${endpoint}\", \"timestamp\": ${timestamp},\"step\": ${step},\"value\": ${orderCount},\"counterType\": \"${counterType}\",\"tags\": \"${tags}\"}]" ${falcon_api} 19 | 20 | usleep 10 21 | 22 | curl -X POST -d "[{\"metric\": \"order.amount\", \"endpoint\": \"${endpoint}\", \"timestamp\": ${timestamp},\"step\": ${step},\"value\": ${orderTotalAmount},\"counterType\": \"${counterType}\",\"tags\": \"${tags}\"}]" ${falcon_api} 23 | -------------------------------------------------------------------------------- /proc-killer/proc-killer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RETRY=3 4 | HIT=0 5 | THRESHOLD=50 6 | 7 | PNAME="/path/to/proc/procname" 8 | PROC="proc" 9 | 10 | # get process id 11 | PID=$(ps -ef | grep "${PNAME}" | grep -v "grep" | awk '{print $2}') 12 | 13 | if [[ -z ${PID} ]];then 14 | echo "Can not find the process ID!" 15 | exit 1 16 | fi 17 | 18 | if [[ 1 -eq ${PID} ]];then 19 | echo "Process ID 1 is unexpected!" 20 | exit 1 21 | fi 22 | 23 | # get process cpu usage 24 | for i in $(seq 1 ${RETRY});do 25 | CPU=$(ps -p "${PID}" -o pcpu --no-header) 26 | sleep 1 27 | echo "Try: ${i} CPU: ${CPU}" 28 | FLAG=$(echo "${CPU}>${THRESHOLD}" | bc) 29 | if [[ 1 -eq ${FLAG} ]];then 30 | HIT=$((HIT+1)) 31 | fi 32 | done 33 | 34 | # kill process when cpu is busy 35 | if [[ ${RETRY} -eq ${HIT} ]];then 36 | kill ${PID} 37 | sleep 5 38 | COMMAND=$(ps -p ${PID} -o comm --no-header) 39 | if [[ ${PROC} == ${COMMAND} ]];then 40 | kill ${PID} 41 | sleep 5 42 | COMMAND=$(ps -p ${PID} -o comm --no-header) 43 | if [[ ${PROC} == ${COMMAND} ]];then 44 | kill -9 ${PID} 45 | echo "kill process ${PID} force. Please check!!!" 46 | exit 2 47 | else 48 | echo "kill process ${PID} normally." 49 | exit 0 50 | fi 51 | else 52 | echo "kill process ${PID} normally." 53 | exit 0 54 | fi 55 | fi 56 | 57 | echo "CPU is not busy, killer go away." 58 | exit 0 59 | -------------------------------------------------------------------------------- /redismon/check.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | "strconv" 7 | "strings" 8 | "time" 9 | 10 | "github.com/garyburd/redigo/redis" 11 | ) 12 | 13 | var ErrConn = errors.New("failed to connect redis") 14 | 15 | func GetInfo(endpoint string, timeout time.Duration) (string, error) { 16 | opt := redis.DialConnectTimeout(timeout) 17 | conn, err := redis.Dial("tcp", endpoint, opt) 18 | if err != nil { 19 | log.Printf("failed to connect to redis %s: %s", endpoint, err.Error()) 20 | return "", ErrConn 21 | } 22 | defer conn.Close() 23 | 24 | info, err := redis.String(conn.Do("INFO")) 25 | if err != nil { 26 | log.Printf("failed to get info from redis %s: %s", endpoint, err.Error()) 27 | return "", err 28 | } 29 | return info, nil 30 | } 31 | 32 | func GetMaxMemory(endpoint string, timeout time.Duration) int64 { 33 | opt := redis.DialConnectTimeout(timeout) 34 | conn, err := redis.Dial("tcp", endpoint, opt) 35 | if err != nil { 36 | log.Printf("failed to connect to redis %s: %s", endpoint, err.Error()) 37 | return 0 38 | } 39 | defer conn.Close() 40 | 41 | mm, err := redis.Strings(conn.Do("config", "get", "maxmemory")) 42 | if err != nil { 43 | log.Printf("failed to get max memory from redis %s: %s", endpoint, err.Error()) 44 | return 0 45 | } 46 | 47 | if len(mm) != 2 { 48 | log.Printf("unexpect max memory result.") 49 | return 0 50 | } 51 | max, err := strconv.ParseInt(mm[1], 10, 64) 52 | if err != nil { 53 | log.Printf("failed to parse max memory: %s", err.Error()) 54 | return 0 55 | } 56 | return max 57 | } 58 | 59 | func GetMaxClients(endpoint string, timeout time.Duration) int64 { 60 | opt := redis.DialConnectTimeout(timeout) 61 | conn, err := redis.Dial("tcp", endpoint, opt) 62 | if err != nil { 63 | log.Printf("failed to connect to redis %s: %s", endpoint, err.Error()) 64 | return 0 65 | } 66 | defer conn.Close() 67 | 68 | mc, err := redis.Strings(conn.Do("config", "get", "maxclients")) 69 | if err != nil { 70 | log.Printf("failed to get max client from redis %s: %s", endpoint, err.Error()) 71 | return 0 72 | } 73 | 74 | if len(mc) != 2 { 75 | log.Printf("unexpect max clients result.") 76 | return 0 77 | } 78 | max, err := strconv.ParseInt(mc[1], 10, 64) 79 | if err != nil { 80 | log.Printf("failed to parse max client: %s", err.Error()) 81 | return 0 82 | } 83 | return max 84 | } 85 | 86 | func ParseMetrics(info string) (version, max_memory, used_memory, connected_clients, blocked_clients int64) { 87 | version = int64(3) 88 | max_memory = int64(0) 89 | used_memory = int64(0) 90 | connected_clients = int64(0) 91 | blocked_clients = int64(0) 92 | 93 | elements := strings.Split(info, "\n") 94 | for _, elem := range elements { 95 | if strings.HasPrefix(elem, "redis_version:") { 96 | kv := strings.Split(elem, ":") 97 | if strings.Compare(kv[1], "2") <= 0 { 98 | version = 2 99 | } 100 | } else if strings.HasPrefix(elem, "maxmemory:") { 101 | kv := strings.Split(elem, ":") 102 | v, err := strconv.ParseInt(strings.TrimSpace(kv[1]), 10, 64) 103 | if err != nil { 104 | log.Printf("failed to parse maxmemory: %s", err.Error()) 105 | continue 106 | } 107 | max_memory = v 108 | } else if strings.HasPrefix(elem, "used_memory:") { 109 | kv := strings.Split(elem, ":") 110 | v, err := strconv.ParseInt(strings.TrimSpace(kv[1]), 10, 64) 111 | if err != nil { 112 | log.Printf("failed to parse used_memory: %s", err.Error()) 113 | continue 114 | } 115 | used_memory = v 116 | } else if strings.HasPrefix(elem, "connected_clients:") { 117 | kv := strings.Split(elem, ":") 118 | v, err := strconv.ParseInt(strings.TrimSpace(kv[1]), 10, 64) 119 | if err != nil { 120 | log.Printf("failed to parse connected_clients: %s", err.Error()) 121 | continue 122 | } 123 | connected_clients = v 124 | } else if strings.HasPrefix(elem, "blocked_clients:") { 125 | kv := strings.Split(elem, ":") 126 | v, err := strconv.ParseInt(strings.TrimSpace(kv[1]), 10, 64) 127 | if err != nil { 128 | log.Printf("failed to parse blocked_clients: %s", err.Error()) 129 | continue 130 | } 131 | blocked_clients = v 132 | } else { 133 | continue 134 | } 135 | } 136 | return 137 | } 138 | 139 | func CheckRedis(endpoint string, timeout time.Duration) (ok bool, version, max_memory, used_memory, memory_used_percent, max_clients, connected_clients, blocked_clients int64) { 140 | ok = true 141 | version = int64(3) 142 | max_memory = int64(0) 143 | used_memory = int64(0) 144 | memory_used_percent = int64(0) 145 | max_clients = int64(0) 146 | connected_clients = int64(0) 147 | blocked_clients = int64(0) 148 | 149 | info, err := GetInfo(endpoint, timeout) 150 | if err == ErrConn { 151 | ok = false 152 | return 153 | } 154 | if err != nil { 155 | log.Printf("failed to get info") 156 | return 157 | } 158 | 159 | version, max_memory, used_memory, connected_clients, blocked_clients = ParseMetrics(info) 160 | 161 | if version <= 2 { 162 | max_memory = GetMaxMemory(endpoint, timeout) 163 | } 164 | 165 | max_clients = GetMaxClients(endpoint, timeout) 166 | 167 | memory_used_percent = int64(0) 168 | if max_memory > 0 { 169 | memory_used_percent = used_memory * int64(100) / max_memory 170 | } 171 | 172 | return 173 | } 174 | -------------------------------------------------------------------------------- /redismon/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "log" 7 | ) 8 | 9 | var Cfg *Config 10 | 11 | type Config struct { 12 | Lepus string `json:"lepus"` 13 | Manager string `json:"manager"` 14 | Timeout int `json:"timeout"` 15 | Falcon FalconConfig `json:"falcon"` 16 | } 17 | 18 | type FalconConfig struct { 19 | API string `json:"api"` 20 | Endpoint string `json:"endpoint"` 21 | Step int64 `json:"step"` 22 | } 23 | 24 | // Load configuration from config.json 25 | func LoadConfig() *Config { 26 | cfg := new(Config) 27 | 28 | buf, err := ioutil.ReadFile("config.json") 29 | if err != nil { 30 | log.Printf("fail to read config.json: %s", err.Error()) 31 | return nil 32 | } 33 | if err := json.Unmarshal(buf, cfg); err != nil { 34 | log.Printf("fail to unmarshal config.json: %s", err.Error()) 35 | return nil 36 | } 37 | log.Printf("load application configuration successfully: %v", cfg) 38 | 39 | return cfg 40 | } 41 | -------------------------------------------------------------------------------- /redismon/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "lepus": "username:password@tcp(127.0.0.1:3306)/lepus?charset=utf8&parseTime=True&loc=Local", 3 | "manager": "username:password(127.0.0.1:3306)/db_manager?charset=utf8&parseTime=True&loc=Local", 4 | "timeout": 10, 5 | "falcon": { 6 | "api":"http://127.0.0.1:1988/v1/push", 7 | "endpoint": "redismon", 8 | "step": 60 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /redismon/database.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "errors" 6 | "log" 7 | 8 | _ "github.com/go-sql-driver/mysql" 9 | ) 10 | 11 | type Instance struct { 12 | Host string 13 | Port string 14 | Tags string 15 | Role string 16 | } 17 | 18 | var nilDbErr = errors.New("db is nil") 19 | 20 | func Connect(dsn string) (*sql.DB, error) { 21 | db, err := sql.Open("mysql", dsn) 22 | if err != nil { 23 | return nil, err 24 | } 25 | if db == nil { 26 | return nil, nilDbErr 27 | } 28 | return db, nil 29 | } 30 | 31 | func GetInstances(db *sql.DB) ([]*Instance, error) { 32 | instances := []*Instance{} 33 | 34 | sql := "SELECT db_servers_redis.host AS host, db_servers_redis.port AS port, db_servers_redis.tags AS tags, redis_status.redis_role AS role FROM db_servers_redis JOIN redis_status ON db_servers_redis.host=redis_status.host AND db_servers_redis.port=redis_status.port WHERE db_servers_redis.monitor=1" 35 | stmtSel, err := db.Prepare(sql) 36 | if err != nil { 37 | log.Printf("failed to prepare SQL: %s", err.Error()) 38 | return nil, err 39 | } 40 | defer stmtSel.Close() 41 | 42 | rows, err := stmtSel.Query() 43 | if err != nil { 44 | log.Printf("failed to execute SQL: %s", err.Error()) 45 | return nil, err 46 | } 47 | 48 | for rows.Next() { 49 | var host string 50 | var port string 51 | var tags string 52 | var role string 53 | 54 | if err := rows.Scan(&host, &port, &tags, &role); err != nil { 55 | log.Printf("failed to scan a row: %s", err.Error()) 56 | return nil, err 57 | } 58 | 59 | instance := &Instance{ 60 | Host: host, 61 | Port: port, 62 | Tags: tags, 63 | Role: role} 64 | instances = append(instances, instance) 65 | } 66 | stmtSel.Close() 67 | 68 | return instances, nil 69 | } 70 | 71 | func UpdateStatus(db *sql.DB, tag, host, port, role string, alive, memory_used_percent, max_clients, connected_clients, blocked_clients int64) error { 72 | is_exist := false 73 | 74 | sql_select := "SELECT * FROM tbl_codis_monitor WHERE redis_ip=? AND port=?" 75 | stmt_select, err := db.Prepare(sql_select) 76 | if err != nil { 77 | log.Printf("failed to prepare SQL: %s", err.Error()) 78 | return err 79 | } 80 | defer stmt_select.Close() 81 | 82 | rows, err := stmt_select.Query(host, port) 83 | if err != nil { 84 | log.Printf("failed to execute SQL: %s", err.Error()) 85 | return err 86 | } 87 | 88 | for rows.Next() { 89 | is_exist = true 90 | } 91 | 92 | if is_exist { 93 | sql_update := "UPDATE tbl_codis_monitor SET online_status=?,mem_used_pct=?,max_conns=?,clients=?,blocked_clients=? WHERE redis_ip=? AND port=?" 94 | stmt_update, err := db.Prepare(sql_update) 95 | if err != nil { 96 | log.Printf("failed to prepare SQL: %s", err.Error()) 97 | return err 98 | } 99 | defer stmt_update.Close() 100 | 101 | result, err := stmt_update.Exec(alive, memory_used_percent, max_clients, connected_clients, blocked_clients, host, port) 102 | if err != nil { 103 | log.Printf("failed to update monitor data: %s", err.Error()) 104 | return err 105 | } 106 | affect, err := result.RowsAffected() 107 | if err != nil { 108 | log.Printf("failed to get affected count: %s", err.Error()) 109 | return err 110 | } 111 | log.Printf("update %d row(s)", affect) 112 | } else { 113 | sql_insert := "INSERT INTO tbl_codis_monitor(tag,redis_ip,port,role,online_status,mem_used_pct,clients,blocked_clients,max_conns) VALUES(?,?,?,?,?,?,?,?,?)" 114 | stmt_insert, err := db.Prepare(sql_insert) 115 | if err != nil { 116 | log.Printf("failed to prepare SQL: %s", err.Error()) 117 | return err 118 | } 119 | defer stmt_insert.Close() 120 | 121 | result, err := stmt_insert.Exec(tag, host, port, role, alive, memory_used_percent, connected_clients, blocked_clients, max_clients) 122 | if err != nil { 123 | log.Printf("failed to insert monitor data: %s", err.Error()) 124 | return err 125 | } 126 | last_id, err := result.LastInsertId() 127 | if err != nil { 128 | log.Printf("failed to get last id: %s", err.Error()) 129 | return err 130 | } 131 | log.Printf("insert [%d] row", last_id) 132 | } 133 | return nil 134 | } 135 | -------------------------------------------------------------------------------- /redismon/falcon.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "os" 11 | "time" 12 | ) 13 | 14 | type FalconData struct { 15 | Metric string `json:"metric"` 16 | Endpoint string `json:"endpoint"` 17 | Value interface{} `json:"value"` 18 | CounterType string `json:"counterType"` 19 | Tags string `json:"tags"` 20 | Timestamp int64 `json:"timestamp"` 21 | Step int64 `json:"step"` 22 | } 23 | 24 | func (data *FalconData) SetValue(v interface{}) { 25 | data.Value = v 26 | } 27 | 28 | func (data *FalconData) String() string { 29 | s := fmt.Sprintf("FalconData Metric:%s Endpoint:%s Value:%v CounterType:%s Tags:%s Timestamp:%d Step:%d", 30 | data.Metric, data.Endpoint, data.Value, data.CounterType, data.Tags, data.Timestamp, data.Step) 31 | return s 32 | } 33 | 34 | func NewFalconData(metric string, endpoint string, value interface{}, counterType string, tags string, timestamp int64, step int64) *FalconData { 35 | point := &FalconData{ 36 | Metric: metric, 37 | Endpoint: GetEndpoint(endpoint), 38 | CounterType: counterType, 39 | Tags: tags, 40 | Timestamp: GetTimestamp(timestamp), 41 | Step: step, 42 | } 43 | point.SetValue(value) 44 | return point 45 | } 46 | 47 | func GetEndpoint(endpoint string) string { 48 | if endpoint != "" { 49 | return endpoint 50 | } 51 | 52 | hostname, err := os.Hostname() 53 | if err != nil { 54 | hostname = "localhost" 55 | } 56 | return hostname 57 | } 58 | 59 | func GetTimestamp(timestamp int64) int64 { 60 | if timestamp > 0 { 61 | return timestamp 62 | } else { 63 | return time.Now().Unix() 64 | } 65 | } 66 | 67 | func PushToFalcon(api string, data []*FalconData) ([]byte, error) { 68 | points, err := json.Marshal(data) 69 | if err != nil { 70 | log.Printf("fail to convert data to json: %s", err.Error()) 71 | return nil, err 72 | } 73 | 74 | response, err := http.Post(api, "Content-Type: application/json", bytes.NewBuffer(points)) 75 | if err != nil { 76 | log.Printf("fail to call falcon API: %s", err.Error()) 77 | return nil, err 78 | } 79 | defer response.Body.Close() 80 | 81 | content, err := ioutil.ReadAll(response.Body) 82 | if err != nil { 83 | return nil, err 84 | } 85 | 86 | return content, nil 87 | } 88 | -------------------------------------------------------------------------------- /redismon/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "time" 7 | ) 8 | 9 | func init() { 10 | log.SetFlags(log.LstdFlags | log.Lshortfile) 11 | 12 | Cfg = LoadConfig() 13 | if Cfg == nil { 14 | panic("fail to load application configuration") 15 | } 16 | } 17 | 18 | func main() { 19 | fmt.Println("=====redismon=====") 20 | fmt.Printf("Start:%s\n", time.Now().Format("200601021504")) 21 | 22 | lepus, err := Connect(Cfg.Lepus) 23 | if err != nil { 24 | log.Printf("failed to connect to the lepus: %s", err.Error()) 25 | return 26 | } 27 | defer lepus.Close() 28 | 29 | manager, err := Connect(Cfg.Manager) 30 | if err != nil { 31 | log.Printf("failed to connect to the db_manager: %s", err.Error()) 32 | return 33 | } 34 | defer manager.Close() 35 | 36 | instances, err := GetInstances(lepus) 37 | if err != nil { 38 | log.Printf("failed to get redis instance from lepus: %s", err.Error()) 39 | return 40 | } 41 | 42 | count := time.Duration(Cfg.Timeout) 43 | 44 | fep := Cfg.Falcon.Endpoint 45 | step := Cfg.Falcon.Step 46 | 47 | ts := time.Now().Unix() 48 | 49 | for _, instance := range instances { 50 | timeout := time.Second * count 51 | endpoint := fmt.Sprintf("%s:%s", instance.Host, instance.Port) 52 | host := instance.Host 53 | port := instance.Port 54 | itags := instance.Tags 55 | role := instance.Role 56 | fmt.Printf("Endpoint: %s\n", endpoint) 57 | fmt.Printf("Role: %s\n", role) 58 | ok, version, max_memory, used_memory, memory_used_percent, max_clients, connected_clients, blocked_clients := CheckRedis(endpoint, timeout) 59 | is_alive := int64(1) 60 | if !ok { 61 | is_alive = int64(0) 62 | } 63 | fmt.Printf("alive: %t\nversion: %d\nmax_memory: %d, used_memory: %d, memory_used_percent: %d\nmax_clients: %d, connected_clients: %d, blocked_clients: %d\n\n", ok, version, max_memory, used_memory, memory_used_percent, max_clients, connected_clients, blocked_clients) 64 | 65 | // prepare and push falcon data 66 | from := GetEndpoint("") 67 | tags := fmt.Sprintf("from=%s,host=%s,port=%s", from, instance.Host, instance.Port) 68 | 69 | data := []*FalconData{} 70 | point := NewFalconData("redis.alive", fep, is_alive, "GAUGE", tags, ts, step) 71 | data = append(data, point) 72 | point = NewFalconData("redis.version", fep, version, "GAUGE", tags, ts, step) 73 | data = append(data, point) 74 | point = NewFalconData("redis.max.memory", fep, max_memory, "GAUGE", tags, ts, step) 75 | data = append(data, point) 76 | point = NewFalconData("redis.used.memory", fep, used_memory, "GAUGE", tags, ts, step) 77 | data = append(data, point) 78 | point = NewFalconData("redis.memory.used.percent", fep, memory_used_percent, "GAUGE", tags, ts, step) 79 | data = append(data, point) 80 | point = NewFalconData("redis.max.clients", fep, max_clients, "GAUGE", tags, ts, step) 81 | data = append(data, point) 82 | point = NewFalconData("redis.connected.clients", fep, connected_clients, "GAUGE", tags, ts, step) 83 | data = append(data, point) 84 | point = NewFalconData("redis.blocked.clients", fep, blocked_clients, "GAUGE", tags, ts, step) 85 | data = append(data, point) 86 | response, err := PushToFalcon(Cfg.Falcon.API, data) 87 | if err != nil { 88 | log.Printf("failed to push falcon data: %s", err.Error()) 89 | continue 90 | } 91 | fmt.Printf("push data to falcon successfully: %s\n", string(response)) 92 | 93 | if err := UpdateStatus(manager, itags, host, port, role, is_alive, memory_used_percent, max_clients, connected_clients, blocked_clients); err != nil { 94 | log.Printf("failed to save monitor data: %s", err.Error()) 95 | continue 96 | } 97 | } 98 | fmt.Printf("End:%s\n", time.Now().Format("200601021504")) 99 | return 100 | } 101 | -------------------------------------------------------------------------------- /request/request.json: -------------------------------------------------------------------------------- 1 | { 2 | "endpoint": "localhost", 3 | "step":60, 4 | "counterType":"GAUGE", 5 | "tags":"product=prod", 6 | "request":[ 7 | { 8 | "metric":"prod.basic", 9 | "protocol":"grpc", 10 | "ip":"127.0.0.1", 11 | "port":"6566", 12 | "judgement":"status" 13 | }, 14 | { 15 | "metric":"prod.api", 16 | "protocol":"http", 17 | "url":"http://127.0.0.1:8080/health", 18 | "method":"GET", 19 | "judgement":"semanteme", 20 | "keyword":"DOWN" 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /request/request.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | #-*- coding:utf-8 -*- 3 | # 4 | # Author: yezhiqin@hualala.com 5 | # Date : 2016/11/11 create 6 | # Date : 2017/05/29 modify 7 | 8 | import commands 9 | import json 10 | import os 11 | import requests 12 | import time 13 | 14 | class Checker: 15 | def __init__(self, cfg): 16 | with open(cfg) as file: 17 | config = json.load(file) 18 | self.endpoint = os.uname()[1] 19 | self.step = config['step'] 20 | self.counterType = config['counterType'] 21 | self.tags = config['tags'] 22 | self.request = config['request'] 23 | self.data = [] 24 | 25 | # function to dispatch request items. 26 | def handleRequest(self): 27 | for item in self.request: 28 | metric = item['metric'] 29 | protocol = item['protocol'] 30 | judgement = item['judgement'] 31 | url = item.get('url','') 32 | method = item.get('method','GET') 33 | ip = item.get('ip','127.0.0.1') 34 | port = item.get('port','') 35 | keyword = item.get('keyword','') 36 | if protocol == 'http': 37 | value = self.checkHTTP(metric, judgement, url, method, keyword) 38 | self.addPoint(self.endpoint, metric, self.step, value, self.counterType, self.tags) 39 | print "==========" 40 | elif protocol == 'grpc': 41 | value = self.checkGRPC(metric, judgement, ip, port, method, keyword) 42 | self.addPoint(self.endpoint, metric, self.step, value, self.counterType, self.tags) 43 | print "==========" 44 | else: 45 | print "%s Unknown Protocol!" % metric 46 | print "==========" 47 | self.push2falcon() 48 | print "~~~~~End~~~~~" 49 | 50 | # function to check HTTP status. 51 | def checkHTTP(self, metric, judgement, url, method="", keyword=""): 52 | try: 53 | if judgement == "status": 54 | response = requests.request(method, url, timeout=5) 55 | code = response.status_code 56 | if code == 200 or code == 503 or code == 404: 57 | print "%s is OK" % metric 58 | print " The status code is %d" % code 59 | return 0 60 | else: 61 | print "%s is abnormal!" % metric 62 | print " The status code is %d" % code 63 | return 1 64 | elif judgement == "semanteme": 65 | response = requests.request(method, url, timeout=5) 66 | code = response.status_code 67 | if code == 200 or code == 503 or code == 404: 68 | print "%s is OK." % metric 69 | print " The status code is %d" % code 70 | else: 71 | print "%s is abnormal!" % metric 72 | print " The status code is %d" % code 73 | return 1 74 | content = response.text 75 | if keyword not in content: 76 | print "%s is OK." % metric 77 | print " The response text is: %s" % content 78 | return 0 79 | else: 80 | print "%s is abnormal!" % metric 81 | print " The response text is: %s" % content 82 | return 2 83 | else: 84 | print "Unknown Judgement!" 85 | return -1 86 | except Exception,e: 87 | print "Request Error!" 88 | print e 89 | return -1 90 | 91 | # function to check GRPC status. 92 | def checkGRPC(self, metric, judgement, ip, port, method="", keyword=""): 93 | try: 94 | if judgement == "status": 95 | cmd = "./HealthCheck %s:%s" % (ip, port) 96 | status, output = commands.getstatusoutput(cmd) 97 | if status == 0: 98 | print "%s is OK" % metric 99 | print " The status code is %d" % status 100 | return 0 101 | else: 102 | print "%s is abnormal!" % metric 103 | print " The status code is %d" % status 104 | return 1 105 | elif judgement == "semanteme": 106 | pass 107 | else: 108 | print "Unknown Judgement!" 109 | return -1 110 | except Exception,e: 111 | print "Request Error!" 112 | print e 113 | return -1 114 | 115 | def addPoint(self, endpoint, metric, step, value, counterType, tags): 116 | point = { 117 | 'endpoint': endpoint, 118 | 'metric': metric, 119 | 'timestamp': int(time.time()), 120 | 'step': step, 121 | 'value': value, 122 | 'counterType': counterType, 123 | 'tags': tags 124 | } 125 | self.data.append(point) 126 | 127 | def push2falcon(self): 128 | print "Push data to falcon: %s" % json.dumps(self.data) 129 | response = requests.post("http://127.0.0.1:1988/v1/push", json.dumps(self.data), timeout=5) 130 | code = response.status_code 131 | text = response.text 132 | print "Push data to falcon, status code is %d and response text is %s" % (code,text) 133 | 134 | def send2Wechat(self, tos, content): 135 | url="http://127.0.0.1:9090/msg" 136 | data={'tos':tos,'content':content} 137 | response = requests.post(url, data=data) 138 | code = response.status_code 139 | text = response.text 140 | print "Send message to wechat, status code is %d and response text is %s" % (code,text) 141 | 142 | # start to check 143 | path = os.path.split(os.path.realpath(__file__))[0] 144 | cfg = path+"/request.json" 145 | checker = Checker(cfg) 146 | checker.handleRequest(); 147 | -------------------------------------------------------------------------------- /zkmon/zkmon.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #!/bin/bash 4 | # 5 | # Author: ye.zhiqin@outlook.com 6 | # Date: 2018/4/20 7 | # Brief: 8 | # Launch the processes provided by proc file. 9 | # Globals: 10 | # ZK_PROC 11 | # ZK_ADDR 12 | # ZK_PORT 13 | # FALCON_API 14 | # Arguments: 15 | # None 16 | # Return: 17 | # None 18 | 19 | # set 20 | set -e 21 | set -u 22 | set -o pipefail 23 | 24 | # global variables 25 | ZK_PROC="QuorumPeerMain" 26 | ZK_ADDR="127.0.0.1" 27 | ZK_PORT="2181" 28 | FALCON_API=http://127.0.0.1:1988/v1/push 29 | 30 | # variables 31 | endpoint=$(hostname) 32 | ts=$(date +%s) 33 | step=60 34 | counterType="GAUGE" 35 | tags="module=zookeeper" 36 | 37 | # get zookeeper process id 38 | pid=$(ps -ef | grep "${ZK_PROC}" | grep -v "grep" | awk '{print $2}') 39 | if [[ -z ${pid} ]];then 40 | echo "can not find the process id!" 41 | exit 1 42 | else 43 | echo "zookeeper process id is: ${pid}" 44 | fi 45 | 46 | if [[ 1 -eq ${pid} ]];then 47 | echo "process ID 1 is unexpected!" 48 | exit 1 49 | fi 50 | 51 | # get zookeeper cpu usage 52 | zk_cpu=$(ps -p "${pid}" -o pcpu --no-header) 53 | echo "zookeeper CPU is: ${zk_cpu}" 54 | 55 | # get zookeeper mem usage 56 | mem_K=$(ps -p "${pid}" -o rss --no-header) 57 | mem_M=$(echo "scale=2;${mem_K} / 1024.0" | bc) 58 | zk_mem=$(echo "scale=2;${mem_M} / 1024.0" | bc | awk '{printf "%.2f", $0}') 59 | echo "zookeeper MEM is: ${zk_mem}" 60 | 61 | # is zookeeper ok? 62 | ruok_resp=$(echo ruok | nc ${ZK_ADDR} ${ZK_PORT}) 63 | if [[ "imok" == ${ruok_resp} ]];then 64 | zk_ok=0 65 | echo "zookeeper stat is: ${zk_ok}" 66 | else 67 | zk_ok=1 68 | echo "zookeeper stat is: ${zk_ok}" 69 | fi 70 | 71 | # get zookeeper data 72 | echo conf | nc ${ZK_ADDR} ${ZK_PORT} > conf.txt 73 | echo stat | nc ${ZK_ADDR} ${ZK_PORT} > stat.txt 74 | echo wchs | nc ${ZK_ADDR} ${ZK_PORT} > wchs.txt 75 | 76 | zk_max_cons=$(grep -oE "maxClientCnxns=[0-9]*" conf.txt | awk -F"=" '{print $NF}') 77 | echo "zookeeper max connection number is: ${zk_max_cons}" 78 | 79 | zk_cons=$(grep -oE "Connections: [0-9]*" stat.txt | awk '{print $NF}') 80 | echo "zookeeper connection number is: ${zk_cons}" 81 | 82 | zk_free_cons=$((zk_max_cons - zk_cons)) 83 | echo "zookeeper free connection number is: ${zk_free_cons}" 84 | 85 | mode=$(grep -oE "Mode: [a-z]*" stat.txt | awk '{print $NF}') 86 | if [[ "leader" == ${mode} ]];then 87 | zk_leader=0 88 | echo "zookeeper mode is: ${zk_leader}" 89 | else 90 | zk_leader=1 91 | echo "zookeeper mode is: ${zk_leader}" 92 | fi 93 | 94 | zk_wchs=$(grep -oE "watches:[0-9]*" wchs.txt | awk -F":" '{print $NF}') 95 | echo "zookeeper watch count is: ${zk_wchs}" 96 | 97 | 98 | # push to falcon 99 | curl -X POST -d "[{\"metric\": \"zk.cpu\", \"endpoint\": \"${endpoint}\", \"timestamp\": ${ts},\"step\": ${step},\"value\": ${zk_cpu},\"counterType\": \"${counterType}\",\"tags\": \"${tags}\"},{\"metric\": \"zk.mem\", \"endpoint\": \"${endpoint}\", \"timestamp\": ${ts},\"step\": ${step},\"value\": ${zk_mem},\"counterType\": \"${counterType}\",\"tags\": \"${tags}\"},{\"metric\": \"zk.ok\", \"endpoint\": \"${endpoint}\", \"timestamp\": ${ts},\"step\": ${step},\"value\": ${zk_ok},\"counterType\": \"${counterType}\",\"tags\": \"${tags}\"},{\"metric\": \"zk.cons.max\", \"endpoint\": \"${endpoint}\", \"timestamp\": ${ts},\"step\": ${step},\"value\": ${zk_max_cons},\"counterType\": \"${counterType}\",\"tags\": \"${tags}\"},{\"metric\": \"zk.cons\", \"endpoint\": \"${endpoint}\", \"timestamp\": ${ts},\"step\": ${step},\"value\": ${zk_cons},\"counterType\": \"${counterType}\",\"tags\": \"${tags}\"},{\"metric\": \"zk.cons.free\", \"endpoint\": \"${endpoint}\", \"timestamp\": ${ts},\"step\": ${step},\"value\": ${zk_free_cons},\"counterType\": \"${counterType}\",\"tags\": \"${tags}\"},{\"metric\": \"zk.lead\", \"endpoint\": \"${endpoint}\", \"timestamp\": ${ts},\"step\": ${step},\"value\": ${zk_leader},\"counterType\": \"${counterType}\",\"tags\": \"${tags}\"}]" ${FALCON_API} 100 | --------------------------------------------------------------------------------