├── .gitignore ├── README.md ├── cfg.example.json ├── control ├── g ├── cfg.go ├── g.go └── git.go ├── http ├── common.go ├── http.go └── proc_http.go ├── main.go ├── monitor └── monitor.go ├── perfcounter.json └── test └── debug /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | *.swp 27 | *.swo 28 | *.log 29 | .idea 30 | .DS_Store 31 | 32 | ## tycs 33 | /var 34 | /tycs* 35 | /cfg.json 36 | .gitversion 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # anteye 2 | anteye is a small and simple monitor system. anteye should monitor cluster less then 50 instances. it can send notice msgs via **mail**、**sms** or **callback(TODO)**. 3 | we suggest you deploy more than one anteye instances in the production environment. 4 | 5 | ## install 6 | 7 | You can install anteye from the latest [release](https://github.com/niean/anteye/releases/download/v0.0.2/tycs-anteye-0.0.2.tar.gz), 8 | 9 | ```bash 10 | # download release 11 | wget -q https://github.com/niean/anteye/releases/download/v0.0.2/tycs-anteye-0.0.2.tar.gz 12 | tar -zxf tycs-anteye-$vsn.tar.gz 13 | 14 | # config, change configs as you like 15 | mv cfg.example.json cfg.json 16 | vim cfg.json 17 | ... 18 | 19 | # start 20 | ./control start 21 | 22 | # stop 23 | ./control stop 24 | 25 | ``` 26 | 27 | Or you can install anteye from scratch 28 | 29 | ```bash 30 | # download src 31 | cd $GOPATH/src/github.com/niean 32 | git clone https://github.com/niean/anteye.git 33 | cd anteye 34 | go get ./... 35 | 36 | # build, get bin tycs-anteye 37 | ./control build 38 | 39 | # config, change configs as you like 40 | mv cfg.example.json cfg.json 41 | vim cfg.json 42 | ... 43 | 44 | # start 45 | ./control start 46 | 47 | # stop 48 | ./control stop 49 | 50 | ``` 51 | 52 | ## config 53 | ```python 54 | 55 | debug: true/false, open debug log or not 56 | 57 | http 58 | - enable: true/false, enable http-server or not 59 | - listen: listening port of http-server 60 | 61 | mail 62 | - enable: true/false, enable sending alarm mails or not 63 | - url: http-url used to post mail content 64 | - receivers: mail accounts. if you have multiple accounts, then separate them by commas. eg. "a@gmail.com,b@yahoo.com" 65 | 66 | sms 67 | - enable: true/false, enable sending alarm sms or not 68 | - url: http-url used to post sms content 69 | - receivers: mobile numbers. if you have multiple numbers, then separate them by commas. eg. "18001163876,13811685233" 70 | 71 | callback 72 | - enable: true/false, enable alarm callback or not 73 | - url: http-url used to post alarm content 74 | 75 | monitor 76 | - cluster: host instances to be monitored, one item goes like "module,hostname:port/health/url" 77 | ``` 78 | 79 | ## interface 80 | anteye sends msgs via http interfaces. these interfaces defined as followings: 81 | 82 | ```bash 83 | # sms interface 84 | method: http.post 85 | params: 86 | - tos: mobile numbers separated by commas 87 | - content: content of sms 88 | - from: optional, indicates who sends this sms 89 | 90 | # mail interface 91 | method: http.post 92 | params: 93 | - tos: mail accounts separated by commas 94 | - content: content of mail 95 | - subject: subject of mail 96 | - from: optional, indicates who sends this mail 97 | 98 | # callback 99 | method: http.post 100 | params: body. anteye will post you a string object like '[date][status][err.cnt][instance]', eg. [2015-07-02 08:40:30][err][8][task,127.0.0.1:16269/health] 101 | 102 | ``` 103 | 104 | ## debug 105 | ```bash 106 | # log 107 | ./test/debug tail 108 | 109 | # get internal status 110 | ./test/debug counter 111 | 112 | ``` 113 | 114 | ## reference 115 | -------------------------------------------------------------------------------- /cfg.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "debug": false, 3 | "http": { 4 | "enable": true, 5 | "listen": "0.0.0.0:8001" 6 | }, 7 | "mail" : { 8 | "enable": true, 9 | "url" : "http://mail.server:1986/mail/sender", 10 | "receivers" : "nieanan@xiaomi.com,niean.sail@gmail.com" 11 | }, 12 | "sms" : { 13 | "enable": false, 14 | "url" : "http://sms.server:1986/sms/sender", 15 | "receivers" : "18001163885,13811685238" 16 | }, 17 | "callback" : { 18 | "enable": false, 19 | "url" : "http://tycloudstart.com:1986/anteys/alarm" 20 | }, 21 | "monitor" : { 22 | "cluster" : [ 23 | "transfer,test.hostname01:6060/health", 24 | "task,test.hostname01:8002/health", 25 | "graph,test.hostname01:6071/health" 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /control: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | workspace=$(cd $(dirname $0) && pwd) 3 | cd $workspace 4 | 5 | module=anteye 6 | app=tycs-$module 7 | conf=cfg.json 8 | pidfile=var/app.pid 9 | logfile=var/app.log 10 | gitversion=.gitversion 11 | pfc=perfcounter.json 12 | 13 | mkdir -p var &>/dev/null 14 | 15 | 16 | ## build & pack 17 | function build() { 18 | update_gitversion 19 | go build -o $app main.go 20 | sc=$? 21 | if [ $sc -ne 0 ];then 22 | echo "build error" 23 | exit $sc 24 | else 25 | echo -n "build ok, vsn=" 26 | version 27 | fi 28 | } 29 | 30 | function pack() { 31 | build 32 | version=`./$app -v` 33 | tar zcvf $app-$version.tar.gz control $app cfg.example.json $pfc $gitversion ./test/debug 34 | } 35 | 36 | function packbin() { 37 | build 38 | version=`./$app -v` 39 | tar zcvf $app-bin-$version.tar.gz $app $gitversion 40 | } 41 | 42 | ## opt 43 | function start() { 44 | check_pid 45 | running=$? 46 | if [ $running -gt 0 ];then 47 | echo -n "$app started, pid=" 48 | cat $pidfile 49 | return 1 50 | fi 51 | 52 | nohup ./$app -c $conf >>$logfile 2>&1 & 53 | echo $! > $pidfile 54 | echo "$app start ok, pid=$!" 55 | } 56 | 57 | function stop() { 58 | pid=`cat $pidfile` 59 | kill $pid 60 | echo "$app stoped" 61 | } 62 | 63 | function restart() { 64 | stop && sleep 1 && start 65 | } 66 | 67 | function reload() { 68 | build && stop && sleep 1 && start && sleep 1 && printf "\n" && tailf 69 | } 70 | 71 | ## other 72 | function status() { 73 | check_pid 74 | running=$? 75 | if [ $running -gt 0 ];then 76 | echo -n "$app running, pid=" 77 | cat $pidfile 78 | else 79 | echo "$app stoped" 80 | fi 81 | } 82 | 83 | function version() { 84 | v=`./$app -v` 85 | if [ -f $gitversion ];then 86 | g=`cat $gitversion` 87 | fi 88 | echo "$v $g" 89 | } 90 | 91 | function tailf() { 92 | tail -f $logfile 93 | } 94 | 95 | ## internal 96 | function check_pid() { 97 | if [ -f $pidfile ];then 98 | pid=`cat $pidfile` 99 | if [ -n $pid ]; then 100 | running=`ps -p $pid|grep -v "PID TTY" |wc -l` 101 | return $running 102 | fi 103 | fi 104 | return 0 105 | } 106 | 107 | function update_gitversion() { 108 | git log -1 --pretty=%h > $gitversion 109 | } 110 | 111 | ## usage 112 | function usage() { 113 | echo "$0 build|pack|packbin|start|stop|restart|reload|status|tail|version" 114 | } 115 | 116 | ## main 117 | action=$1 118 | case $action in 119 | ## build 120 | "build" ) 121 | build 122 | ;; 123 | "pack" ) 124 | pack 125 | ;; 126 | "packbin" ) 127 | packbin 128 | ;; 129 | ## opt 130 | "start" ) 131 | start 132 | ;; 133 | "stop" ) 134 | stop 135 | ;; 136 | "restart" ) 137 | restart 138 | ;; 139 | "reload" ) 140 | reload 141 | ;; 142 | ## other 143 | "status" ) 144 | status 145 | ;; 146 | "version" ) 147 | version 148 | ;; 149 | "tail" ) 150 | tailf 151 | ;; 152 | * ) 153 | usage 154 | ;; 155 | esac -------------------------------------------------------------------------------- /g/cfg.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/toolkits/file" 6 | "log" 7 | "sync" 8 | ) 9 | 10 | type HttpConfig struct { 11 | Enable bool `json:"enable"` 12 | Listen string `json:"listen"` 13 | } 14 | 15 | type MailConfig struct { 16 | Enable bool `json:"enable"` 17 | Url string `json:"url"` 18 | Receivers string `json:"receivers"` 19 | } 20 | 21 | type SmsConfig struct { 22 | Enable bool `json:"enable"` 23 | Url string `json:"url"` 24 | Receivers string `json:"receivers"` 25 | } 26 | 27 | type CallbackConfig struct { 28 | Enable bool `json:"enable"` 29 | Url string `json:"url"` 30 | } 31 | 32 | type MonitorConfig struct { 33 | Cluster []string `json:"cluster"` 34 | } 35 | 36 | type GlobalConfig struct { 37 | Debug bool `json:"debug"` 38 | Http *HttpConfig `json:"http"` 39 | Mail *MailConfig `json:"mail"` 40 | Sms *SmsConfig `json:"sms"` 41 | Callback *CallbackConfig `json:"callback"` 42 | Monitor *MonitorConfig `json:"monitor"` 43 | } 44 | 45 | var ( 46 | ConfigFile string 47 | config *GlobalConfig 48 | configLock = new(sync.RWMutex) 49 | ) 50 | 51 | func Config() *GlobalConfig { 52 | configLock.RLock() 53 | defer configLock.RUnlock() 54 | return config 55 | } 56 | 57 | func ParseConfig(cfg string) { 58 | if cfg == "" { 59 | log.Fatalln("use -c to specify configuration file") 60 | } 61 | 62 | if !file.IsExist(cfg) { 63 | log.Fatalln("config file:", cfg, "is not existent. maybe you need `mv cfg.example.json cfg.json`") 64 | } 65 | 66 | ConfigFile = cfg 67 | 68 | configContent, err := file.ToTrimString(cfg) 69 | if err != nil { 70 | log.Fatalln("read config file:", cfg, "fail:", err) 71 | } 72 | 73 | var c GlobalConfig 74 | err = json.Unmarshal([]byte(configContent), &c) 75 | if err != nil { 76 | log.Fatalln("parse config file:", cfg, "fail:", err) 77 | } 78 | 79 | configLock.Lock() 80 | defer configLock.Unlock() 81 | config = &c 82 | 83 | log.Println("g:ParseConfig, ok, ", cfg) 84 | } 85 | -------------------------------------------------------------------------------- /g/g.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "log" 5 | "runtime" 6 | ) 7 | 8 | // changelog: 9 | // 0.0.1: init project 10 | // 0.0.2: fmt import path, modify control and debug scripts 11 | // 0.0.3: use pfc, rm vg 12 | 13 | const ( 14 | VERSION = "0.0.3" 15 | ) 16 | 17 | func init() { 18 | runtime.GOMAXPROCS(runtime.NumCPU()) 19 | log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) 20 | } 21 | -------------------------------------------------------------------------------- /g/git.go: -------------------------------------------------------------------------------- 1 | package g 2 | const ( 3 | COMMIT = "ffc29d6" 4 | ) 5 | -------------------------------------------------------------------------------- /http/common.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "fmt" 5 | "github.com/toolkits/file" 6 | "net/http" 7 | 8 | "github.com/niean/anteye/g" 9 | ) 10 | 11 | func configCommonRoutes() { 12 | http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { 13 | w.Write([]byte("ok\n")) 14 | }) 15 | 16 | http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { 17 | w.Write([]byte(fmt.Sprintf("%s\n", g.VERSION))) 18 | }) 19 | 20 | http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) { 21 | w.Write([]byte(fmt.Sprintf("%s\n", file.SelfDir()))) 22 | }) 23 | 24 | http.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) { 25 | RenderDataJson(w, g.Config()) 26 | }) 27 | } 28 | -------------------------------------------------------------------------------- /http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/niean/anteye/g" 9 | ) 10 | 11 | type Dto struct { 12 | Msg string `json:"msg"` 13 | Data interface{} `json:"data"` 14 | } 15 | 16 | // start http server 17 | func Start() { 18 | go startHttpServer() 19 | } 20 | 21 | func configRoutes() { 22 | configCommonRoutes() 23 | configProcHttpRoutes() 24 | } 25 | 26 | func startHttpServer() { 27 | if !g.Config().Http.Enable { 28 | return 29 | } 30 | 31 | addr := g.Config().Http.Listen 32 | if addr == "" { 33 | return 34 | } 35 | 36 | // init url mapping 37 | configRoutes() 38 | 39 | s := &http.Server{ 40 | Addr: addr, 41 | MaxHeaderBytes: 1 << 30, 42 | } 43 | 44 | log.Println("http.startHttpServer ok, listening ", addr) 45 | log.Fatalln(s.ListenAndServe()) 46 | } 47 | 48 | func RenderJson(w http.ResponseWriter, v interface{}) { 49 | bs, err := json.Marshal(v) 50 | if err != nil { 51 | http.Error(w, err.Error(), http.StatusInternalServerError) 52 | return 53 | } 54 | w.Header().Set("Content-Type", "application/json; charset=UTF-8") 55 | w.Write(bs) 56 | } 57 | 58 | func RenderDataJson(w http.ResponseWriter, data interface{}) { 59 | RenderJson(w, Dto{Msg: "success", Data: data}) 60 | } 61 | 62 | func RenderMsgJson(w http.ResponseWriter, msg string) { 63 | RenderJson(w, map[string]string{"msg": msg}) 64 | } 65 | 66 | func AutoRender(w http.ResponseWriter, data interface{}, err error) { 67 | if err != nil { 68 | RenderMsgJson(w, err.Error()) 69 | return 70 | } 71 | RenderDataJson(w, data) 72 | } 73 | -------------------------------------------------------------------------------- /http/proc_http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | func configProcHttpRoutes() { 8 | // counter 9 | http.HandleFunc("/counter/all", func(w http.ResponseWriter, r *http.Request) { 10 | RenderDataJson(w, make([]interface{}, 0)) 11 | }) 12 | } 13 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/niean/anteye/g" 9 | "github.com/niean/anteye/http" 10 | "github.com/niean/anteye/monitor" 11 | ) 12 | 13 | func main() { 14 | cfg := flag.String("c", "cfg.json", "configuration file") 15 | version := flag.Bool("v", false, "version") 16 | flag.Parse() 17 | 18 | if *version { 19 | fmt.Println(g.VERSION) 20 | os.Exit(0) 21 | } 22 | 23 | // global config 24 | g.ParseConfig(*cfg) 25 | 26 | // monitor 27 | monitor.Start() 28 | 29 | // http 30 | http.Start() 31 | 32 | select {} 33 | } 34 | -------------------------------------------------------------------------------- /monitor/monitor.go: -------------------------------------------------------------------------------- 1 | package monitor 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "os" 10 | "strings" 11 | "sync" 12 | "time" 13 | 14 | ncron "github.com/niean/cron" 15 | pfc "github.com/niean/goperfcounter" 16 | nmap "github.com/niean/gotools/container/nmap" 17 | nhttpclient "github.com/niean/gotools/http/httpclient" 18 | ntime "github.com/niean/gotools/time" 19 | 20 | "github.com/niean/anteye/g" 21 | ) 22 | 23 | var ( 24 | // cron 25 | monitorCron = ncron.New() 26 | cronSpec = "30 * * * * ?" 27 | // cache 28 | statusCache = nmap.NewSafeMap() 29 | alarmCache = nmap.NewSafeMap() 30 | ) 31 | 32 | func Start() { 33 | monitorCron.AddFuncCC(cronSpec, func() { monitor() }, 1) 34 | monitorCron.Start() 35 | go alarmJudge() 36 | log.Println("monitor.Start ok") 37 | } 38 | 39 | // alarm judge 40 | func alarmJudge() { 41 | interval := time.Duration(10) * time.Second 42 | for { 43 | time.Sleep(interval) 44 | var content bytes.Buffer 45 | 46 | keys := alarmCache.Keys() 47 | if len(keys) == 0 { 48 | continue 49 | } 50 | for _, key := range keys { 51 | aitem, found := alarmCache.GetAndRemove(key) 52 | if !found { 53 | continue 54 | } 55 | content.WriteString(aitem.(*Alarm).String() + "\n") 56 | } 57 | 58 | if content.Len() < 6 { 59 | return 60 | } 61 | 62 | cfg := g.Config() 63 | // mail 64 | if cfg.Mail.Enable { 65 | hn, _ := os.Hostname() 66 | mailContent := formAlarmMailContent(cfg.Mail.Receivers, "AntEye.Alarm.From.["+hn+"]", 67 | content.String(), "AntEye") 68 | err := sendMail(cfg.Mail.Url, mailContent) 69 | if err != nil { 70 | log.Println("alarm send mail error, mail:", mailContent, "", err) 71 | } else { 72 | // statistics 73 | pfc.Meter("MonitorAlarmMail", 1) 74 | } 75 | } 76 | 77 | // sms 78 | if cfg.Sms.Enable { 79 | smsContent := formAlarmSmsContent(cfg.Sms.Receivers, content.String(), "AntEye") 80 | err := sendSms(cfg.Sms.Url, smsContent) 81 | if err != nil { 82 | log.Println("alarm send sms error, sms:", smsContent, "", err) 83 | } else { 84 | // statistics 85 | pfc.Meter("MonitorAlarmSms", 1) 86 | } 87 | } 88 | 89 | // callback 90 | if cfg.Callback.Enable { 91 | cbc := content.String() 92 | err := alarmCallback(cfg.Callback.Url, cbc) 93 | if err != nil { 94 | log.Println("alarm callback error, callback:", cfg.Callback, ", content:", cbc, "", err) 95 | } else { 96 | // statistics 97 | pfc.Meter("MonitorAlarmCallback", 1) 98 | } 99 | } 100 | } 101 | } 102 | 103 | func formAlarmMailContent(tos string, subject string, content string, from string) string { 104 | return fmt.Sprintf("tos=%s&subject=%s&content=%s&user=%s", tos, subject, content, from) 105 | } 106 | 107 | func sendMail(mailUrl string, content string) error { 108 | client := nhttpclient.GetHttpClient("monitor.mail", 5*time.Second, 10*time.Second) 109 | // send by http-post 110 | req, err := http.NewRequest("POST", mailUrl, bytes.NewBufferString(content)) 111 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") 112 | req.Header.Set("Connection", "close") 113 | postResp, err := client.Do(req) 114 | if err != nil { 115 | return err 116 | } 117 | defer postResp.Body.Close() 118 | 119 | if postResp.StatusCode/100 != 2 { 120 | return fmt.Errorf("Http-Post Error, Code %d", postResp.StatusCode) 121 | } 122 | return nil 123 | } 124 | 125 | func formAlarmSmsContent(tos string, content string, from string) string { 126 | return fmt.Sprintf("tos=%s&content=%s&from=%s", tos, content, from) 127 | } 128 | 129 | func sendSms(smsUrl string, content string) error { 130 | client := nhttpclient.GetHttpClient("monitor.sms", 5*time.Second, 10*time.Second) 131 | // send by http-post 132 | req, err := http.NewRequest("POST", smsUrl, bytes.NewBufferString(content)) 133 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") 134 | req.Header.Set("Connection", "close") 135 | postResp, err := client.Do(req) 136 | if err != nil { 137 | return err 138 | } 139 | defer postResp.Body.Close() 140 | 141 | if postResp.StatusCode/100 != 2 { 142 | return fmt.Errorf("Http-Post Error, Code %d", postResp.StatusCode) 143 | } 144 | return nil 145 | } 146 | 147 | func alarmCallback(caUrl string, content string) error { 148 | client := nhttpclient.GetHttpClient("monitor.callback", 5*time.Second, 10*time.Second) 149 | // send by http-post 150 | req, err := http.NewRequest("POST", caUrl, bytes.NewBufferString(content)) 151 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") 152 | req.Header.Set("Connection", "close") 153 | postResp, err := client.Do(req) 154 | if err != nil { 155 | return err 156 | } 157 | defer postResp.Body.Close() 158 | 159 | if postResp.StatusCode/100 != 2 { 160 | return fmt.Errorf("Http-Post Error, Code %d", postResp.StatusCode) 161 | } 162 | return nil 163 | } 164 | 165 | // status calc 166 | func monitor() { 167 | startTs := time.Now().Unix() 168 | _monitor() 169 | endTs := time.Now().Unix() 170 | log.Printf("monitor, startTs %s, time-consuming %d sec\n", ntime.FormatTs(startTs), endTs-startTs) 171 | 172 | // statistics 173 | pfc.Meter("MonitorCronCnt", 1) 174 | pfc.Gauge("MonitorCronTs", endTs-startTs) 175 | } 176 | 177 | func _monitor() { 178 | client := nhttpclient.GetHttpClient("monitor.get", 5*time.Second, 10*time.Second) 179 | for _, host := range g.Config().Monitor.Cluster { 180 | hostInfo := strings.Split(host, ",") // "module,hostname:port/health/monitor/url" 181 | if len(hostInfo) != 2 { 182 | continue 183 | } 184 | //hostType := hostInfo[0] 185 | hostUrl := hostInfo[1] 186 | if !strings.Contains(hostUrl, "http://") { 187 | hostUrl = "http://" + hostUrl 188 | } 189 | 190 | req, _ := http.NewRequest("GET", hostUrl, nil) 191 | req.Header.Set("Connection", "close") 192 | getResp, err := client.Do(req) 193 | if err != nil { 194 | log.Printf(host+", monitor error,", err) 195 | onMonitorErr(host) 196 | continue 197 | } 198 | defer getResp.Body.Close() 199 | 200 | body, err := ioutil.ReadAll(getResp.Body) // body=['o','k',...] 201 | if !(err == nil && len(body) >= 2 && string(body[:2]) == "ok") { // err 202 | log.Println(host, ", error,", err) 203 | onMonitorErr(host) 204 | } else { // get "ok" 205 | onMonitorOk(host) 206 | } 207 | } 208 | } 209 | 210 | func onMonitorErr(host string) { 211 | // change status 212 | s, found := statusCache.Get(host) 213 | if !found { 214 | s = NewStatus() 215 | statusCache.Put(host, s) 216 | } 217 | ss := s.(*Status) 218 | ss.OnErr() 219 | 220 | // alarm 221 | errCnt := ss.GetErrCnt() 222 | if errCnt >= 4 && errCnt <= 16 { 223 | for i := 4; i <= errCnt; i *= 2 { 224 | if errCnt == i { 225 | a := NewAlarm(host, "err", ss.GetErrCnt()) 226 | alarmCache.Put(host, a) 227 | break 228 | } 229 | } 230 | } 231 | } 232 | 233 | func onMonitorOk(host string) { 234 | // change status 235 | s, found := statusCache.Get(host) 236 | if !found { 237 | s = NewStatus() 238 | statusCache.Put(host, s) 239 | } 240 | ss := s.(*Status) 241 | errCnt := ss.GetErrCnt() 242 | ss.OnOk() 243 | 244 | if ss.IsTurnToOk() { 245 | if errCnt >= 4 { //有过alarm, 才能turnOk 246 | // alarm 247 | a := NewAlarm(host, "ok", ss.GetErrCnt()) 248 | alarmCache.Put(host, a) 249 | } 250 | } 251 | } 252 | 253 | // Status Struct 254 | type Status struct { 255 | sync.RWMutex 256 | Status string 257 | LastStatus string 258 | ErrCnt int 259 | OkCnt int 260 | } 261 | 262 | func NewStatus() *Status { 263 | return &Status{Status: "ok", LastStatus: "ok", ErrCnt: 0, OkCnt: 0} 264 | } 265 | 266 | func (s *Status) GetErrCnt() int { 267 | s.RLock() 268 | cnt := s.ErrCnt 269 | s.RUnlock() 270 | return cnt 271 | } 272 | 273 | func (s *Status) OnErr() { 274 | s.Lock() 275 | s.LastStatus = s.Status 276 | s.Status = "err" 277 | s.OkCnt = 0 278 | s.ErrCnt += 1 279 | s.Unlock() 280 | } 281 | 282 | func (s *Status) OnOk() { 283 | s.Lock() 284 | s.LastStatus = s.Status 285 | s.Status = "ok" 286 | s.OkCnt += 1 287 | s.ErrCnt = 0 288 | s.Unlock() 289 | } 290 | 291 | func (s *Status) IsTurnToOk() bool { 292 | s.RLock() 293 | ret := false 294 | if s.LastStatus == "err" && s.Status == "ok" { 295 | ret = true 296 | } 297 | s.RUnlock() 298 | return ret 299 | } 300 | 301 | // AlarmItem Struct 302 | type Alarm struct { 303 | ObjName string 304 | AlarmType string 305 | AlarmCnt int 306 | Ts int64 307 | } 308 | 309 | func NewAlarm(obj string, atype string, cnt int) *Alarm { 310 | return &Alarm{AlarmType: atype, ObjName: obj, AlarmCnt: cnt, Ts: time.Now().Unix()} 311 | } 312 | 313 | func (a *Alarm) String() string { 314 | return fmt.Sprintf("[%s][%s][%d][%s]", ntime.FormatTs(a.Ts), a.AlarmType, a.AlarmCnt, a.ObjName) 315 | } 316 | -------------------------------------------------------------------------------- /perfcounter.json: -------------------------------------------------------------------------------- 1 | { 2 | "tags": "service=anteye", 3 | "bases": ["runtime"], 4 | "push": { 5 | "enabled": true 6 | }, 7 | "http": { 8 | "enabled": true 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /test/debug: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## test home 3 | testdir=$(cd $(dirname $0)/; pwd) 4 | ## word home 5 | workdir=$(dirname $testdir) 6 | cd $workdir 7 | 8 | control=./control 9 | 10 | cfg=./cfg.json 11 | httpport=80 12 | if [ -f $cfg ]; then 13 | httpport=`cat $cfg | grep -A3 "\"http\":" | grep "\"listen\"" | cut -d\" -f4 | cut -d: -f2` 14 | fi 15 | httpprex="127.0.0.1:$httpport" 16 | 17 | ## health 18 | function health(){ 19 | url="/health" 20 | curl -s "$httpprex$url" && printf "\n" 21 | } 22 | 23 | ## proc 24 | function proc(){ 25 | args=$@ 26 | for i in $@; do 27 | url="$url/$i" 28 | done 29 | curl -s "$httpprex$url" | python -m json.tool 30 | } 31 | 32 | ## config 33 | function config(){ 34 | args=$@ 35 | for i in $@; do 36 | url="$url/$i" 37 | done 38 | curl -s "$httpprex$url" | python -m json.tool 39 | } 40 | 41 | ## control 42 | function control(){ 43 | $control $@ 44 | } 45 | 46 | ## debug 47 | function debug(){ 48 | args=$@ 49 | for i in $@; do 50 | url="$url/$i" 51 | done 52 | curl -s "$httpprex$url" && printf "\n" 53 | } 54 | 55 | ## pfc 56 | function pfc(){ 57 | args=$@ 58 | for i in $@; do 59 | url="$url/$i" 60 | done 61 | curl -s "$httpprex$url" | python -m json.tool 62 | } 63 | 64 | action=$1 65 | case $action in 66 | "health") 67 | health 68 | ;; 69 | "debug") 70 | debug $@ 71 | ;; 72 | "config") 73 | config $@ 74 | ;; 75 | "proc") 76 | proc $@ 77 | ;; 78 | "pfc") 79 | pfc $@ 80 | ;; 81 | "") 82 | pfc "pfc/proc/metrics/gauge,meter/json" 83 | ;; 84 | *) 85 | control $@ 86 | ;; 87 | esac --------------------------------------------------------------------------------