├── README.md ├── cfg.sample.json ├── cron └── healthcheck.go ├── g ├── cfg.go ├── db.go └── g.go └── main.go /README.md: -------------------------------------------------------------------------------- 1 | HM 2 | ========== 3 | 4 | HM(Health Monitor)用于APP rs (docker container) 的7层健康检查。 5 | 如要开启APP rs的7层健康检查，在DashBoard的APP页面的health栏填写健康检查的URL (如/health)。 6 | 7 | HM每隔check_interval (单位s) 8 | - 从Server模块的DB中查询需要健康检查的APP列表 (即health字段不为空的APP)， 9 | - 从Server模块的HTTP接口中查询APP及相应的rs列表， 10 | 然后curl相应的rs，如果在response_timeout (单位s) 内返回内容包括health_sign，则视该rs为健康；否则将杀掉该rs (docker container)，由Server模块进行调度创建新的rs。 11 | 12 | ## 配置项说明 13 | 14 | - **debug**: true/false 只影响打印的log 15 | - **check_interval**: 健康检查的周期，单位s 16 | - **dockerPort**: Docker Daemon的侦听端口 17 | - **response_timeout**: APP health接口的响应超时时间，单位s 18 | - **health_sign**: APP health接口的返回内容，如'''ok''' 19 | - **server_http_api**: DINP Server模块的http api接口 20 | - **db**: DINP Server模块数据库的地址，以及超时时间 21 | 22 | ## Install 23 | 24 | ``` 25 | mkdir -p $GOPATH/src/github.com/dinp 26 | cd $GOPATH/src/github.com/dinp; git clone https://github.com/dinp/hm.git 27 | cd hm 28 | go get ./... 29 | 30 | # check cfg.json, depend docker daemon and server 31 | hm -c cfg.json 32 | ``` 33 | -------------------------------------------------------------------------------- /cfg.sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "debug": false, 3 | "check_interval": 5, 4 | "dockerPort": 2375, 5 | "response_timeout": 3, 6 | "health_sign": "ok", 7 | "server_http_api": "http://10.1.2.3:1980/real", 8 | "db": { 9 | "dsn": "root:1234@tcp(10.1.2.3:3306)/dash?loc=Local&parseTime=true", 10 | "maxIdle": 2 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /cron/healthcheck.go: -------------------------------------------------------------------------------- 1 | package cron 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/dinp/common/model" 7 | "github.com/dinp/hm/g" 8 | "github.com/fsouza/go-dockerclient" 9 | "github.com/go-av/curl" 10 | "log" 11 | "strconv" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | func HealthCheck() { 17 | duration := time.Duration(g.Config().CheckInterval) * time.Second 18 | time.Sleep(duration) 19 | for { 20 | time.Sleep(duration) 21 | healthCheck() 22 | } 23 | } 24 | 25 | func healthCheck() { 26 | err, httpBody := curl.String(g.Config().ServerHttpApi, "timeout=", time.Second*5) 27 | if err != nil { 28 | log.Printf("[ERROR] curl server http api fail: %s", err) 29 | return 30 | } 31 | 32 | json.Unmarshal([]byte(httpBody), &g.RealState) 33 | 34 | appNames := g.RealState.Keys() 35 | checkList, _ := getCheckList() 36 | for _, name := range appNames { 37 | if len(checkList) > 0 { 38 | for app, health := range checkList { 39 | if name == app { 40 | sa, _ := g.RealState.GetSafeApp(name) 41 | for _, c := range sa.M { 42 | rsAddress := c.Ip + ":" + strconv.Itoa(c.Ports[0].PublicPort) 43 | rsHealthAddress := "http://" + rsAddress + health 44 | 45 | err, responseBody := curl.String(rsHealthAddress, "timeout=", time.Second*time.Duration(g.Config().ResponseTimeout)) 46 | if err != nil { 47 | log.Printf("[ERROR] curl app:%s rs:%s fail: %s", name, rsHealthAddress, err) 48 | log.Printf("[WARN] kill rs %s", rsAddress) 49 | dropContainer(c) 50 | 51 | continue 52 | } 53 | 54 | if strings.Contains(responseBody, g.Config().HealthSign) == false { 55 | log.Printf("[ERROR] app:%s rs:%s health check fail", name, rsHealthAddress) 56 | log.Printf("[WARN] kill rs %s", rsAddress) 57 | dropContainer(c) 58 | 59 | continue 60 | } 61 | } 62 | } 63 | } 64 | } 65 | } 66 | } 67 | 68 | func getCheckList() (map[string]string, error) { 69 | sql := "select name, health from app where health <> ''" 70 | rows, err := g.DB.Query(sql) 71 | if err != nil { 72 | log.Printf("[ERROR] exec %s fail: %s", sql, err) 73 | return nil, err 74 | } 75 | 76 | var checkList = make(map[string]string) 77 | for rows.Next() { 78 | var app, health string 79 | err = rows.Scan(&app, &health) 80 | if err != nil { 81 | log.Printf("[ERROR] %s scan fail: %s", sql, err) 82 | return nil, err 83 | } 84 | 85 | checkList[app] = health 86 | } 87 | 88 | return checkList, nil 89 | } 90 | 91 | func dropContainer(c *model.Container) { 92 | 93 | if g.Config().Debug { 94 | log.Println("drop container:", c) 95 | } 96 | 97 | addr := fmt.Sprintf("http://%s:%d", c.Ip, g.Config().DockerPort) 98 | client, err := docker.NewClient(addr) 99 | if err != nil { 100 | log.Println("docker.NewClient fail:", err) 101 | return 102 | } 103 | 104 | err = client.RemoveContainer(docker.RemoveContainerOptions{ID: c.Id, Force: true}) 105 | if err != nil { 106 | log.Println("docker.RemoveContainer fail:", err) 107 | return 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /g/cfg.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/toolkits/file" 6 | "log" 7 | "sync" 8 | ) 9 | 10 | const ( 11 | VERSION = "1.0.0" 12 | ) 13 | 14 | type DBConfig struct { 15 | Dsn string `json:"dsn"` 16 | MaxIdle int `json:"maxIdle"` 17 | } 18 | 19 | type GlobalConfig struct { 20 | Debug bool `json:"debug"` 21 | CheckInterval int `json:"check_interval"` 22 | DockerPort int `json:"dockerPort"` 23 | ResponseTimeout int `json:"response_timeout"` 24 | HealthSign string `json:"health_sign"` 25 | ServerHttpApi string `json:"server_http_api"` 26 | DB *DBConfig `json:"db"` 27 | } 28 | 29 | var ( 30 | ConfigFile string 31 | config *GlobalConfig 32 | configLock = new(sync.RWMutex) 33 | ) 34 | 35 | func Config() *GlobalConfig { 36 | configLock.RLock() 37 | defer configLock.RUnlock() 38 | return config 39 | } 40 | 41 | func ParseConfig(cfg string) { 42 | if cfg == "" { 43 | log.Fatalln("use -c to specify configuration file") 44 | } 45 | 46 | if !file.IsExist(cfg) { 47 | log.Fatalln("config file:", cfg, "is not existent") 48 | } 49 | 50 | ConfigFile = cfg 51 | 52 | configContent, err := file.ToTrimString(cfg) 53 | if err != nil { 54 | log.Fatalln("read config file:", cfg, "fail:", err) 55 | } 56 | 57 | var c GlobalConfig 58 | err = json.Unmarshal([]byte(configContent), &c) 59 | if err != nil { 60 | log.Fatalln("parse config file:", cfg, "fail:", err) 61 | } 62 | 63 | configLock.Lock() 64 | defer configLock.Unlock() 65 | 66 | config = &c 67 | 68 | if config.Debug { 69 | log.Println("read config file:", cfg, "successfully") 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /g/db.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "database/sql" 5 | _ "github.com/go-sql-driver/mysql" 6 | "log" 7 | ) 8 | 9 | var DB *sql.DB 10 | 11 | func InitDbConnPool() { 12 | var err error 13 | dbDsn := Config().DB.Dsn 14 | 15 | DB, err = sql.Open("mysql", dbDsn) 16 | if err != nil { 17 | log.Fatalf("sql.Open %s fail: %s", dbDsn, err) 18 | } 19 | 20 | DB.SetMaxIdleConns(Config().DB.MaxIdle) 21 | 22 | err = DB.Ping() 23 | if err != nil { 24 | log.Fatalf("Ping() fail: %s", err) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /g/g.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "github.com/dinp/common/model" 5 | "log" 6 | "runtime" 7 | ) 8 | 9 | func init() { 10 | runtime.GOMAXPROCS(runtime.NumCPU()) 11 | log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) 12 | } 13 | 14 | var ( 15 | RealState = model.NewSafeRealState() 16 | ) 17 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/dinp/hm/cron" 7 | "github.com/dinp/hm/g" 8 | "os" 9 | ) 10 | 11 | func main() { 12 | cfg := flag.String("c", "cfg.json", "configuration file") 13 | version := flag.Bool("v", false, "show version") 14 | flag.Parse() 15 | 16 | if *version { 17 | fmt.Println(g.VERSION) 18 | os.Exit(0) 19 | } 20 | 21 | g.ParseConfig(*cfg) 22 | 23 | g.InitDbConnPool() 24 | 25 | go cron.HealthCheck() 26 | 27 | select {} 28 | } 29 | --------------------------------------------------------------------------------