├── g ├── g.go ├── redis.go ├── db.go ├── cfg.go └── nodes.go ├── cron ├── check_stale.go ├── sync_routes.go ├── sync_domain.go └── compare.go ├── .gitignore ├── cfg.example.json ├── main.go ├── hbs ├── hbs.go └── node.go ├── README.md └── http └── http.go /g/g.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "github.com/dinp/common/model" 5 | "log" 6 | "runtime" 7 | ) 8 | 9 | func init() { 10 | runtime.GOMAXPROCS(runtime.NumCPU()) 11 | log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) 12 | } 13 | 14 | var ( 15 | RealState = model.NewSafeRealState() 16 | ) 17 | -------------------------------------------------------------------------------- /g/redis.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "github.com/garyburd/redigo/redis" 5 | "time" 6 | ) 7 | 8 | var RedisConnPool *redis.Pool 9 | 10 | func InitRedisConnPool() { 11 | RedisConnPool = &redis.Pool{ 12 | MaxIdle: Config().Redis.MaxIdle, 13 | IdleTimeout: 240 * time.Second, 14 | Dial: func() (redis.Conn, error) { 15 | return redis.Dial("tcp", Config().Redis.Dsn) 16 | }, 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /cron/check_stale.go: -------------------------------------------------------------------------------- 1 | package cron 2 | 3 | import ( 4 | "github.com/dinp/server/g" 5 | "time" 6 | ) 7 | 8 | func CheckStale() { 9 | duration := time.Duration(g.Config().Interval) * time.Second 10 | for { 11 | time.Sleep(duration) 12 | checkStale() 13 | } 14 | } 15 | 16 | func checkStale() { 17 | now := time.Now().Unix() 18 | before := now - 3*int64(g.Config().Interval) 19 | g.DeleteStaleNode(before) 20 | g.RealState.DeleteStale(before) 21 | } 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | *.swp 27 | *.swo 28 | *.log 29 | .idea 30 | .DS_Store 31 | /paas-server 32 | /dinp-server 33 | /var 34 | /server 35 | 36 | -------------------------------------------------------------------------------- /cfg.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "debug": true, 3 | "interval": 5, 4 | "dockerPort": 2375, 5 | "domain": "apps.io", 6 | "localIp": "", 7 | "redis": { 8 | "dsn":"127.0.0.1:6379", 9 | "maxIdle": 10, 10 | "rsPrefix": "/rs/", 11 | "cnamePrefix": "/cname/" 12 | }, 13 | "db": { 14 | "dsn": "root:1234@tcp(127.0.0.1:3306)/dinp?loc=Local&parseTime=true", 15 | "maxIdle": 2 16 | }, 17 | "scribe": { 18 | "ip": "10.5.6.7", 19 | "port": 1463 20 | }, 21 | "http": { 22 | "addr": "0.0.0.0", 23 | "port": 1980 24 | }, 25 | "rpc": { 26 | "addr": "0.0.0.0", 27 | "port": 1970 28 | } 29 | } -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/dinp/server/cron" 7 | "github.com/dinp/server/g" 8 | "github.com/dinp/server/hbs" 9 | "github.com/dinp/server/http" 10 | "os" 11 | ) 12 | 13 | func main() { 14 | cfg := flag.String("c", "cfg.json", "configuration file") 15 | version := flag.Bool("v", false, "show version") 16 | flag.Parse() 17 | 18 | if *version { 19 | fmt.Println(g.VERSION) 20 | os.Exit(0) 21 | } 22 | 23 | g.ParseConfig(*cfg) 24 | 25 | g.InitRedisConnPool() 26 | g.InitDbConnPool() 27 | 28 | go cron.CompareState() 29 | go cron.CheckStale() 30 | go cron.SyncRoutes() 31 | go cron.SyncDomain() 32 | 33 | go http.Start() 34 | hbs.Start() 35 | } 36 | -------------------------------------------------------------------------------- /hbs/hbs.go: -------------------------------------------------------------------------------- 1 | package hbs 2 | 3 | import ( 4 | "fmt" 5 | "github.com/dinp/server/g" 6 | "log" 7 | "net" 8 | "net/rpc" 9 | ) 10 | 11 | func Start() { 12 | addr := fmt.Sprintf("%s:%d", g.Config().Rpc.Addr, g.Config().Rpc.Port) 13 | 14 | tcpAddr, err := net.ResolveTCPAddr("tcp", addr) 15 | if err != nil { 16 | log.Fatalf("net.ResolveTCPAddr fail: %s", err) 17 | } 18 | 19 | listener, err := net.ListenTCP("tcp", tcpAddr) 20 | if err != nil { 21 | log.Fatalf("listen %s fail: %s", addr, err) 22 | } 23 | 24 | rpc.Register(new(NodeState)) 25 | 26 | for { 27 | conn, err := listener.Accept() 28 | if err != nil { 29 | log.Printf("listener.Accept occur error: %s", err) 30 | continue 31 | } 32 | go rpc.ServeConn(conn) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Server 2 | ========== 3 | 4 | 这是整个系统最核心的模块了，主要有以下功能 5 | 6 | - 1. 开一个rpc端口，接收Agent的心跳信息，收集所有计算节点的负载和container列表，存入内存，称之为real state 7 | - 2. 开一个http端口用于调试，暴露内存信息 8 | - 3. 连接Dashboard的DB，定期获取当前app的目标状态，称之为desired state 9 | - 4. 比较real state和desired state，如果container挂了，或者扩容了，就可以比较得知差异，然后创建新的container或者销毁多余的container 10 | - 5. 分析Agent上报的container列表，组织出路由信息，写入redis 11 | - 6. 把配置的scribe连接地址通过环境变量写入container，container中的app就可以把log推到这个scribe服务器 12 | 13 | ## 问题 14 | 15 | **如果server挂了怎么办？** 16 | 17 | Agent配置了两个server的地址，平时只启动一个server，如果server所在机器挂了，去启动另一个即可，Agent会自动重连。此处可以做一个自动选主之类的逻辑，太麻烦，没做 18 | 19 | **如果存放路由信息的redis挂了怎么办？** 20 | 21 | router每次从redis获取了路由之后会缓存到本地内存，redis挂了问题不大，只是不能感知后端container变化了。 22 | 23 | *其实，即使dinp的大部分组件都挂了，只要router没挂，container没挂，已有的服务都不会受影响，只是没法上线、扩容了* 24 | -------------------------------------------------------------------------------- /hbs/node.go: -------------------------------------------------------------------------------- 1 | package hbs 2 | 3 | import ( 4 | "github.com/dinp/common/model" 5 | "github.com/dinp/server/g" 6 | "time" 7 | ) 8 | 9 | type NodeState int 10 | 11 | func (this *NodeState) Push(req *model.NodeRequest, resp *model.NodeResponse) error { 12 | if req == nil { 13 | resp.Code = 1 14 | return nil 15 | } 16 | 17 | g.UpdateNode(&req.Node) 18 | 19 | if req.Containers == nil || len(req.Containers) == 0 { 20 | return nil 21 | } 22 | 23 | now := time.Now().Unix() 24 | 25 | for _, dto := range req.Containers { 26 | container := &model.Container{ 27 | Id: dto.Id, 28 | Ip: req.Ip, 29 | Image: dto.Image, 30 | AppName: dto.AppName, 31 | Ports: dto.Ports, 32 | Status: dto.Status, 33 | UpdateAt: now, 34 | } 35 | g.RealState.UpdateContainer(container) 36 | } 37 | 38 | return nil 39 | } 40 | 41 | func (this *NodeState) NodeDown(ip string, resp *model.NodeResponse) error { 42 | if ip == "" { 43 | resp.Code = 1 44 | return nil 45 | } 46 | 47 | g.DeleteNode(ip) 48 | g.RealState.DeleteByIp(ip) 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /cron/sync_routes.go: -------------------------------------------------------------------------------- 1 | package cron 2 | 3 | import ( 4 | "fmt" 5 | "github.com/dinp/common/model" 6 | "github.com/dinp/server/g" 7 | "github.com/garyburd/redigo/redis" 8 | "log" 9 | "time" 10 | ) 11 | 12 | func SyncRoutes() { 13 | duration := time.Duration(g.Config().Interval) * time.Second 14 | for { 15 | syncRoutes() 16 | time.Sleep(duration) 17 | } 18 | } 19 | 20 | func syncRoutes() { 21 | realNames := g.RealState.Keys() 22 | 23 | rc := g.RedisConnPool.Get() 24 | defer rc.Close() 25 | 26 | err := rc.Send("MULTI") 27 | if err != nil { 28 | log.Printf("[ERROR] rc.Do(\"MULTI\") fail: %v", err) 29 | return 30 | } 31 | 32 | for _, name := range realNames { 33 | sa, _ := g.RealState.GetSafeApp(name) 34 | if !sa.IsNeedUpdateRouter() { 35 | continue 36 | } 37 | _syncOneApp(rc, name, sa) 38 | } 39 | 40 | _, err = rc.Do("EXEC") 41 | if err != nil { 42 | log.Printf("[ERROR] rc.Do(\"EXEC\") fail: %v", err) 43 | } 44 | } 45 | 46 | func _syncOneApp(rc redis.Conn, appName string, app *model.SafeApp) error { 47 | uriKey := fmt.Sprintf("%s%s.%s", g.Config().Redis.RsPrefix, appName, g.Config().Domain) 48 | 49 | debug := g.Config().Debug 50 | if debug { 51 | log.Printf("[Redis] DEL %s", uriKey) 52 | } 53 | 54 | err := rc.Send("DEL", uriKey) 55 | if err != nil { 56 | log.Printf("[ERROR] DEL %s fail: %v", uriKey, err) 57 | return err 58 | } 59 | 60 | cs := app.Containers() 61 | if len(cs) == 0 { 62 | app.NeedUpdateRouter(false) 63 | return nil 64 | } 65 | 66 | args := []interface{}{uriKey} 67 | for _, c := range cs { 68 | args = append(args, fmt.Sprintf("%s:%d", c.Ip, c.Ports[0].PublicPort)) 69 | } 70 | 71 | if debug { 72 | log.Printf("[Redis] LPUSH %v", args) 73 | } 74 | 75 | err = rc.Send("LPUSH", args...) 76 | if err != nil { 77 | log.Printf("[ERROR] LPUSH %v fail: %v", args, err) 78 | } else { 79 | app.NeedUpdateRouter(false) 80 | } 81 | 82 | return err 83 | } 84 | -------------------------------------------------------------------------------- /cron/sync_domain.go: -------------------------------------------------------------------------------- 1 | package cron 2 | 3 | import ( 4 | "fmt" 5 | "github.com/dinp/server/g" 6 | "log" 7 | "time" 8 | ) 9 | 10 | var ( 11 | Domains = make(map[string]string) 12 | DomainsToUpdate = make(map[string]bool) 13 | ) 14 | 15 | func SyncDomain() { 16 | duration := time.Duration(g.Config().Interval) * time.Second 17 | for { 18 | syncDomain() 19 | time.Sleep(duration) 20 | } 21 | } 22 | 23 | func syncDomain() { 24 | _sql := "select domain, app_name from domain where app_id <> 0" 25 | rows, err := g.DB.Query(_sql) 26 | if err != nil { 27 | log.Printf("[ERROR] exec %s fail: %s", _sql, err) 28 | return 29 | } 30 | 31 | needUpdateRedis := false 32 | 33 | for rows.Next() { 34 | var domain, appName string 35 | err = rows.Scan(&domain, &appName) 36 | if err != nil { 37 | log.Printf("[ERROR] %s scan fail: %s", _sql, err) 38 | return 39 | } 40 | 41 | name, existent := Domains[domain] 42 | if !existent || name != appName { 43 | Domains[domain] = appName 44 | DomainsToUpdate[domain] = true 45 | needUpdateRedis = true 46 | } 47 | } 48 | 49 | if !needUpdateRedis { 50 | return 51 | } 52 | 53 | rc := g.RedisConnPool.Get() 54 | defer rc.Close() 55 | 56 | err = rc.Send("MULTI") 57 | if err != nil { 58 | log.Printf("[ERROR] rc.Do(\"MULTI\") fail: %v", err) 59 | return 60 | } 61 | 62 | rsPrefix := g.Config().Redis.RsPrefix 63 | cnamePrefix := g.Config().Redis.CNamePrefix 64 | domain := g.Config().Domain 65 | debug := g.Config().Debug 66 | 67 | for d, toUp := range DomainsToUpdate { 68 | if !toUp { 69 | continue 70 | } 71 | 72 | uriKey := fmt.Sprintf("%s%s.%s", rsPrefix, Domains[d], domain) 73 | cname := fmt.Sprintf("%s%s", cnamePrefix, d) 74 | if debug { 75 | log.Printf("[Redis] SET %s %s", cname, uriKey) 76 | } 77 | rc.Send("SET", cname, uriKey) 78 | DomainsToUpdate[d] = false 79 | } 80 | 81 | _, err = rc.Do("EXEC") 82 | if err != nil { 83 | log.Printf("[ERROR] rc.Do(\"EXEC\") fail: %v", err) 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/dinp/common/model" 7 | "github.com/dinp/server/g" 8 | "log" 9 | "net/http" 10 | ) 11 | 12 | func healthHandler(w http.ResponseWriter, r *http.Request) { 13 | w.Write([]byte("ok")) 14 | } 15 | 16 | func nodesHandler(w http.ResponseWriter, r *http.Request) { 17 | js, err := json.Marshal(g.Nodes) 18 | if err != nil { 19 | http.Error(w, err.Error(), http.StatusInternalServerError) 20 | return 21 | } 22 | 23 | w.Header().Set("Content-Type", "application/json") 24 | w.Write(js) 25 | } 26 | 27 | func realStateHandler(w http.ResponseWriter, r *http.Request) { 28 | js, err := json.Marshal(g.RealState) 29 | if err != nil { 30 | http.Error(w, err.Error(), http.StatusInternalServerError) 31 | return 32 | } 33 | 34 | w.Header().Set("Content-Type", "application/json") 35 | w.Write(js) 36 | } 37 | 38 | func appHandler(w http.ResponseWriter, r *http.Request) { 39 | appName := r.URL.Path[len("/app/"):] 40 | if appName == "" { 41 | http.NotFound(w, r) 42 | return 43 | } 44 | 45 | safeApp, exists := g.RealState.GetSafeApp(appName) 46 | if !exists { 47 | http.NotFound(w, r) 48 | return 49 | } 50 | 51 | cs := safeApp.Containers() 52 | vs := make([]*model.Container, len(cs)) 53 | idx := 0 54 | for _, v := range cs { 55 | vs[idx] = v 56 | idx++ 57 | } 58 | 59 | js, err := json.Marshal(vs) 60 | if err != nil { 61 | http.Error(w, err.Error(), http.StatusInternalServerError) 62 | return 63 | } 64 | 65 | w.Header().Set("Content-Type", "application/json") 66 | w.Write(js) 67 | } 68 | 69 | func Start() { 70 | http.HandleFunc("/health", healthHandler) 71 | http.HandleFunc("/nodes", nodesHandler) 72 | http.HandleFunc("/real", realStateHandler) 73 | http.HandleFunc("/app/", appHandler) 74 | addr := fmt.Sprintf("%s:%d", g.Config().Http.Addr, g.Config().Http.Port) 75 | err := http.ListenAndServe(addr, nil) 76 | if err != nil { 77 | log.Fatalf("ListenAndServe %s fail: %s", addr, err) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /g/db.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/dinp/common/model" 6 | _ "github.com/go-sql-driver/mysql" 7 | "log" 8 | ) 9 | 10 | var DB *sql.DB 11 | 12 | func InitDbConnPool() { 13 | var err error 14 | dbDsn := Config().DB.Dsn 15 | 16 | DB, err = sql.Open("mysql", dbDsn) 17 | if err != nil { 18 | log.Fatalf("sql.Open %s fail: %s", dbDsn, err) 19 | } 20 | 21 | DB.SetMaxIdleConns(Config().DB.MaxIdle) 22 | 23 | err = DB.Ping() 24 | if err != nil { 25 | log.Fatalf("Ping() fail: %s", err) 26 | } 27 | } 28 | 29 | func LoadEnvVarsOf(appName string) (envVars map[string]string, err error) { 30 | envVars = make(map[string]string) 31 | 32 | sq := "select k, v from env where app_name = ?" 33 | 34 | var stmt *sql.Stmt 35 | stmt, err = DB.Prepare(sq) 36 | if err != nil { 37 | log.Printf("[ERROR] prepare sql: %s fail: %v, params: [%s]", sq, err, appName) 38 | return 39 | } 40 | 41 | defer stmt.Close() 42 | 43 | var rows *sql.Rows 44 | rows, err = stmt.Query(appName) 45 | if err != nil { 46 | log.Printf("[ERROR] exec sql: %s fail: %v, params: [%s]", sq, err, appName) 47 | return 48 | } 49 | 50 | for rows.Next() { 51 | var k, v string 52 | err = rows.Scan(&k, &v) 53 | if err != nil { 54 | log.Printf("[ERROR] %s scan fail: %s", sq, err) 55 | return 56 | } 57 | 58 | envVars[k] = v 59 | } 60 | 61 | return 62 | } 63 | 64 | func UpdateAppStatus(app *model.App, status int) error { 65 | if Config().Debug { 66 | log.Printf("[INFO] udpate app: %s status to: %d", app.Name, status) 67 | } 68 | 69 | sq := "update app set status = ? where name = ?" 70 | stmt, err := DB.Prepare(sq) 71 | if err != nil { 72 | log.Printf("[ERROR] prepare sql: %s fail: %v, params: [%d, %s]", sq, err, status, app.Name) 73 | return err 74 | } 75 | defer stmt.Close() 76 | 77 | _, err = stmt.Exec(status, app.Name) 78 | if err != nil { 79 | log.Printf("[ERROR] exec sql: %s fail: %v, params: [%d, %s]", sq, err, status, app.Name) 80 | return err 81 | } 82 | 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /g/cfg.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/toolkits/file" 6 | "github.com/toolkits/net" 7 | "log" 8 | "sync" 9 | ) 10 | 11 | const ( 12 | VERSION = "1.0.1" 13 | ) 14 | 15 | type RedisConfig struct { 16 | Dsn string `json:"dsn"` 17 | MaxIdle int `json:"maxIdle"` 18 | RsPrefix string `json:"rsPrefix"` 19 | CNamePrefix string `json:"cnamePrefix"` 20 | } 21 | 22 | type DBConfig struct { 23 | Dsn string `json:"dsn"` 24 | MaxIdle int `json:"maxIdle"` 25 | } 26 | 27 | type ScribeConfig struct { 28 | Ip string `json:"ip"` 29 | Port int `json:"port"` 30 | } 31 | 32 | type HttpConfig struct { 33 | Addr string `json:"addr"` 34 | Port int `json:"port"` 35 | } 36 | 37 | type RpcConfig struct { 38 | Addr string `json:"addr"` 39 | Port int `json:"port"` 40 | } 41 | 42 | type GlobalConfig struct { 43 | Debug bool `json:"debug"` 44 | Interval int `json:"interval"` 45 | DockerPort int `json:"dockerPort"` 46 | Domain string `json:"domain"` 47 | LocalIp string `json:"localIp"` 48 | Redis *RedisConfig `json:"redis"` 49 | DB *DBConfig `json:"db"` 50 | Scribe *ScribeConfig `json:"scribe"` 51 | Http *HttpConfig `json:"http"` 52 | Rpc *RpcConfig `json:"rpc"` 53 | } 54 | 55 | var ( 56 | ConfigFile string 57 | config *GlobalConfig 58 | configLock = new(sync.RWMutex) 59 | ) 60 | 61 | func Config() *GlobalConfig { 62 | configLock.RLock() 63 | defer configLock.RUnlock() 64 | return config 65 | } 66 | 67 | func ParseConfig(cfg string) { 68 | if cfg == "" { 69 | log.Fatalln("use -c to specify configuration file") 70 | } 71 | 72 | if !file.IsExist(cfg) { 73 | log.Fatalln("config file:", cfg, "is not existent") 74 | } 75 | 76 | ConfigFile = cfg 77 | 78 | configContent, err := file.ToTrimString(cfg) 79 | if err != nil { 80 | log.Fatalln("read config file:", cfg, "fail:", err) 81 | } 82 | 83 | var c GlobalConfig 84 | err = json.Unmarshal([]byte(configContent), &c) 85 | if err != nil { 86 | log.Fatalln("parse config file:", cfg, "fail:", err) 87 | } 88 | 89 | if c.LocalIp == "" { 90 | // detect local ip 91 | localIps, err := net.IntranetIP() 92 | if err != nil { 93 | log.Fatalln("get intranet ip fail:", err) 94 | } 95 | 96 | if len(localIps) == 0 { 97 | log.Fatalln("no intranet ip found") 98 | } 99 | 100 | c.LocalIp = localIps[0] 101 | } 102 | 103 | if c.Http.Addr == "" { 104 | c.Http.Addr = c.LocalIp 105 | } 106 | 107 | if c.Rpc.Addr == "" { 108 | c.Rpc.Addr = c.LocalIp 109 | } 110 | 111 | configLock.Lock() 112 | defer configLock.Unlock() 113 | 114 | config = &c 115 | 116 | if config.Debug { 117 | log.Println("read config file:", cfg, "successfully") 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /g/nodes.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "github.com/dinp/common/model" 5 | "log" 6 | "sort" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | var ( 12 | Nodes = make(map[string]*model.Node) 13 | NodeMutex = new(sync.RWMutex) 14 | ) 15 | 16 | func Clone() map[string]*model.Node { 17 | ret := make(map[string]*model.Node) 18 | NodeMutex.RLock() 19 | defer NodeMutex.RUnlock() 20 | for ip, n := range Nodes { 21 | ret[ip] = n 22 | } 23 | return ret 24 | } 25 | 26 | func UpdateNode(node *model.Node) { 27 | node.UpdateAt = time.Now().Unix() 28 | NodeMutex.Lock() 29 | defer NodeMutex.Unlock() 30 | Nodes[node.Ip] = node 31 | } 32 | 33 | func DeleteStaleNode(before int64) { 34 | need_delete := make([]*model.Node, 0) 35 | NodeMutex.RLock() 36 | for _, node := range Nodes { 37 | if node.UpdateAt < before { 38 | need_delete = append(need_delete, node) 39 | } 40 | } 41 | NodeMutex.RUnlock() 42 | 43 | NodeMutex.Lock() 44 | for _, node := range need_delete { 45 | log.Printf("[NodeDown] ip: %s", node.Ip) 46 | delete(Nodes, node.Ip) 47 | } 48 | NodeMutex.Unlock() 49 | } 50 | 51 | func DeleteNode(ip string) { 52 | NodeMutex.Lock() 53 | defer NodeMutex.Unlock() 54 | delete(Nodes, ip) 55 | } 56 | 57 | func NodeCount() int { 58 | NodeMutex.RLock() 59 | defer NodeMutex.RUnlock() 60 | return len(Nodes) 61 | } 62 | 63 | func TheOne() *model.Node { 64 | NodeMutex.RLock() 65 | defer NodeMutex.RUnlock() 66 | for _, n := range Nodes { 67 | return n 68 | } 69 | return nil 70 | } 71 | 72 | func GetNode(ip string) *model.Node { 73 | NodeMutex.RLock() 74 | defer NodeMutex.RUnlock() 75 | return Nodes[ip] 76 | } 77 | 78 | func ChooseNode(app *model.App, deployCnt int) map[string]int { 79 | ret := make(map[string]int) 80 | size := NodeCount() 81 | if size == 0 { 82 | return ret 83 | } 84 | 85 | if size == 1 { 86 | n := TheOne() 87 | if n != nil && n.MemFree > uint64(deployCnt*app.Memory) { 88 | ret[n.Ip] = deployCnt 89 | return ret 90 | } 91 | log.Printf(">>> memory not enough for %d instance of %s <<<", app.InstanceCnt, app.Name) 92 | return ret 93 | } 94 | 95 | copyNodes := Clone() 96 | 97 | // order by MemFree desc 98 | ns := make(model.NodeSlice, 0, size) 99 | for _, n := range copyNodes { 100 | ns = append(ns, n) 101 | } 102 | 103 | sort.Sort(ns) 104 | 105 | // delete node which MemFree < app.Memory 106 | memFreeIsOK := make([]*model.Node, 0, size) 107 | for _, n := range ns { 108 | if n.MemFree > uint64(app.Memory) { 109 | memFreeIsOK = append(memFreeIsOK, n) 110 | } 111 | } 112 | 113 | size = len(memFreeIsOK) 114 | if size == 0 { 115 | return ret 116 | } 117 | 118 | // node not enough 119 | if size < deployCnt { 120 | 121 | // every node at least create one container 122 | for _, n := range memFreeIsOK { 123 | ret[n.Ip] = 1 124 | } 125 | 126 | done := len(memFreeIsOK) 127 | for { 128 | for _, n := range memFreeIsOK { 129 | ret[n.Ip] += 1 130 | done++ 131 | if done == deployCnt { 132 | goto CHK_MEM 133 | } 134 | } 135 | } 136 | 137 | CHK_MEM: 138 | 139 | for _, n := range memFreeIsOK { 140 | if n.MemFree < uint64(app.Memory*ret[n.Ip]) { 141 | log.Printf(">>> memory not enough for %d instance of %s <<<", app.InstanceCnt, app.Name) 142 | return make(map[string]int) 143 | } 144 | } 145 | 146 | return ret 147 | } 148 | 149 | if size == deployCnt { 150 | for _, n := range memFreeIsOK { 151 | ret[n.Ip] = 1 152 | } 153 | return ret 154 | } 155 | 156 | // node enough 157 | has_deployed_count := app.InstanceCnt - deployCnt 158 | if has_deployed_count == 0 { 159 | // first deploy 160 | done := 0 161 | for _, n := range memFreeIsOK { 162 | ret[n.Ip] = 1 163 | done++ 164 | if done == deployCnt { 165 | return ret 166 | } 167 | } 168 | } 169 | 170 | // we have enough nodes. delete the node which has deployed this app. 171 | // we can delete a maximum of size - deployCnt 172 | can_delete_node_count := size - deployCnt 173 | // the nodes not deploy this app, order by MemFree asc 174 | not_deploy_this_app := make([]*model.Node, 0) 175 | 176 | all_ok := false 177 | for i := size - 1; i >= 0; i-- { 178 | if all_ok { 179 | not_deploy_this_app = append(not_deploy_this_app, memFreeIsOK[i]) 180 | continue 181 | } 182 | 183 | if !RealState.HasRelation(app.Name, memFreeIsOK[i].Ip) { 184 | not_deploy_this_app = append(not_deploy_this_app, memFreeIsOK[i]) 185 | continue 186 | } 187 | 188 | if can_delete_node_count > 0 { 189 | has_deployed_count-- 190 | if has_deployed_count == 0 { 191 | // the rest nodes are all not deploy this app 192 | all_ok = true 193 | continue 194 | } 195 | can_delete_node_count-- 196 | } else { 197 | not_deploy_this_app = append(not_deploy_this_app, memFreeIsOK[i]) 198 | all_ok = true 199 | } 200 | } 201 | 202 | // order by MemFree desc 203 | cnt := 0 204 | for i := len(not_deploy_this_app) - 1; i >= 0; i-- { 205 | ret[not_deploy_this_app[i].Ip] = 1 206 | cnt++ 207 | if cnt == deployCnt { 208 | return ret 209 | } 210 | } 211 | 212 | return make(map[string]int) 213 | } 214 | -------------------------------------------------------------------------------- /cron/compare.go: -------------------------------------------------------------------------------- 1 | package cron 2 | 3 | import ( 4 | "fmt" 5 | "github.com/dinp/common/model" 6 | "github.com/dinp/server/g" 7 | "github.com/fsouza/go-dockerclient" 8 | "github.com/toolkits/slice" 9 | "log" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | func getDesiredState() (map[string]*model.App, error) { 15 | sql := "select name, memory, instance, image, status from app where status = 0 and image <> ''" 16 | rows, err := g.DB.Query(sql) 17 | if err != nil { 18 | log.Printf("[ERROR] exec %s fail: %s", sql, err) 19 | return nil, err 20 | } 21 | 22 | var desiredState = make(map[string]*model.App) 23 | for rows.Next() { 24 | var app model.App 25 | err = rows.Scan(&app.Name, &app.Memory, &app.InstanceCnt, &app.Image, &app.Status) 26 | if err != nil { 27 | log.Printf("[ERROR] %s scan fail: %s", sql, err) 28 | return nil, err 29 | } 30 | 31 | desiredState[app.Name] = &app 32 | } 33 | 34 | return desiredState, nil 35 | } 36 | 37 | func CompareState() { 38 | duration := time.Duration(g.Config().Interval) * time.Second 39 | time.Sleep(duration) 40 | for { 41 | time.Sleep(duration) 42 | compareState() 43 | } 44 | } 45 | 46 | func compareState() { 47 | desiredState, err := getDesiredState() 48 | if err != nil { 49 | log.Println("[ERROR] get desired state fail:", err) 50 | return 51 | } 52 | 53 | debug := g.Config().Debug 54 | 55 | if debug { 56 | log.Println("comparing......") 57 | } 58 | 59 | if len(desiredState) == 0 { 60 | if debug { 61 | log.Println("no desired app. do nothing") 62 | } 63 | // do nothing. 64 | return 65 | } 66 | 67 | newAppSlice := []string{} 68 | 69 | for name, app := range desiredState { 70 | if !g.RealState.RealAppExists(name) { 71 | if debug && app.InstanceCnt > 0 { 72 | log.Println("[=-NEW-=]:", name) 73 | } 74 | newAppSlice = append(newAppSlice, name) 75 | createNewContainer(app, app.InstanceCnt) 76 | } 77 | } 78 | 79 | realNames := g.RealState.Keys() 80 | 81 | for ii, name := range realNames { 82 | if debug { 83 | log.Printf("#%d: %s", ii, name) 84 | } 85 | 86 | if slice.ContainsString(newAppSlice, name) { 87 | continue 88 | } 89 | 90 | app, exists := desiredState[name] 91 | if !exists { 92 | if debug { 93 | log.Println("[=-DEL-=]:", name) 94 | } 95 | dropApp(name) 96 | continue 97 | } 98 | 99 | sa, _ := g.RealState.GetSafeApp(name) 100 | isOld, olds := sa.IsOldVersion(app.Image) 101 | if isOld { 102 | if len(olds) > 0 || app.InstanceCnt > 0 { 103 | log.Println("[=-UPGRADE-=]") 104 | } 105 | // deploy new instances 106 | createNewContainer(app, app.InstanceCnt) 107 | // delete old instances 108 | for _, c := range olds { 109 | dropContainer(c) 110 | } 111 | 112 | continue 113 | } 114 | 115 | nowCnt := sa.ContainerCount() 116 | 117 | if nowCnt < app.InstanceCnt { 118 | if debug { 119 | log.Printf("add:%d", app.InstanceCnt-nowCnt) 120 | } 121 | createNewContainer(app, app.InstanceCnt-nowCnt) 122 | continue 123 | } 124 | 125 | if nowCnt > app.InstanceCnt { 126 | if debug { 127 | log.Printf("del:%d", nowCnt-app.InstanceCnt) 128 | } 129 | dropContainers(sa.Containers(), nowCnt-app.InstanceCnt) 130 | } 131 | } 132 | } 133 | 134 | func createNewContainer(app *model.App, deployCnt int) { 135 | if deployCnt == 0 { 136 | return 137 | } 138 | 139 | if app.Status != model.AppStatus_Success { 140 | if g.Config().Debug { 141 | log.Printf("!!! App=%s Status = %d", app.Name, app.Status) 142 | } 143 | return 144 | } 145 | 146 | ip_count := g.ChooseNode(app, deployCnt) 147 | if len(ip_count) == 0 { 148 | log.Println("no node..zZ") 149 | return 150 | } 151 | 152 | for ip, count := range ip_count { 153 | for k := 0; k < count; k++ { 154 | DockerRun(app, ip) 155 | } 156 | } 157 | } 158 | 159 | func dropApp(appName string) { 160 | if appName == "" { 161 | return 162 | } 163 | 164 | if g.Config().Debug { 165 | log.Println("drop app:", appName) 166 | } 167 | 168 | sa, _ := g.RealState.GetSafeApp(appName) 169 | cs := sa.Containers() 170 | for _, c := range cs { 171 | dropContainer(c) 172 | } 173 | g.RealState.DeleteSafeApp(appName) 174 | 175 | rc := g.RedisConnPool.Get() 176 | defer rc.Close() 177 | 178 | uriKey := fmt.Sprintf("%s%s.%s", g.Config().Redis.RsPrefix, appName, g.Config().Domain) 179 | rc.Do("DEL", uriKey) 180 | } 181 | 182 | func dropContainers(cs []*model.Container, cnt int) { 183 | if cnt == 0 { 184 | return 185 | } 186 | 187 | done := 0 188 | for _, c := range cs { 189 | dropContainer(c) 190 | done++ 191 | if done == cnt { 192 | break 193 | } 194 | } 195 | } 196 | 197 | func dropContainer(c *model.Container) { 198 | 199 | if g.Config().Debug { 200 | log.Println("drop container:", c) 201 | } 202 | 203 | addr := fmt.Sprintf("http://%s:%d", c.Ip, g.Config().DockerPort) 204 | client, err := docker.NewClient(addr) 205 | if err != nil { 206 | log.Println("docker.NewClient fail:", err) 207 | return 208 | } 209 | 210 | err = client.RemoveContainer(docker.RemoveContainerOptions{ID: c.Id, Force: true}) 211 | if err != nil { 212 | log.Println("docker.RemoveContainer fail:", err) 213 | return 214 | } 215 | 216 | // remember to delete real state map item 217 | sa, exists := g.RealState.GetSafeApp(c.AppName) 218 | if exists { 219 | sa.DeleteContainer(c) 220 | } 221 | } 222 | 223 | func BuildEnvArray(envVars map[string]string) []string { 224 | size := len(envVars) 225 | if size == 0 { 226 | return []string{} 227 | } 228 | 229 | arr := make([]string, size) 230 | idx := 0 231 | for k, v := range envVars { 232 | arr[idx] = fmt.Sprintf("%s=%s", k, v) 233 | idx++ 234 | } 235 | 236 | return arr 237 | } 238 | 239 | func ParseRepositoryTag(repos string) (string, string) { 240 | n := strings.LastIndex(repos, ":") 241 | if n < 0 { 242 | return repos, "" 243 | } 244 | if tag := repos[n+1:]; !strings.Contains(tag, "/") { 245 | return repos[:n], tag 246 | } 247 | return repos, "" 248 | } 249 | 250 | func DockerRun(app *model.App, ip string) { 251 | if g.Config().Debug { 252 | log.Printf("create container. app:%s, ip:%s\n", app.Name, ip) 253 | } 254 | 255 | envVars, err := g.LoadEnvVarsOf(app.Name) 256 | if err != nil { 257 | log.Println("[ERROR] load env fail:", err) 258 | return 259 | } 260 | 261 | envVars["APP_NAME"] = app.Name 262 | envVars["HOST_IP"] = ip 263 | if g.Config().Scribe.Ip != "" { 264 | envVars["SCRIBE_IP"] = g.Config().Scribe.Ip 265 | } else { 266 | envVars["SCRIBE_IP"] = ip 267 | } 268 | envVars["SCRIBE_PORT"] = fmt.Sprintf("%d", g.Config().Scribe.Port) 269 | 270 | addr := fmt.Sprintf("http://%s:%d", ip, g.Config().DockerPort) 271 | 272 | client, err := docker.NewClient(addr) 273 | if err != nil { 274 | log.Println("[ERROR] docker.NewClient fail:", err) 275 | return 276 | } 277 | 278 | opts := docker.CreateContainerOptions{ 279 | Config: &docker.Config{ 280 | Memory: int64(app.Memory * 1024 * 1024), 281 | ExposedPorts: map[docker.Port]struct{}{ 282 | docker.Port("8080/tcp"): {}, 283 | }, 284 | Image: app.Image, 285 | AttachStdin: false, 286 | AttachStdout: false, 287 | AttachStderr: false, 288 | Env: BuildEnvArray(envVars), 289 | }, 290 | } 291 | 292 | container, err := client.CreateContainer(opts) 293 | 294 | if err != nil { 295 | if err == docker.ErrNoSuchImage { 296 | repos, tag := ParseRepositoryTag(app.Image) 297 | e := client.PullImage(docker.PullImageOptions{Repository: repos, Tag: tag}, docker.AuthConfiguration{}) 298 | if e != nil { 299 | log.Println("[ERROR] pull image", app.Image, "fail:", e) 300 | return 301 | } 302 | 303 | // retry 304 | container, err = client.CreateContainer(opts) 305 | if err != nil { 306 | log.Println("[ERROR] retry create container fail:", err, "ip:", ip) 307 | g.UpdateAppStatus(app, model.AppStatus_CreateContainerFail) 308 | return 309 | } 310 | } else { 311 | log.Println("[ERROR] create container fail:", err, "ip:", ip) 312 | if err != nil && strings.Contains(err.Error(), "cannot connect") { 313 | g.DeleteNode(ip) 314 | g.RealState.DeleteByIp(ip) 315 | return 316 | } 317 | g.UpdateAppStatus(app, model.AppStatus_CreateContainerFail) 318 | return 319 | } 320 | } 321 | 322 | err = client.StartContainer(container.ID, &docker.HostConfig{ 323 | PortBindings: map[docker.Port][]docker.PortBinding{ 324 | "8080/tcp": []docker.PortBinding{docker.PortBinding{}}, 325 | }, 326 | }) 327 | 328 | if err != nil { 329 | log.Println("[ERROR] docker.StartContainer fail:", err) 330 | g.UpdateAppStatus(app, model.AppStatus_StartContainerFail) 331 | return 332 | } 333 | 334 | if g.Config().Debug { 335 | log.Println("start container success:-)") 336 | } 337 | 338 | } 339 | --------------------------------------------------------------------------------