├── .gitignore ├── api ├── api.go ├── request.go ├── utils.go └── http.go ├── defines ├── app.go ├── wrapper.go ├── config.go ├── lenz.go └── manager.go ├── g ├── transfer.go ├── conn.go └── config.go ├── common └── common.go ├── README.md ├── network ├── vlan.go ├── route_darwin.go ├── vlan_darwin.go ├── route_linux.go └── vlan_linux.go ├── health └── ping.go ├── agent.yaml.sample ├── make ├── logs └── logs.go ├── main.go ├── app ├── metrics.go ├── eru.go └── limit.go ├── lenz ├── lenz.go ├── streamer.go ├── conn.go ├── routes.go └── attacher.go ├── utils ├── utils_test.go └── utils.go └── status └── status.go /.gitignore: -------------------------------------------------------------------------------- 1 | agent 2 | .vendor/ 3 | *.out 4 | agent.test 5 | agent.yaml 6 | eru-agent 7 | build/ 8 | -------------------------------------------------------------------------------- /api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/projecteru/eru-agent/g" 5 | ) 6 | 7 | func Serve() { 8 | if g.Config.API.Addr != "" { 9 | go HTTPServe() 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /defines/app.go: -------------------------------------------------------------------------------- 1 | package defines 2 | 3 | type Meta struct { 4 | ID string 5 | Pid int 6 | Name string 7 | EntryPoint string 8 | Ident string 9 | Extend map[string]interface{} 10 | } 11 | -------------------------------------------------------------------------------- /g/transfer.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "github.com/projecteru/eru-agent/logs" 5 | "github.com/projecteru/eru-agent/utils" 6 | ) 7 | 8 | var Transfers *utils.HashBackends 9 | 10 | func InitTransfers() { 11 | Transfers = utils.NewHashBackends(Config.Metrics.Transfers) 12 | logs.Info("Transfers initiated") 13 | } 14 | -------------------------------------------------------------------------------- /common/common.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | const ( 4 | STATUS_DIE = "die" 5 | STATUS_START = "start" 6 | 7 | CNAME_NUM = 3 8 | VLAN_PREFIX = "vnbe" 9 | DEFAULT_BR = "eth0" 10 | 11 | VERSION = "0.15.2" 12 | 13 | OOM_KILLED = 1 14 | DATETIME_FORMAT = "2006-01-02 15:04:05" 15 | 16 | LENZ_DEFAULT = "lenz_default" 17 | 18 | STATS_TIMEOUT = 2 19 | STATS_FORCE_DONE = 3 20 | ) 21 | -------------------------------------------------------------------------------- /api/request.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/projecteru/eru-agent/logs" 7 | ) 8 | 9 | type Request struct { 10 | http.Request 11 | Start int 12 | Limit int 13 | } 14 | 15 | func (r *Request) Init() { 16 | r.Start = Atoi(r.Form.Get("start"), 0) 17 | r.Limit = Atoi(r.Form.Get("limit"), 20) 18 | } 19 | 20 | func NewRequest(r *http.Request) *Request { 21 | req := &Request{*r, 0, 20} 22 | req.Init() 23 | logs.Debug("HTTP request", req.Method, req.URL.Path) 24 | return req 25 | } 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Eru Agent 2 | ========= 3 | 4 | ## Features 5 | 6 | Agent of Project Eru, for those reasons: 7 | 8 | 1. Check containers status, if crashed, report to eru core. 9 | 2. Get containers metrics, send to [Graphite](http://graphite.wikidot.com/). 10 | 3. Forwarding containers' log to remote collectors like syslog. 11 | 4. Support macvlan and calico SDN. 12 | 13 | ## Run 14 | 15 | Agent has a configure file named `agent.yaml`, you can execute agent like: 16 | 17 | agent -c agent.yaml [-DEBUG] 18 | 19 | ## APIs 20 | 21 | ### HTTP 22 | 23 | Coming soon... 24 | -------------------------------------------------------------------------------- /defines/wrapper.go: -------------------------------------------------------------------------------- 1 | package defines 2 | 3 | import "reflect" 4 | 5 | func MakeWrapper(fptr interface{}) { 6 | var maker = func(in []reflect.Value) []reflect.Value { 7 | wrapper := in[0].Elem() 8 | client := in[1] 9 | wrapperType := wrapper.Type() 10 | for i := 1; i < wrapperType.NumField(); i++ { 11 | field := wrapper.Field(i) 12 | f := client.MethodByName(wrapperType.Field(i).Name) 13 | field.Set(f) 14 | } 15 | return []reflect.Value{in[0]} 16 | } 17 | fn := reflect.ValueOf(fptr).Elem() 18 | v := reflect.MakeFunc(fn.Type(), maker) 19 | fn.Set(v) 20 | } 21 | -------------------------------------------------------------------------------- /network/vlan.go: -------------------------------------------------------------------------------- 1 | package network 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/projecteru/eru-agent/g" 7 | "github.com/projecteru/eru-agent/logs" 8 | "github.com/projecteru/eru-agent/utils" 9 | "github.com/vishvananda/netlink" 10 | ) 11 | 12 | var Devices *utils.HashBackends 13 | var lock sync.Mutex 14 | 15 | func InitVlan() { 16 | Devices = utils.NewHashBackends(g.Config.VLan.Physical) 17 | logs.Info("Vlan initiated") 18 | } 19 | 20 | func DelVlan(link netlink.Link) { 21 | if err := netlink.LinkDel(link); err != nil { 22 | logs.Debug("Delete device failed", err) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /health/ping.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/projecteru/eru-agent/g" 8 | "github.com/projecteru/eru-agent/logs" 9 | "github.com/projecteru/eru-agent/utils" 10 | ) 11 | 12 | func Check() { 13 | go ping() 14 | } 15 | 16 | func ping() { 17 | ticker := time.Tick(time.Duration(g.Config.Docker.Health) * time.Second) 18 | for _ = range ticker { 19 | if err := g.Docker.Ping(); err != nil { 20 | url := fmt.Sprintf("%s/api/host/%s/down/", g.Config.Eru.Endpoint, g.Config.HostName) 21 | utils.DoPut(url) 22 | logs.Assert(err, "Docker exit") 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /api/utils.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "strconv" 7 | ) 8 | 9 | type JSON map[string]interface{} 10 | 11 | func JSONWrapper(f func(*Request) (int, interface{})) func(http.ResponseWriter, *http.Request) { 12 | return func(w http.ResponseWriter, req *http.Request) { 13 | r := NewRequest(req) 14 | w.Header().Set("Content-Type", "application/json") 15 | code, result := f(r) 16 | w.WriteHeader(code) 17 | json.NewEncoder(w).Encode(result) 18 | } 19 | } 20 | 21 | func Atoi(s string, def int) int { 22 | if r, err := strconv.Atoi(s); err != nil { 23 | return def 24 | } else { 25 | return r 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /agent.yaml.sample: -------------------------------------------------------------------------------- 1 | pidfile: /mnt/sda1/tmp/agent.pid 2 | 3 | eru: 4 | endpoint: http://localhost:5000 5 | 6 | limit: 7 | memory: 5293824 8 | 9 | docker: 10 | endpoint: tcp://192.168.59.103:2376 11 | cert: cert.pem 12 | key: key.pem 13 | ca: ca.pem 14 | health: 30 15 | 16 | lenz: 17 | forwards: 18 | - udp://10.100.1.154:50433 19 | stdout: False 20 | count: 10 21 | 22 | metrics: 23 | step: 30 24 | transfers: 25 | - 10.1.201.45:8125 26 | 27 | vlan: 28 | physical: 29 | - em1 30 | calico: 10.10.177.49:2379 31 | 32 | redis: 33 | host: 127.0.0.1 34 | port: 6379 35 | min: 5 36 | max: 100 37 | 38 | api: 39 | addr: 0.0.0.0:12345 40 | -------------------------------------------------------------------------------- /network/route_darwin.go: -------------------------------------------------------------------------------- 1 | package network 2 | 3 | import ( 4 | "github.com/projecteru/eru-agent/g" 5 | "github.com/projecteru/eru-agent/logs" 6 | ) 7 | 8 | func SetDefaultRoute(cid, gateway string) bool { 9 | _, err := g.Docker.InspectContainer(cid) 10 | if err != nil { 11 | logs.Info("VLanSetter inspect docker failed", err) 12 | return false 13 | } 14 | logs.Info("Set default route success", cid, gateway) 15 | return true 16 | } 17 | 18 | func AddRoute(cid, CIDR string, ifc string) bool { 19 | _, err := g.Docker.InspectContainer(cid) 20 | if err != nil { 21 | logs.Info("VLanSetter inspect docker failed", err) 22 | return false 23 | } 24 | logs.Info("Add route success", cid, CIDR, ifc) 25 | return true 26 | } 27 | -------------------------------------------------------------------------------- /make: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ROOT="`pwd`/build" 4 | BIN="$ROOT/usr/local/bin" 5 | RPMDIR="/root/rpmbuild/RPMS/x86_64" 6 | METADIR="/root/rpmbuild/" 7 | 8 | if [ ! -x "$ROOT" ]; then 9 | echo $ROOT not exists 10 | exit 1 11 | fi 12 | 13 | if [ ! -x "$BIN" ]; then 14 | mkdir -p $BIN 15 | fi 16 | 17 | go build -ldflags "-s -w" -a -tags netgo -installsuffix netgo 18 | mv eru-agent $BIN 19 | OUTPUT=($($BIN/eru-agent -v 2>&1)) 20 | VERSION=${OUTPUT[-1]} 21 | echo $VERSION build begin 22 | 23 | fpm -f -s dir -t rpm -n eru-agent --epoch 0 -v $VERSION --iteration 1.el7 -C $ROOT -p $RPMDIR --verbose --category 'Development/App' --description 'docker eru agent' --url 'github.com' --license 'BSD' --no-rpm-sign usr etc 24 | createrepo --update $METADIR 25 | 26 | -------------------------------------------------------------------------------- /logs/logs.go: -------------------------------------------------------------------------------- 1 | package logs 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "runtime" 7 | "strings" 8 | ) 9 | 10 | var Mode bool = false 11 | 12 | func init() { 13 | log.SetPrefix("Agent ") 14 | } 15 | 16 | func Info(v ...interface{}) { 17 | f := getShortFile() 18 | v = append([]interface{}{f}, v...) 19 | log.Println(v...) 20 | } 21 | 22 | func Debug(v ...interface{}) { 23 | if Mode { 24 | f := getShortFile() 25 | v = append([]interface{}{f}, v...) 26 | log.Println(v...) 27 | } 28 | } 29 | 30 | func Assert(err error, context string) { 31 | if err != nil { 32 | f := getShortFile() 33 | log.Fatal(f, context+": ", err) 34 | } 35 | } 36 | 37 | func getShortFile() string { 38 | _, file, line, ok := runtime.Caller(2) 39 | if !ok { 40 | file = "/???/???" 41 | line = 0 42 | } 43 | s := strings.Split(file, "/") 44 | l := len(s) 45 | f := fmt.Sprintf("%s.%s:%d", s[l-2], s[l-1], line) 46 | return f 47 | } 48 | -------------------------------------------------------------------------------- /defines/config.go: -------------------------------------------------------------------------------- 1 | package defines 2 | 3 | type DockerConfig struct { 4 | Endpoint string 5 | Ca string 6 | Key string 7 | Cert string 8 | Health int 9 | } 10 | 11 | type EruConfig struct { 12 | Endpoint string 13 | } 14 | 15 | type LenzConfig struct { 16 | Routes string 17 | Forwards []string 18 | Stdout bool 19 | Count int 20 | } 21 | 22 | type MetricsConfig struct { 23 | Step int64 24 | Transfers []string 25 | } 26 | 27 | type RedisConfig struct { 28 | Host string 29 | Port int 30 | Min int 31 | Max int 32 | } 33 | 34 | type VLanConfig struct { 35 | Physical []string 36 | Calico string 37 | } 38 | 39 | type APIConfig struct { 40 | Addr string 41 | } 42 | 43 | type LimitConfig struct { 44 | Memory uint64 45 | } 46 | 47 | type AgentConfig struct { 48 | HostName string `yaml:"hostname"` 49 | PidFile string 50 | 51 | Docker DockerConfig 52 | Eru EruConfig 53 | Lenz LenzConfig 54 | Metrics MetricsConfig 55 | VLan VLanConfig 56 | Redis RedisConfig 57 | API APIConfig 58 | Limit LimitConfig 59 | } 60 | -------------------------------------------------------------------------------- /network/vlan_darwin.go: -------------------------------------------------------------------------------- 1 | package network 2 | 3 | import ( 4 | "github.com/projecteru/eru-agent/g" 5 | "github.com/projecteru/eru-agent/logs" 6 | "github.com/vishvananda/netlink" 7 | ) 8 | 9 | func AddVlan(vethName, ips, cid string) bool { 10 | _, err := g.Docker.InspectContainer(cid) 11 | if err != nil { 12 | logs.Info("VLanSetter inspect docker failed", err) 13 | return false 14 | } 15 | logs.Info("Add VLAN device success", cid, vethName) 16 | return true 17 | } 18 | 19 | func DelMacVlanDevice(vethName string) error { 20 | } 21 | 22 | func AddMacVlanDevice(vethName, seq string) error { 23 | return nil 24 | } 25 | 26 | func BindAndSetup(veth netlink.Link, ips string) error { 27 | return nil 28 | } 29 | 30 | func AddCalico(multiple bool, cid, vethName, ip string) error { 31 | return nil 32 | } 33 | 34 | func BindCalicoProfile(env []string, cid, profile string) error { 35 | return nil 36 | } 37 | 38 | func AddPrerouting(eip, dest, ident string) error { 39 | return nil 40 | } 41 | 42 | func DelPrerouting(eip, dest, ident string) error { 43 | return nil 44 | } 45 | 46 | func SetBroadcast(vethName, ip string) error { 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | "syscall" 7 | 8 | "github.com/projecteru/eru-agent/api" 9 | "github.com/projecteru/eru-agent/app" 10 | "github.com/projecteru/eru-agent/g" 11 | "github.com/projecteru/eru-agent/health" 12 | "github.com/projecteru/eru-agent/lenz" 13 | "github.com/projecteru/eru-agent/logs" 14 | "github.com/projecteru/eru-agent/network" 15 | "github.com/projecteru/eru-agent/status" 16 | "github.com/projecteru/eru-agent/utils" 17 | ) 18 | 19 | func main() { 20 | g.LoadConfig() 21 | g.InitialConn() 22 | g.InitTransfers() 23 | defer g.CloseConn() 24 | 25 | lenz.InitLenz() 26 | status.InitStatus() 27 | network.InitVlan() 28 | defer lenz.CloseLenz() 29 | 30 | utils.WritePid(g.Config.PidFile) 31 | defer os.Remove(g.Config.PidFile) 32 | 33 | app.Limit() 34 | app.Metric() 35 | api.Serve() 36 | status.Start() 37 | health.Check() 38 | 39 | var c = make(chan os.Signal, 1) 40 | signal.Notify(c, os.Interrupt) 41 | signal.Notify(c, syscall.SIGTERM) 42 | signal.Notify(c, syscall.SIGHUP) 43 | signal.Notify(c, syscall.SIGKILL) 44 | signal.Notify(c, syscall.SIGQUIT) 45 | logs.Info("Eru Agent Catch", <-c) 46 | } 47 | -------------------------------------------------------------------------------- /g/conn.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/projecteru/eru-agent/defines" 7 | "github.com/projecteru/eru-agent/logs" 8 | "github.com/keimoon/gore" 9 | ) 10 | 11 | var Docker defines.ContainerManager 12 | var Rds *gore.Pool 13 | 14 | func InitialConn() { 15 | var err error 16 | Rds = &gore.Pool{ 17 | InitialConn: Config.Redis.Min, 18 | MaximumConn: Config.Redis.Max, 19 | } 20 | 21 | redisHost := fmt.Sprintf("%s:%d", Config.Redis.Host, Config.Redis.Port) 22 | if err := Rds.Dial(redisHost); err != nil { 23 | logs.Assert(err, "Redis init failed") 24 | } 25 | 26 | if Docker, err = defines.NewDocker( 27 | Config.Docker.Endpoint, 28 | Config.Docker.Cert, 29 | Config.Docker.Key, 30 | Config.Docker.Ca, 31 | ); err != nil { 32 | logs.Assert(err, "Docker") 33 | } 34 | logs.Info("Global connections initiated") 35 | } 36 | 37 | func CloseConn() { 38 | Rds.Close() 39 | logs.Info("Global connections closed") 40 | } 41 | 42 | func GetRedisConn() *gore.Conn { 43 | conn, err := Rds.Acquire() 44 | if err != nil || conn == nil { 45 | logs.Assert(err, "Get redis conn") 46 | } 47 | return conn 48 | } 49 | 50 | func ReleaseRedisConn(conn *gore.Conn) { 51 | Rds.Release(conn) 52 | } 53 | -------------------------------------------------------------------------------- /g/config.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | "os" 7 | 8 | "github.com/projecteru/eru-agent/common" 9 | "github.com/projecteru/eru-agent/defines" 10 | "github.com/projecteru/eru-agent/logs" 11 | "gopkg.in/yaml.v2" 12 | ) 13 | 14 | var Config = defines.AgentConfig{} 15 | 16 | func LoadConfig() { 17 | var configPath string 18 | var version bool 19 | flag.BoolVar(&logs.Mode, "DEBUG", false, "enable debug") 20 | flag.StringVar(&configPath, "c", "agent.yaml", "config file") 21 | flag.BoolVar(&version, "v", false, "show version") 22 | flag.Parse() 23 | if version { 24 | logs.Info("Version", common.VERSION) 25 | os.Exit(0) 26 | } 27 | load(configPath) 28 | } 29 | 30 | func load(configPath string) { 31 | if _, err := os.Stat(configPath); err != nil { 32 | logs.Assert(err, "config file invaild") 33 | } 34 | 35 | b, err := ioutil.ReadFile(configPath) 36 | if err != nil { 37 | logs.Assert(err, "Read config file failed") 38 | } 39 | 40 | if err := yaml.Unmarshal(b, &Config); err != nil { 41 | logs.Assert(err, "Load config file failed") 42 | } 43 | 44 | if Config.HostName, err = os.Hostname(); err != nil { 45 | logs.Assert(err, "Load hostname failed") 46 | } 47 | logs.Debug("Configure:", Config) 48 | } 49 | -------------------------------------------------------------------------------- /app/metrics.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/projecteru/eru-agent/common" 7 | "github.com/projecteru/eru-agent/g" 8 | "github.com/projecteru/eru-agent/logs" 9 | "github.com/projecteru/eru-metric/metric" 10 | ) 11 | 12 | func Metric() { 13 | metric.SetGlobalSetting( 14 | g.Docker, time.Duration(common.STATS_TIMEOUT), 15 | time.Duration(common.STATS_FORCE_DONE), 16 | common.VLAN_PREFIX, common.DEFAULT_BR, 17 | ) 18 | logs.Info("Metrics initiated") 19 | } 20 | 21 | func (self *EruApp) Report() { 22 | t := time.NewTicker(self.Step) 23 | defer t.Stop() 24 | defer self.Client.Close() 25 | defer logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report stop") 26 | logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report start") 27 | for { 28 | select { 29 | case now := <-t.C: 30 | go func() { 31 | if info, err := self.UpdateStats(self.ID); err == nil { 32 | if isLimit { 33 | limitChan <- SoftLimit{self.ID, info} 34 | } 35 | rate := self.CalcRate(info, now) 36 | self.SaveLast(info) 37 | go self.Send(rate) 38 | } else { 39 | logs.Info("Update mertic failed", self.ID[:12]) 40 | } 41 | }() 42 | case <-self.Stop: 43 | return 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /defines/lenz.go: -------------------------------------------------------------------------------- 1 | package defines 2 | 3 | import "github.com/projecteru/eru-agent/utils" 4 | 5 | type AttachEvent struct { 6 | Type string 7 | App *Meta 8 | } 9 | 10 | type Log struct { 11 | ID string `json:"id"` 12 | Name string `json:"name"` 13 | Type string `json:"type"` 14 | EntryPoint string `json:"entrypoint"` 15 | Ident string `json:"ident"` 16 | Data string `json:"data"` 17 | Tag string `json:"tag"` 18 | Count int64 `json:"count"` 19 | Datetime string `json:"datetime"` 20 | } 21 | 22 | type Route struct { 23 | ID string `json:"id"` 24 | Source *Source `json:"source,omitempty"` 25 | Target *Target `json:"target"` 26 | Backends *utils.HashBackends 27 | Closer chan bool 28 | Done chan struct{} 29 | } 30 | 31 | func (s *Route) LoadBackends() { 32 | s.Backends = utils.NewHashBackends(s.Target.Addrs) 33 | } 34 | 35 | type Source struct { 36 | ID string `json:"id,omitempty"` 37 | Name string `json:"name,omitempty"` 38 | Filter string `json:"filter,omitempty"` 39 | Types []string `json:"types,omitempty"` 40 | } 41 | 42 | func (s *Source) All() bool { 43 | return s.ID == "" && s.Name == "" && s.Filter == "" 44 | } 45 | 46 | type Target struct { 47 | Addrs []string `json:"addrs"` 48 | AppendTag string `json:"append_tag,omitempty"` 49 | } 50 | -------------------------------------------------------------------------------- /lenz/lenz.go: -------------------------------------------------------------------------------- 1 | package lenz 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/projecteru/eru-agent/common" 7 | "github.com/projecteru/eru-agent/defines" 8 | "github.com/projecteru/eru-agent/g" 9 | "github.com/projecteru/eru-agent/logs" 10 | ) 11 | 12 | var Attacher *AttachManager 13 | var Router *RouteManager 14 | var Routefs RouteFileStore 15 | 16 | func InitLenz() { 17 | Attacher = NewAttachManager() 18 | Router = NewRouteManager(Attacher) 19 | Routefs = RouteFileStore(g.Config.Lenz.Routes) 20 | if len(g.Config.Lenz.Forwards) > 0 { 21 | logs.Debug("Lenz Routing all to", g.Config.Lenz.Forwards) 22 | target := defines.Target{Addrs: g.Config.Lenz.Forwards} 23 | route := defines.Route{ID: common.LENZ_DEFAULT, Target: &target} 24 | route.LoadBackends() 25 | Router.Add(&route) 26 | } 27 | if _, err := os.Stat(g.Config.Lenz.Routes); err == nil { 28 | logs.Debug("Loading and persisting routes in", g.Config.Lenz.Routes) 29 | logs.Assert(Router.Load(Routefs), "persistor") 30 | } 31 | logs.Info("Lenz initiated") 32 | } 33 | 34 | func CloseLenz() { 35 | logs.Info("Close all lenz streamer") 36 | routes, err := Router.GetAll() 37 | if err != nil { 38 | logs.Info("Get all lenz route failed", err) 39 | return 40 | } 41 | for _, route := range routes { 42 | if !Router.Remove(route.ID) { 43 | logs.Info("Close lenz route failed", route.ID) 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /defines/manager.go: -------------------------------------------------------------------------------- 1 | package defines 2 | 3 | import ( 4 | "github.com/fsouza/go-dockerclient" 5 | ) 6 | 7 | type ContainerManager interface { 8 | PushImage(docker.PushImageOptions, docker.AuthConfiguration) error 9 | PullImage(docker.PullImageOptions, docker.AuthConfiguration) error 10 | CreateContainer(docker.CreateContainerOptions) (*docker.Container, error) 11 | StartContainer(string, *docker.HostConfig) error 12 | BuildImage(docker.BuildImageOptions) error 13 | KillContainer(docker.KillContainerOptions) error 14 | StopContainer(string, uint) error 15 | InspectContainer(string) (*docker.Container, error) 16 | ListContainers(docker.ListContainersOptions) ([]docker.APIContainers, error) 17 | ListImages(docker.ListImagesOptions) ([]docker.APIImages, error) 18 | RemoveContainer(docker.RemoveContainerOptions) error 19 | WaitContainer(string) (int, error) 20 | RemoveImage(string) error 21 | CreateExec(docker.CreateExecOptions) (*docker.Exec, error) 22 | StartExec(string, docker.StartExecOptions) error 23 | Ping() error 24 | Stats(docker.StatsOptions) error 25 | AttachToContainer(opts docker.AttachToContainerOptions) error 26 | AddEventListener(listener chan<- *docker.APIEvents) error 27 | } 28 | 29 | func NewDocker(endpoint, cert, key, ca string) (ContainerManager, error) { 30 | client, err := docker.NewTLSClient(endpoint, cert, key, ca) 31 | if err != nil { 32 | return nil, err 33 | } 34 | return client, nil 35 | } 36 | -------------------------------------------------------------------------------- /utils/utils_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func Test_GetAppInfo(t *testing.T) { 9 | var containerName string 10 | containerName = "test_1234_abc" 11 | appname, entrypoint, ident := GetAppInfo(containerName) 12 | if appname != "test" { 13 | t.Error("Get appname failed") 14 | } 15 | if entrypoint != "1234" { 16 | t.Error("Get entrypoint failed") 17 | } 18 | if ident != "abc" { 19 | t.Error("Get ident failed") 20 | } 21 | containerName = "eru_test_flask_1234_abc" 22 | appname, entrypoint, ident = GetAppInfo(containerName) 23 | if appname != "eru_test_flask" { 24 | t.Error("Get appname failed") 25 | } 26 | if entrypoint != "1234" { 27 | t.Error("Get entrypoint failed") 28 | } 29 | if ident != "abc" { 30 | t.Error("Get ident failed") 31 | } 32 | } 33 | 34 | func Test_UrlJoin(t *testing.T) { 35 | strs := []string{"http://a.b.c", "d", "e"} 36 | ss := UrlJoin(strs...) 37 | if ss != "http://a.b.c/d/e" { 38 | t.Error("Join invaild") 39 | } 40 | } 41 | 42 | func Test_WritePid(t *testing.T) { 43 | p := "/tmp/test.pid" 44 | WritePid(p) 45 | defer os.RemoveAll(p) 46 | if _, err := os.Stat(p); err != nil { 47 | t.Error(err) 48 | } 49 | } 50 | 51 | func Test_CopyDir(t *testing.T) { 52 | defer func() { 53 | os.RemoveAll("/tmp/t1") 54 | os.RemoveAll("/tmp/t2") 55 | }() 56 | if err := MakeDir("/tmp/t1"); err != nil { 57 | t.Error(err) 58 | } 59 | if err := CopyDir("/tmp/t1", "/tmp/t2"); err != nil { 60 | t.Error(err) 61 | } 62 | if _, err := os.Stat("/tmp/t2"); err != nil { 63 | t.Error(err) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /lenz/streamer.go: -------------------------------------------------------------------------------- 1 | package lenz 2 | 3 | import ( 4 | "math" 5 | 6 | "github.com/projecteru/eru-agent/defines" 7 | "github.com/projecteru/eru-agent/g" 8 | "github.com/projecteru/eru-agent/logs" 9 | ) 10 | 11 | func Streamer(route *defines.Route, logstream chan *defines.Log) { 12 | var upstreams map[string]*UpStream = map[string]*UpStream{} 13 | var types map[string]struct{} 14 | var count int64 = 0 15 | if route.Source != nil { 16 | types = make(map[string]struct{}) 17 | for _, t := range route.Source.Types { 18 | types[t] = struct{}{} 19 | } 20 | } 21 | defer func() { 22 | logs.Debug("Flush", route.ID, "cache logs") 23 | for _, remote := range upstreams { 24 | remote.Flush() 25 | for _, log := range remote.Tail() { 26 | logs.Info("Streamer can't send to remote", log) 27 | } 28 | remote.Close() 29 | } 30 | route.Done <- struct{}{} 31 | }() 32 | for logline := range logstream { 33 | if types != nil { 34 | if _, ok := types[logline.Type]; !ok { 35 | continue 36 | } 37 | } 38 | logline.Tag = route.Target.AppendTag 39 | logline.Count = count 40 | if g.Config.Lenz.Stdout { 41 | logs.Info("Debug Output", logline) 42 | continue 43 | } 44 | var f bool = false 45 | for offset := 0; offset < route.Backends.Len(); offset++ { 46 | addr := route.Backends.Get(logline.Name, offset) 47 | if _, ok := upstreams[addr]; !ok { 48 | if ups, err := NewUpStream(addr); err != nil || ups == nil { 49 | continue 50 | } else { 51 | upstreams[addr] = ups 52 | } 53 | } 54 | f = true 55 | if err := upstreams[addr].WriteData(logline); err != nil { 56 | logs.Info("Sent to remote failed", err) 57 | upstreams[addr].Close() 58 | go func(upstream *UpStream) { 59 | for _, log := range upstream.Tail() { 60 | logstream <- log 61 | } 62 | }(upstreams[addr]) 63 | delete(upstreams, addr) 64 | continue 65 | } 66 | //logs.Debug("Lenz Send", logline.Name, logline.EntryPoint, logline.ID, "to", addr) 67 | break 68 | } 69 | if !f { 70 | logs.Info("Lenz failed", logline.ID[:12], logline.Name, logline.EntryPoint, logline.Data) 71 | } 72 | if count == math.MaxInt64 { 73 | count = 0 74 | } else { 75 | count++ 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /app/eru.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "sync" 7 | "time" 8 | 9 | "github.com/fsouza/go-dockerclient" 10 | "github.com/projecteru/eru-agent/defines" 11 | "github.com/projecteru/eru-agent/g" 12 | "github.com/projecteru/eru-agent/logs" 13 | "github.com/projecteru/eru-agent/utils" 14 | "github.com/projecteru/eru-metric/metric" 15 | "github.com/projecteru/eru-metric/statsd" 16 | ) 17 | 18 | type EruApp struct { 19 | defines.Meta 20 | metric.Metric 21 | } 22 | 23 | func NewEruApp(container *docker.Container, extend map[string]interface{}) *EruApp { 24 | name, entrypoint, ident := utils.GetAppInfo(container.Name) 25 | if name == "" { 26 | logs.Info("Container name invaild", container.Name) 27 | return nil 28 | } 29 | logs.Debug("Eru App", name, entrypoint, ident) 30 | 31 | transfer := g.Transfers.Get(container.ID, 0) 32 | client := statsd.CreateStatsDClient(transfer) 33 | 34 | step := time.Duration(g.Config.Metrics.Step) * time.Second 35 | //TODO remove version meta data 36 | version := extend["__version__"] 37 | delete(extend, "__version__") 38 | var tagString string 39 | if len(extend) > 0 { 40 | tag := []string{} 41 | for _, v := range extend { 42 | tag = append(tag, fmt.Sprintf("%v", v)) 43 | } 44 | tagString = fmt.Sprintf("%s.%s.%s", g.Config.HostName, strings.Join(tag, "."), container.ID[:12]) 45 | } else { 46 | tagString = fmt.Sprintf("%s.%s", g.Config.HostName, container.ID[:12]) 47 | } 48 | endpoint := fmt.Sprintf("%s.%s.%s", name, version, entrypoint) 49 | 50 | meta := defines.Meta{container.ID, container.State.Pid, name, entrypoint, ident, extend} 51 | metric := metric.CreateMetric(step, client, tagString, endpoint) 52 | eruApp := &EruApp{meta, metric} 53 | return eruApp 54 | } 55 | 56 | var lock sync.RWMutex 57 | var Apps map[string]*EruApp = map[string]*EruApp{} 58 | 59 | func Add(app *EruApp) { 60 | lock.Lock() 61 | defer lock.Unlock() 62 | if _, ok := Apps[app.ID]; ok { 63 | // safe add 64 | return 65 | } 66 | if err := app.InitMetric(app.ID, app.Pid); err != nil { 67 | logs.Info("Init app metric failed", err) 68 | return 69 | } 70 | go app.Report() 71 | Apps[app.ID] = app 72 | } 73 | 74 | func Remove(ID string) { 75 | lock.Lock() 76 | defer lock.Unlock() 77 | if _, ok := Apps[ID]; !ok { 78 | return 79 | } 80 | Apps[ID].Exit() 81 | delete(Apps, ID) 82 | } 83 | 84 | func Valid(ID string) bool { 85 | lock.RLock() 86 | defer lock.RUnlock() 87 | _, ok := Apps[ID] 88 | return ok 89 | } 90 | -------------------------------------------------------------------------------- /app/limit.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/projecteru/eru-agent/common" 7 | "github.com/projecteru/eru-agent/g" 8 | "github.com/projecteru/eru-agent/logs" 9 | "github.com/keimoon/gore" 10 | ) 11 | 12 | type SoftLimit struct { 13 | cid string 14 | info map[string]uint64 15 | } 16 | 17 | var limitChan chan SoftLimit = make(chan SoftLimit) 18 | var usage map[string]uint64 = make(map[string]uint64) 19 | var isLimit bool = false 20 | 21 | func Limit() { 22 | if g.Config.Limit.Memory != 0 { 23 | logs.Info("App memory soft limit start") 24 | isLimit = true 25 | go calcMemoryUsage() 26 | } 27 | } 28 | 29 | func calcMemoryUsage() { 30 | for { 31 | select { 32 | case d := <-limitChan: 33 | if v, ok := d.info["mem_usage"]; ok { 34 | usage[d.cid] = v 35 | } else { 36 | usage[d.cid] = 0 37 | } 38 | var doCalc bool = true 39 | for id, _ := range Apps { 40 | if _, ok := usage[id]; !ok { 41 | doCalc = false 42 | break 43 | } 44 | } 45 | if doCalc { 46 | judgeMemoryUsage() 47 | } 48 | } 49 | } 50 | } 51 | 52 | func judgeMemoryUsage() { 53 | var totalUsage uint64 = 0 54 | var rate map[string]float64 = make(map[string]float64) 55 | for cid, usage := range usage { 56 | totalUsage = totalUsage + usage 57 | //TODO ugly 58 | if v, ok := Apps[cid].Extend["__memory__"]; !ok { 59 | rate[cid] = 0.0 60 | continue 61 | } else { 62 | define, _ := v.(float64) 63 | rate[cid] = float64(usage) / define 64 | } 65 | } 66 | logs.Debug("Current memory usage", totalUsage, "max", g.Config.Limit.Memory) 67 | for { 68 | if totalUsage < g.Config.Limit.Memory { 69 | return 70 | } 71 | var exceedRate float64 = 0.0 72 | var cid string = "" 73 | for k, v := range rate { 74 | if exceedRate >= v { 75 | continue 76 | } 77 | exceedRate = v 78 | cid = k 79 | } 80 | if cid == "" { 81 | logs.Info("MemLimit can not stop containers") 82 | break 83 | } 84 | softOOMKill(cid, exceedRate) 85 | totalUsage -= usage[cid] 86 | delete(rate, cid) 87 | } 88 | for k, _ := range usage { 89 | delete(usage, k) 90 | } 91 | } 92 | 93 | func softOOMKill(cid string, rate float64) { 94 | logs.Debug("OOM killed", cid[:12]) 95 | conn := g.GetRedisConn() 96 | defer g.ReleaseRedisConn(conn) 97 | 98 | key := fmt.Sprintf("eru:agent:%s:container:reason", cid) 99 | if _, err := gore.NewCommand("SET", key, common.OOM_KILLED).Run(conn); err != nil { 100 | logs.Info("OOM killed set flag", err) 101 | } 102 | if err := g.Docker.StopContainer(cid, 10); err != nil { 103 | logs.Info("OOM killed failed", cid[:12]) 104 | return 105 | } 106 | logs.Info("OOM killed success", cid[:12]) 107 | } 108 | -------------------------------------------------------------------------------- /lenz/conn.go: -------------------------------------------------------------------------------- 1 | package lenz 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "log/syslog" 9 | "net" 10 | "net/url" 11 | 12 | "github.com/projecteru/eru-agent/defines" 13 | "github.com/projecteru/eru-agent/g" 14 | "github.com/projecteru/eru-agent/logs" 15 | ) 16 | 17 | type UpStream struct { 18 | addr string 19 | scheme string 20 | conn io.Writer 21 | encoder *json.Encoder 22 | buffer []*defines.Log 23 | count int 24 | write func(logline *defines.Log) error 25 | Close func() error 26 | } 27 | 28 | func NewUpStream(addr string) (up *UpStream, err error) { 29 | u, err := url.Parse(addr) 30 | if err != nil { 31 | logs.Info("Parse upstream addr failed", err) 32 | return nil, err 33 | } 34 | up = &UpStream{addr: u.Host} 35 | up.buffer = []*defines.Log{} 36 | up.count = 0 37 | if g.Config.Lenz.Count > 0 { 38 | up.write = func(logline *defines.Log) error { 39 | up.buffer = append(up.buffer, logline) 40 | up.count += 1 41 | if up.count < g.Config.Lenz.Count { 42 | return nil 43 | } 44 | //logs.Debug("Streamer buffer full, send to remote") 45 | return up.Flush() 46 | } 47 | } else { 48 | up.write = func(logline *defines.Log) error { 49 | return up.encoder.Encode(logline) 50 | } 51 | } 52 | switch { 53 | case u.Scheme == "udp": 54 | err = up.createUDPConn() 55 | return up, err 56 | case u.Scheme == "tcp": 57 | err = up.createTCPConn() 58 | return up, err 59 | case u.Scheme == "syslog": 60 | err = up.createSyslog() 61 | return up, err 62 | } 63 | return nil, nil 64 | } 65 | 66 | func (self *UpStream) createUDPConn() error { 67 | self.scheme = "udp" 68 | udpAddr, err := net.ResolveUDPAddr("udp", self.addr) 69 | if err != nil { 70 | logs.Info("Resolve", self.addr, "failed", err) 71 | return err 72 | } 73 | conn, err := net.DialUDP("udp", nil, udpAddr) 74 | if err != nil { 75 | logs.Info("Connect backend failed", err) 76 | return err 77 | } 78 | self.conn = conn 79 | self.encoder = json.NewEncoder(conn) 80 | self.Close = conn.Close 81 | return nil 82 | } 83 | 84 | func (self *UpStream) createTCPConn() error { 85 | self.scheme = "tcp" 86 | tcpAddr, err := net.ResolveTCPAddr("tcp", self.addr) 87 | if err != nil { 88 | logs.Info("Resolve", self.addr, "failed", err) 89 | return err 90 | } 91 | conn, err := net.DialTCP("tcp", nil, tcpAddr) 92 | if err != nil { 93 | logs.Debug("Connect backend failed", err) 94 | return err 95 | } 96 | self.conn = conn 97 | self.encoder = json.NewEncoder(conn) 98 | self.Close = conn.Close 99 | return nil 100 | } 101 | 102 | func (self *UpStream) createSyslog() error { 103 | self.scheme = "syslog" 104 | self.Close = func() error { return nil } 105 | return nil 106 | } 107 | 108 | func (self *UpStream) Tail() []*defines.Log { 109 | return self.buffer 110 | } 111 | 112 | func (self *UpStream) WriteData(logline *defines.Log) error { 113 | switch self.scheme { 114 | case "tcp": 115 | return self.write(logline) 116 | case "udp": 117 | return self.write(logline) 118 | case "syslog": 119 | tag := fmt.Sprintf("%s.%s", logline.Name, logline.Tag) 120 | remote, err := syslog.Dial("udp", self.addr, syslog.LOG_USER|syslog.LOG_INFO, tag) 121 | if err != nil { 122 | return err 123 | } 124 | _, err = io.WriteString(remote, logline.Data) 125 | return err 126 | default: 127 | return errors.New("Not support type") 128 | } 129 | } 130 | 131 | func (self *UpStream) Flush() error { 132 | for i, log := range self.buffer { 133 | if err := self.encoder.Encode(log); err != nil { 134 | self.buffer = self.buffer[i:] 135 | return err 136 | } 137 | } 138 | self.buffer = []*defines.Log{} 139 | self.count = 0 140 | return nil 141 | } 142 | -------------------------------------------------------------------------------- /network/route_linux.go: -------------------------------------------------------------------------------- 1 | package network 2 | 3 | import ( 4 | "net" 5 | "runtime" 6 | 7 | "github.com/projecteru/eru-agent/g" 8 | "github.com/projecteru/eru-agent/logs" 9 | "github.com/vishvananda/netlink" 10 | "github.com/vishvananda/netns" 11 | ) 12 | 13 | //add default route 14 | func addDefaultRoute(gateway string) error { 15 | gwIP := net.ParseIP(gateway) 16 | route := netlink.Route{Gw: gwIP} 17 | 18 | return netlink.RouteAdd(&route) 19 | } 20 | 21 | //delete default routes 22 | //FIXME all default routes will be erased 23 | func delDefaultRoute() error { 24 | routes, _ := netlink.RouteList(nil, netlink.FAMILY_V4) 25 | 26 | for _, route := range routes { 27 | if route.Dst != nil || route.Src != nil { 28 | continue 29 | } 30 | if err := netlink.RouteDel(&route); err != nil { 31 | return err 32 | } 33 | } 34 | return nil 35 | } 36 | 37 | func setDefaultRoute(cid, gateway string, pid int) bool { 38 | runtime.LockOSThread() 39 | defer runtime.UnlockOSThread() 40 | 41 | origins, err := netns.Get() 42 | if err != nil { 43 | logs.Info("Get orignal namespace failed", err) 44 | return false 45 | } 46 | defer origins.Close() 47 | 48 | ns, err := netns.GetFromPid(pid) 49 | if err != nil { 50 | logs.Info("Get container namespace failed", err) 51 | return false 52 | } 53 | 54 | netns.Set(ns) 55 | defer ns.Close() 56 | defer netns.Set(origins) 57 | 58 | if err := delDefaultRoute(); err != nil { 59 | logs.Info("Delete default routing table failed", err) 60 | return false 61 | } 62 | 63 | if err := addDefaultRoute(gateway); err != nil { 64 | logs.Info("Add default route failed", err) 65 | return false 66 | } 67 | 68 | logs.Info("Set default route success", cid[:12], gateway) 69 | return true 70 | } 71 | 72 | func addRouteByLink(CIDR, ifc string) error { 73 | link, err := netlink.LinkByName(ifc) 74 | if err != nil { 75 | return err 76 | } 77 | _, dst, err := net.ParseCIDR(CIDR) 78 | if err != nil { 79 | return err 80 | } 81 | route := netlink.Route{LinkIndex: link.Attrs().Index, Dst: dst} 82 | return netlink.RouteAdd(&route) 83 | } 84 | 85 | func addRoute(cid, CIDR, ifc string, pid int) bool { 86 | runtime.LockOSThread() 87 | defer runtime.UnlockOSThread() 88 | 89 | origins, err := netns.Get() 90 | if err != nil { 91 | logs.Info("Get orignal namespace failed", err) 92 | return false 93 | } 94 | defer origins.Close() 95 | 96 | ns, err := netns.GetFromPid(pid) 97 | if err != nil { 98 | logs.Info("Get container namespace failed", err) 99 | return false 100 | } 101 | 102 | netns.Set(ns) 103 | defer ns.Close() 104 | defer netns.Set(origins) 105 | 106 | if err := addRouteByLink(CIDR, ifc); err != nil { 107 | logs.Info("Add route failed", err) 108 | return false 109 | } 110 | 111 | logs.Info("Add route success", cid[:12], CIDR, ifc) 112 | return true 113 | } 114 | 115 | func AddRoute(cid, CIDR, ifc string) bool { 116 | lock.Lock() 117 | defer lock.Unlock() 118 | 119 | logs.Info("Add", cid[:12], "route", CIDR, ifc) 120 | 121 | container, err := g.Docker.InspectContainer(cid) 122 | if err != nil { 123 | logs.Info("RouteSetter inspect docker failed", err) 124 | return false 125 | } 126 | 127 | pid := container.State.Pid 128 | 129 | return addRoute(cid, CIDR, ifc, pid) 130 | } 131 | 132 | func SetDefaultRoute(cid, gateway string) bool { 133 | lock.Lock() 134 | defer lock.Unlock() 135 | 136 | logs.Info("Set", cid[:12], "default route", gateway) 137 | 138 | container, err := g.Docker.InspectContainer(cid) 139 | if err != nil { 140 | logs.Info("RouteSetter inspect docker failed", err) 141 | return false 142 | } 143 | 144 | pid := container.State.Pid 145 | 146 | return setDefaultRoute(cid, gateway, pid) 147 | } 148 | -------------------------------------------------------------------------------- /network/vlan_linux.go: -------------------------------------------------------------------------------- 1 | package network 2 | 3 | import ( 4 | "os/exec" 5 | "runtime" 6 | 7 | "github.com/projecteru/eru-agent/g" 8 | "github.com/projecteru/eru-agent/logs" 9 | "github.com/vishvananda/netlink" 10 | "github.com/vishvananda/netns" 11 | ) 12 | 13 | func setUpVLan(cid, ips string, pid int, veth netlink.Link) bool { 14 | runtime.LockOSThread() 15 | defer runtime.UnlockOSThread() 16 | 17 | origns, err := netns.Get() 18 | if err != nil { 19 | logs.Info("Get orignal namespace failed", err) 20 | return false 21 | } 22 | defer origns.Close() 23 | 24 | ns, err := netns.GetFromPid(pid) 25 | if err != nil { 26 | logs.Info("Get container namespace failed", err) 27 | return false 28 | } 29 | 30 | netns.Set(ns) 31 | defer ns.Close() 32 | defer netns.Set(origns) 33 | 34 | if err := BindAndSetup(veth, ips); err != nil { 35 | logs.Info("Bind and setup NIC failed", err) 36 | DelVlan(veth) 37 | return false 38 | } 39 | 40 | logs.Info("Add vlan device success", cid[:12]) 41 | return true 42 | } 43 | 44 | func AddVlan(vethName, ips, cid string) bool { 45 | lock.Lock() 46 | defer lock.Unlock() 47 | 48 | container, err := g.Docker.InspectContainer(cid) 49 | if err != nil { 50 | logs.Info("VLanSetter inspect docker failed", err) 51 | return false 52 | } 53 | 54 | veth, err := AddMacVlanDevice(vethName, cid) 55 | if err != nil { 56 | logs.Info("Create macvlan device failed", err) 57 | return false 58 | } 59 | 60 | if err := netlink.LinkSetNsPid(veth, container.State.Pid); err != nil { 61 | logs.Info("Set macvlan device into container failed", err) 62 | DelVlan(veth) 63 | return false 64 | } 65 | 66 | return setUpVLan(cid, ips, container.State.Pid, veth) 67 | } 68 | 69 | func DelMacVlanDevice(vethName string) error { 70 | logs.Info("Release macvlan device", vethName) 71 | link, err := netlink.LinkByName(vethName) 72 | if err != nil { 73 | return err 74 | } 75 | DelVlan(link) 76 | return nil 77 | } 78 | 79 | func AddMacVlanDevice(vethName, seq string) (netlink.Link, error) { 80 | device := Devices.Get(seq, 0) 81 | logs.Info("Add new macvlan device", vethName, device) 82 | 83 | parent, err := netlink.LinkByName(device) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | veth := &netlink.Macvlan{ 89 | LinkAttrs: netlink.LinkAttrs{Name: vethName, ParentIndex: parent.Attrs().Index}, 90 | Mode: netlink.MACVLAN_MODE_BRIDGE, 91 | } 92 | 93 | if err := netlink.LinkAdd(veth); err != nil { 94 | return nil, err 95 | } 96 | return veth, nil 97 | } 98 | 99 | func BindAndSetup(veth netlink.Link, ips string) error { 100 | addr, err := netlink.ParseAddr(ips) 101 | if err != nil { 102 | return err 103 | } 104 | 105 | if err := netlink.AddrAdd(veth, addr); err != nil { 106 | return err 107 | } 108 | 109 | if err := netlink.LinkSetUp(veth); err != nil { 110 | return err 111 | } 112 | return nil 113 | } 114 | 115 | func SetBroadcast(vethName, ip string) error { 116 | cmd := exec.Command("ifconfig", vethName, "broadcast", ip) 117 | return cmd.Run() 118 | } 119 | 120 | func AddCalico(env []string, multiple bool, cid, vethName, ip string) error { 121 | if !multiple { 122 | add := exec.Command("calicoctl", "container", "add", cid, ip, "--interface", vethName) 123 | add.Env = env 124 | return add.Run() 125 | } 126 | add := exec.Command("calicoctl", "container", cid, "ip", "add", ip, "--interface", vethName) 127 | add.Env = env 128 | return add.Run() 129 | } 130 | 131 | func BindCalicoProfile(env []string, cid, profileName string) error { 132 | profile := exec.Command("calicoctl", "container", cid, "profile", "append", profileName) 133 | profile.Env = env 134 | return profile.Run() 135 | } 136 | 137 | func AddPrerouting(eip, dest, ident string) error { 138 | cmd := exec.Command("iptables", "-t", "nat", "-A", "PREROUTING", "-d", eip, "-j", "DNAT", "--to-destination", dest, "-m", "comment", "--comment", ident) 139 | return cmd.Run() 140 | } 141 | 142 | func DelPrerouting(eip, dest, ident string) error { 143 | cmd := exec.Command("iptables", "-t", "nat", "-D", "PREROUTING", "-d", eip, "-j", "DNAT", "--to-destination", dest, "-m", "comment", "--comment", ident) 144 | return cmd.Run() 145 | } 146 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "hash/fnv" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "os" 11 | "strconv" 12 | "strings" 13 | 14 | "github.com/projecteru/eru-agent/common" 15 | "github.com/projecteru/eru-agent/logs" 16 | ) 17 | 18 | func NewHashBackends(data []string) *HashBackends { 19 | return &HashBackends{data, uint32(len(data))} 20 | } 21 | 22 | type HashBackends struct { 23 | data []string 24 | length uint32 25 | } 26 | 27 | func (self *HashBackends) Get(v string, offset int) string { 28 | h := fnv.New32a() 29 | h.Write([]byte(v)) 30 | return self.data[(h.Sum32()+uint32(offset))%self.length] 31 | } 32 | 33 | func (self *HashBackends) Len() int { 34 | return len(self.data) 35 | } 36 | 37 | var httpClient *http.Client 38 | 39 | func init() { 40 | httpClient = &http.Client{ 41 | Transport: &http.Transport{ 42 | DisableKeepAlives: true, 43 | MaxIdleConnsPerHost: 1, 44 | }, 45 | } 46 | } 47 | 48 | func UrlJoin(strs ...string) string { 49 | ss := make([]string, len(strs)) 50 | for i, s := range strs { 51 | if i == 0 { 52 | ss[i] = strings.TrimRight(s, "/") 53 | } else { 54 | ss[i] = strings.TrimLeft(s, "/") 55 | } 56 | } 57 | return strings.Join(ss, "/") 58 | } 59 | 60 | func WritePid(path string) { 61 | if err := ioutil.WriteFile(path, []byte(strconv.Itoa(os.Getpid())), 0755); err != nil { 62 | logs.Assert(err, "Save pid file failed") 63 | } 64 | } 65 | 66 | func MakeDir(p string) error { 67 | if err := os.MkdirAll(p, 0755); err != nil { 68 | return err 69 | } 70 | return nil 71 | } 72 | 73 | func CopyDir(source string, dest string) (err error) { 74 | // get properties of source dir 75 | sourceinfo, err := os.Stat(source) 76 | if err != nil { 77 | return err 78 | } 79 | 80 | // create dest dir 81 | err = os.MkdirAll(dest, sourceinfo.Mode()) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | directory, _ := os.Open(source) 87 | objects, err := directory.Readdir(-1) 88 | 89 | for _, obj := range objects { 90 | sourcefilepointer := source + "/" + obj.Name() 91 | destinationfilepointer := dest + "/" + obj.Name() 92 | 93 | if obj.IsDir() { 94 | // create sub-directories - recursively 95 | err = CopyDir(sourcefilepointer, destinationfilepointer) 96 | if err != nil { 97 | fmt.Println(err) 98 | } 99 | } else { 100 | // perform copy 101 | err = CopyFile(sourcefilepointer, destinationfilepointer) 102 | if err != nil { 103 | fmt.Println(err) 104 | } 105 | } 106 | } 107 | return 108 | } 109 | 110 | func CopyFile(source string, dest string) (err error) { 111 | sourcefile, err := os.Open(source) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | defer sourcefile.Close() 117 | 118 | destfile, err := os.Create(dest) 119 | if err != nil { 120 | return err 121 | } 122 | 123 | defer destfile.Close() 124 | 125 | _, err = io.Copy(destfile, sourcefile) 126 | if err == nil { 127 | sourceinfo, err := os.Stat(source) 128 | if err != nil { 129 | err = os.Chmod(dest, sourceinfo.Mode()) 130 | } 131 | } 132 | return 133 | } 134 | 135 | func Marshal(obj interface{}) []byte { 136 | bytes, err := json.MarshalIndent(obj, "", " ") 137 | if err != nil { 138 | logs.Info("Utils Marshal:", err) 139 | } 140 | return bytes 141 | } 142 | 143 | func Unmarshal(input io.ReadCloser, obj interface{}) error { 144 | body, err := ioutil.ReadAll(input) 145 | if err != nil { 146 | return err 147 | } 148 | err = json.Unmarshal(body, obj) 149 | if err != nil { 150 | return err 151 | } 152 | return nil 153 | } 154 | 155 | func GetAppInfo(containerName string) (name string, entrypoint string, ident string) { 156 | containerName = strings.TrimLeft(containerName, "/") 157 | appinfo := strings.Split(containerName, "_") 158 | if len(appinfo) < common.CNAME_NUM { 159 | return "", "", "" 160 | } 161 | l := len(appinfo) 162 | return strings.Join(appinfo[:l-2], "_"), appinfo[l-2], appinfo[l-1] 163 | } 164 | 165 | func Atoi(s string, def int) int { 166 | if r, err := strconv.Atoi(s); err != nil { 167 | return def 168 | } else { 169 | return r 170 | } 171 | } 172 | 173 | func DoPut(url string) { 174 | req, err := http.NewRequest("PUT", url, nil) 175 | if err != nil { 176 | logs.Debug("Gen request failed", err) 177 | return 178 | } 179 | response, err := httpClient.Do(req) 180 | if err != nil { 181 | logs.Debug("Do request failed", err) 182 | return 183 | } 184 | defer response.Body.Close() 185 | data, err := ioutil.ReadAll(response.Body) 186 | if err != nil { 187 | logs.Debug("Read response failed", err) 188 | return 189 | } 190 | logs.Debug("Response:", string(data)) 191 | } 192 | -------------------------------------------------------------------------------- /lenz/routes.go: -------------------------------------------------------------------------------- 1 | package lenz 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "strings" 7 | "sync" 8 | 9 | "github.com/projecteru/eru-agent/common" 10 | "github.com/projecteru/eru-agent/defines" 11 | "github.com/projecteru/eru-agent/logs" 12 | "github.com/projecteru/eru-agent/utils" 13 | ) 14 | 15 | type RouteStore interface { 16 | Get(id string) (*defines.Route, error) 17 | GetAll() ([]*defines.Route, error) 18 | Add(route *defines.Route) error 19 | Remove(id string) bool 20 | } 21 | 22 | type RouteManager struct { 23 | sync.Mutex 24 | persistor RouteStore 25 | attacher *AttachManager 26 | routes map[string]*defines.Route 27 | } 28 | 29 | func NewRouteManager(attacher *AttachManager) *RouteManager { 30 | return &RouteManager{attacher: attacher, routes: make(map[string]*defines.Route)} 31 | } 32 | 33 | func (rm *RouteManager) Reload() error { 34 | newRoutes, err := rm.persistor.GetAll() 35 | if err != nil { 36 | return err 37 | } 38 | 39 | newRoutesMap := make(map[string]struct{}) 40 | for _, newRoute := range newRoutes { 41 | newRoutesMap[newRoute.ID] = struct{}{} 42 | if route, ok := rm.routes[newRoute.ID]; ok { 43 | route.Source = newRoute.Source 44 | route.Target = newRoute.Target 45 | route.Backends = newRoute.Backends 46 | continue 47 | } 48 | rm.Add(newRoute) 49 | } 50 | 51 | for key, _ := range rm.routes { 52 | if _, ok := newRoutesMap[key]; ok || key == common.LENZ_DEFAULT { 53 | continue 54 | } 55 | rm.Remove(key) 56 | } 57 | return nil 58 | } 59 | 60 | func (rm *RouteManager) Load(persistor RouteStore) error { 61 | routes, err := persistor.GetAll() 62 | if err != nil { 63 | return err 64 | } 65 | for _, route := range routes { 66 | rm.Add(route) 67 | } 68 | rm.persistor = persistor 69 | return nil 70 | } 71 | 72 | func (rm *RouteManager) Get(id string) (*defines.Route, error) { 73 | rm.Lock() 74 | defer rm.Unlock() 75 | route, ok := rm.routes[id] 76 | if !ok { 77 | return nil, os.ErrNotExist 78 | } 79 | return route, nil 80 | } 81 | 82 | func (rm *RouteManager) GetAll() ([]*defines.Route, error) { 83 | rm.Lock() 84 | defer rm.Unlock() 85 | routes := make([]*defines.Route, 0) 86 | for _, route := range rm.routes { 87 | routes = append(routes, route) 88 | } 89 | return routes, nil 90 | } 91 | 92 | func (rm *RouteManager) Add(route *defines.Route) error { 93 | rm.Lock() 94 | defer rm.Unlock() 95 | route.Closer = make(chan bool) 96 | route.Done = make(chan struct{}) 97 | rm.routes[route.ID] = route 98 | go func() { 99 | logstream := make(chan *defines.Log) 100 | go Streamer(route, logstream) 101 | rm.attacher.Listen(route.Source, logstream, route.Closer) 102 | close(logstream) 103 | }() 104 | if rm.persistor != nil { 105 | if err := rm.persistor.Add(route); err != nil { 106 | logs.Info("Lenz Persistor:", err) 107 | } 108 | } 109 | return nil 110 | } 111 | 112 | func (rm *RouteManager) Remove(id string) bool { 113 | rm.Lock() 114 | defer rm.Unlock() 115 | route, ok := rm.routes[id] 116 | if ok && route.Closer != nil { 117 | route.Closer <- true 118 | <-route.Done 119 | } 120 | delete(rm.routes, id) 121 | if rm.persistor != nil { 122 | rm.persistor.Remove(id) 123 | } 124 | return ok 125 | } 126 | 127 | type RouteFileStore string 128 | 129 | func (fs RouteFileStore) Filename(id string) string { 130 | return string(fs) + "/" + id + ".json" 131 | } 132 | 133 | func (fs RouteFileStore) Get(id string) (*defines.Route, error) { 134 | file, err := os.Open(fs.Filename(id)) 135 | if err != nil { 136 | return nil, err 137 | } 138 | route := new(defines.Route) 139 | if err = utils.Unmarshal(file, route); err != nil { 140 | return nil, err 141 | } 142 | if route.ID == "" { 143 | route.ID = id 144 | } 145 | return route, nil 146 | } 147 | 148 | func (fs RouteFileStore) GetAll() ([]*defines.Route, error) { 149 | files, err := ioutil.ReadDir(string(fs)) 150 | if err != nil { 151 | return nil, err 152 | } 153 | var routes []*defines.Route 154 | for _, file := range files { 155 | fileparts := strings.Split(file.Name(), ".") 156 | if len(fileparts) > 1 && fileparts[1] == "json" { 157 | route, err := fs.Get(fileparts[0]) 158 | if err == nil { 159 | routes = append(routes, route) 160 | } 161 | route.LoadBackends() 162 | } 163 | } 164 | return routes, nil 165 | } 166 | 167 | func (fs RouteFileStore) Add(route *defines.Route) error { 168 | return ioutil.WriteFile(fs.Filename(route.ID), utils.Marshal(route), 0644) 169 | } 170 | 171 | func (fs RouteFileStore) Remove(id string) bool { 172 | if _, err := os.Stat(fs.Filename(id)); err == nil { 173 | if err := os.Remove(fs.Filename(id)); err != nil { 174 | return true 175 | } 176 | } 177 | return false 178 | } 179 | -------------------------------------------------------------------------------- /status/status.go: -------------------------------------------------------------------------------- 1 | package status 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/projecteru/eru-agent/app" 9 | "github.com/projecteru/eru-agent/common" 10 | "github.com/projecteru/eru-agent/g" 11 | "github.com/projecteru/eru-agent/lenz" 12 | "github.com/projecteru/eru-agent/logs" 13 | "github.com/projecteru/eru-agent/utils" 14 | "github.com/fsouza/go-dockerclient" 15 | "github.com/keimoon/gore" 16 | ) 17 | 18 | var events chan *docker.APIEvents = make(chan *docker.APIEvents) 19 | 20 | func InitStatus() { 21 | logs.Assert(g.Docker.AddEventListener(events), "Attacher") 22 | logs.Info("Status initiated") 23 | } 24 | 25 | func load() { 26 | containers, err := g.Docker.ListContainers(docker.ListContainersOptions{All: true}) 27 | if err != nil { 28 | logs.Assert(err, "List containers") 29 | } 30 | 31 | conn := g.GetRedisConn() 32 | defer g.ReleaseRedisConn(conn) 33 | containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName) 34 | logs.Debug("Status get targets from", containersKey) 35 | rep, err := gore.NewCommand("HGETALL", containersKey).Run(conn) 36 | if err != nil { 37 | logs.Assert(err, "Status get targets") 38 | } 39 | 40 | if rep.IsNil() { 41 | return 42 | } 43 | 44 | targets, err := rep.Map() 45 | if err != nil { 46 | logs.Assert(err, "Status load targets") 47 | } 48 | 49 | logs.Debug("Status targets:", targets) 50 | logs.Info("Status load container") 51 | for _, container := range containers { 52 | if _, ok := targets[container.ID]; !ok { 53 | continue 54 | } 55 | 56 | status := getStatus(container.Status) 57 | if status != common.STATUS_START { 58 | reportContainerDeath(container.ID) 59 | continue 60 | } 61 | var meta map[string]interface{} 62 | if err := json.Unmarshal([]byte(targets[container.ID]), &meta); err != nil { 63 | logs.Info("Status load failed", err) 64 | continue 65 | } 66 | 67 | c, err := g.Docker.InspectContainer(container.ID) 68 | if err != nil { 69 | logs.Info("Status inspect docker failed", err) 70 | continue 71 | } 72 | 73 | if eruApp := app.NewEruApp(c, meta); eruApp != nil { 74 | lenz.Attacher.Attach(&eruApp.Meta) 75 | app.Add(eruApp) 76 | reportContainerCure(container.ID) 77 | } 78 | } 79 | } 80 | 81 | func Start() { 82 | logs.Info("Status monitor start") 83 | go monitor() 84 | load() 85 | } 86 | 87 | func monitor() { 88 | for event := range events { 89 | switch event.Status { 90 | case common.STATUS_DIE: 91 | logs.Debug("Status", event.Status, event.ID[:12], event.From) 92 | app.Remove(event.ID) 93 | reportContainerDeath(event.ID) 94 | case common.STATUS_START: 95 | logs.Debug("Status", event.Status, event.ID[:12], event.From) 96 | // if not in watching list, just ignore it 97 | if meta := getContainerMeta(event.ID); meta != nil && !app.Valid(event.ID) { 98 | container, err := g.Docker.InspectContainer(event.ID) 99 | if err != nil { 100 | logs.Info("Status inspect docker failed", err) 101 | break 102 | } 103 | eruApp := app.NewEruApp(container, meta) 104 | if eruApp == nil { 105 | logs.Info("Create EruApp failed") 106 | break 107 | } 108 | lenz.Attacher.Attach(&eruApp.Meta) 109 | app.Add(eruApp) 110 | reportContainerCure(event.ID) 111 | } 112 | } 113 | } 114 | } 115 | 116 | func getStatus(s string) string { 117 | switch { 118 | case strings.HasPrefix(s, "Up"): 119 | return common.STATUS_START 120 | default: 121 | return common.STATUS_DIE 122 | } 123 | } 124 | 125 | func getContainerMeta(cid string) map[string]interface{} { 126 | conn := g.GetRedisConn() 127 | defer g.ReleaseRedisConn(conn) 128 | 129 | containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName) 130 | rep, err := gore.NewCommand("HGET", containersKey, cid).Run(conn) 131 | if err != nil { 132 | logs.Info("Status get meta", err) 133 | return nil 134 | } 135 | var result map[string]interface{} 136 | if rep.IsNil() { 137 | return nil 138 | } 139 | if b, err := rep.Bytes(); err != nil { 140 | logs.Info("Status get meta", err) 141 | return nil 142 | } else { 143 | if err := json.Unmarshal(b, &result); err != nil { 144 | logs.Info("Status unmarshal meta", err) 145 | return nil 146 | } 147 | } 148 | return result 149 | } 150 | 151 | func reportContainerDeath(cid string) { 152 | conn := g.GetRedisConn() 153 | defer g.ReleaseRedisConn(conn) 154 | 155 | flagKey := fmt.Sprintf("eru:agent:%s:container:flag", cid) 156 | rep, err := gore.NewCommand("GET", flagKey).Run(conn) 157 | if err != nil { 158 | logs.Info("Status failed in get flag", err) 159 | return 160 | } 161 | if !rep.IsNil() { 162 | gore.NewCommand("DEL", flagKey).Run(conn) 163 | logs.Debug(cid[:12], "Status flag set, ignore") 164 | return 165 | } 166 | 167 | url := fmt.Sprintf("%s/api/container/%s/kill/", g.Config.Eru.Endpoint, cid) 168 | utils.DoPut(url) 169 | logs.Debug(cid[:12], "dead, remove from watching list") 170 | } 171 | 172 | func reportContainerCure(cid string) { 173 | url := fmt.Sprintf("%s/api/container/%s/cure/", g.Config.Eru.Endpoint, cid) 174 | utils.DoPut(url) 175 | logs.Debug(cid[:12], "cured, added in watching list") 176 | } 177 | -------------------------------------------------------------------------------- /lenz/attacher.go: -------------------------------------------------------------------------------- 1 | package lenz 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "strings" 7 | "sync" 8 | "time" 9 | 10 | "github.com/projecteru/eru-agent/common" 11 | "github.com/projecteru/eru-agent/defines" 12 | "github.com/projecteru/eru-agent/g" 13 | "github.com/projecteru/eru-agent/logs" 14 | "github.com/fsouza/go-dockerclient" 15 | ) 16 | 17 | type AttachManager struct { 18 | sync.Mutex 19 | attached map[string]*LogPump 20 | channels map[chan *defines.AttachEvent]struct{} 21 | } 22 | 23 | func NewAttachManager() *AttachManager { 24 | m := &AttachManager{ 25 | attached: make(map[string]*LogPump), 26 | channels: make(map[chan *defines.AttachEvent]struct{}), 27 | } 28 | return m 29 | } 30 | 31 | func (m *AttachManager) Attached(id string) bool { 32 | _, ok := m.attached[id] 33 | return ok 34 | } 35 | 36 | func (m *AttachManager) Attach(app *defines.Meta) { 37 | // Not Thread Safe 38 | if m.Attached(app.ID) { 39 | return 40 | } 41 | outrd, outwr := io.Pipe() 42 | errrd, errwr := io.Pipe() 43 | go func() { 44 | err := g.Docker.AttachToContainer(docker.AttachToContainerOptions{ 45 | Container: app.ID, 46 | OutputStream: outwr, 47 | ErrorStream: errwr, 48 | Stdin: false, 49 | Stdout: true, 50 | Stderr: true, 51 | Stream: true, 52 | }) 53 | outwr.Close() 54 | errwr.Close() 55 | logs.Debug("Lenz Attach", app.ID[:12], "finished") 56 | if err != nil { 57 | logs.Debug("Lenz Attach", app.ID, "failure:", err) 58 | } 59 | m.send(&defines.AttachEvent{Type: "detach", App: app}) 60 | m.Lock() 61 | defer m.Unlock() 62 | delete(m.attached, app.ID) 63 | }() 64 | m.Lock() 65 | m.attached[app.ID] = NewLogPump(outrd, errrd, app) 66 | m.Unlock() 67 | m.send(&defines.AttachEvent{Type: "attach", App: app}) 68 | logs.Debug("Lenz Attach", app.ID[:12], "success") 69 | } 70 | 71 | func (m *AttachManager) send(event *defines.AttachEvent) { 72 | m.Lock() 73 | defer m.Unlock() 74 | for ch, _ := range m.channels { 75 | // TODO: log err after timeout and continue 76 | ch <- event 77 | } 78 | } 79 | 80 | func (m *AttachManager) addListener(ch chan *defines.AttachEvent) { 81 | m.Lock() 82 | defer m.Unlock() 83 | m.channels[ch] = struct{}{} 84 | go func() { 85 | for _, pump := range m.attached { 86 | ch <- &defines.AttachEvent{Type: "attach", App: pump.app} 87 | } 88 | }() 89 | } 90 | 91 | func (m *AttachManager) removeListener(ch chan *defines.AttachEvent) { 92 | m.Lock() 93 | defer m.Unlock() 94 | delete(m.channels, ch) 95 | } 96 | 97 | func (m *AttachManager) Get(id string) *LogPump { 98 | m.Lock() 99 | defer m.Unlock() 100 | return m.attached[id] 101 | } 102 | 103 | func (m *AttachManager) Listen(source *defines.Source, logstream chan *defines.Log, closer <-chan bool) { 104 | if source == nil { 105 | source = new(defines.Source) 106 | } 107 | events := make(chan *defines.AttachEvent) 108 | m.addListener(events) 109 | defer m.removeListener(events) 110 | for { 111 | select { 112 | case event := <-events: 113 | if event.Type == "attach" && (source.All() || 114 | (source.ID != "" && strings.HasPrefix(event.App.ID, source.ID)) || 115 | (source.Name != "" && event.App.Name == source.Name) || 116 | (source.Filter != "" && strings.Contains(event.App.Name, source.Filter))) { 117 | pump := m.Get(event.App.ID) 118 | pump.AddListener(logstream) 119 | defer func() { 120 | if pump != nil { 121 | pump.RemoveListener(logstream) 122 | } 123 | }() 124 | } else if source.ID != "" && event.Type == "detach" && 125 | strings.HasPrefix(event.App.ID, source.ID) { 126 | return 127 | } 128 | case <-closer: 129 | return 130 | } 131 | } 132 | } 133 | 134 | type LogPump struct { 135 | sync.Mutex 136 | app *defines.Meta 137 | channels map[chan *defines.Log]struct{} 138 | } 139 | 140 | func NewLogPump(stdout, stderr io.Reader, app *defines.Meta) *LogPump { 141 | obj := &LogPump{ 142 | app: app, 143 | channels: make(map[chan *defines.Log]struct{}), 144 | } 145 | pump := func(typ string, source io.Reader) { 146 | buf := bufio.NewReader(source) 147 | for { 148 | data, err := buf.ReadBytes('\n') 149 | if err != nil { 150 | if err != io.EOF { 151 | logs.Debug("Lenz Pump:", app.ID, typ, err) 152 | } 153 | return 154 | } 155 | obj.send(&defines.Log{ 156 | Data: strings.TrimSuffix(string(data), "\n"), 157 | ID: app.ID, 158 | Name: app.Name, 159 | EntryPoint: app.EntryPoint, 160 | Ident: app.Ident, 161 | Type: typ, 162 | Datetime: time.Now().Format(common.DATETIME_FORMAT), 163 | }) 164 | } 165 | } 166 | go pump("stdout", stdout) 167 | go pump("stderr", stderr) 168 | return obj 169 | } 170 | 171 | func (o *LogPump) send(log *defines.Log) { 172 | o.Lock() 173 | defer o.Unlock() 174 | for ch, _ := range o.channels { 175 | // TODO: log err after timeout and continue 176 | ch <- log 177 | } 178 | } 179 | 180 | func (o *LogPump) AddListener(ch chan *defines.Log) { 181 | o.Lock() 182 | defer o.Unlock() 183 | o.channels[ch] = struct{}{} 184 | } 185 | 186 | func (o *LogPump) RemoveListener(ch chan *defines.Log) { 187 | o.Lock() 188 | defer o.Unlock() 189 | delete(o.channels, ch) 190 | } 191 | -------------------------------------------------------------------------------- /api/http.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | "runtime/pprof" 9 | 10 | _ "net/http/pprof" 11 | 12 | "github.com/projecteru/eru-agent/app" 13 | "github.com/projecteru/eru-agent/common" 14 | "github.com/projecteru/eru-agent/g" 15 | "github.com/projecteru/eru-agent/lenz" 16 | "github.com/projecteru/eru-agent/logs" 17 | "github.com/projecteru/eru-agent/network" 18 | "github.com/bmizerany/pat" 19 | ) 20 | 21 | // URL /version/ 22 | func version(req *Request) (int, interface{}) { 23 | return http.StatusOK, JSON{"version": common.VERSION} 24 | } 25 | 26 | // URL /profile/ 27 | func profile(req *Request) (int, interface{}) { 28 | r := JSON{} 29 | for _, p := range pprof.Profiles() { 30 | r[p.Name()] = p.Count() 31 | } 32 | return http.StatusOK, r 33 | } 34 | 35 | // URL /api/app/list/ 36 | func listEruApps(req *Request) (int, interface{}) { 37 | ret := JSON{} 38 | for ID, EruApp := range app.Apps { 39 | ret[ID] = EruApp.Meta 40 | } 41 | return http.StatusOK, ret 42 | } 43 | 44 | // URL /api/eip/release/ 45 | func releaseEIP(req *Request) (int, interface{}) { 46 | type EIP struct { 47 | ID int `json:"id"` 48 | } 49 | type Result struct { 50 | Succ int `json:"succ"` 51 | Err string `json:"err"` 52 | Veth string `json:"id"` 53 | } 54 | 55 | eips := []EIP{} 56 | decoder := json.NewDecoder(req.Body) 57 | if err := decoder.Decode(&eips); err != nil { 58 | return http.StatusBadRequest, JSON{"message": "wrong JSON format"} 59 | } 60 | 61 | rv := []Result{} 62 | for _, eip := range eips { 63 | vethName := fmt.Sprintf("%s%d", common.VLAN_PREFIX, eip.ID) 64 | if err := network.DelMacVlanDevice(vethName); err != nil { 65 | rv = append(rv, Result{Succ: 0, Veth: vethName, Err: err.Error()}) 66 | logs.Info("Release EIP failed", err, vethName) 67 | continue 68 | } 69 | rv = append(rv, Result{Succ: 1, Veth: vethName}) 70 | } 71 | 72 | return http.StatusOK, rv 73 | } 74 | 75 | // URL /api/eip/bind/ 76 | func bindEIP(req *Request) (int, interface{}) { 77 | type EIP struct { 78 | ID int `json:"id"` 79 | IP string `json:"ip"` 80 | Broadcast string `json:"broadcast"` 81 | } 82 | type Result struct { 83 | Succ int `json:"succ"` 84 | Err string `json:"err"` 85 | IP string `json:"ip"` 86 | } 87 | 88 | eips := []EIP{} 89 | decoder := json.NewDecoder(req.Body) 90 | if err := decoder.Decode(&eips); err != nil { 91 | return http.StatusBadRequest, JSON{"message": "wrong JSON format"} 92 | } 93 | 94 | rv := []Result{} 95 | for _, eip := range eips { 96 | vethName := fmt.Sprintf("%s%d", common.VLAN_PREFIX, eip.ID) 97 | veth, err := network.AddMacVlanDevice(vethName, vethName) 98 | if err != nil { 99 | rv = append(rv, Result{Succ: 0, IP: eip.IP, Err: err.Error()}) 100 | logs.Info("API add EIP failed", err) 101 | continue 102 | } 103 | 104 | if err := network.BindAndSetup(veth, eip.IP); err != nil { 105 | rv = append(rv, Result{Succ: 0, IP: eip.IP, Err: err.Error()}) 106 | network.DelVlan(veth) 107 | logs.Info("API bind EIP failed", err) 108 | continue 109 | } 110 | 111 | if err := network.SetBroadcast(vethName, eip.Broadcast); err != nil { 112 | rv = append(rv, Result{Succ: 0, IP: eip.IP, Err: err.Error()}) 113 | network.DelVlan(veth) 114 | logs.Info("API set broadcast failed", err) 115 | continue 116 | } 117 | 118 | rv = append(rv, Result{Succ: 1, IP: eip.IP}) 119 | } 120 | 121 | return http.StatusOK, rv 122 | } 123 | 124 | // URL /api/container/:container_id/addvlan/ 125 | func addVlanForContainer(req *Request) (int, interface{}) { 126 | type Endpoint struct { 127 | Nid int `json:"nid"` 128 | IP string `json:"ip"` 129 | } 130 | type Result struct { 131 | Succ int `json:"succ"` 132 | ContainerID string `json:"container_id"` 133 | VethName string `json:"veth"` 134 | IP string `json:"ip"` 135 | } 136 | 137 | cid := req.URL.Query().Get(":container_id") 138 | 139 | endpoints := []Endpoint{} 140 | decoder := json.NewDecoder(req.Body) 141 | if err := decoder.Decode(&endpoints); err != nil { 142 | return http.StatusBadRequest, JSON{"message": "wrong JSON format"} 143 | } 144 | 145 | rv := []Result{} 146 | for seq, endpoint := range endpoints { 147 | vethName := fmt.Sprintf("%s%d.%d", common.VLAN_PREFIX, endpoint.Nid, seq) 148 | if network.AddVlan(vethName, endpoint.IP, cid) { 149 | rv = append(rv, Result{Succ: 1, ContainerID: cid, VethName: vethName, IP: endpoint.IP}) 150 | continue 151 | } 152 | rv = append(rv, Result{Succ: 0, ContainerID: "", VethName: "", IP: ""}) 153 | } 154 | return http.StatusOK, rv 155 | } 156 | 157 | // URL /api/container/:container_id/addcalico/ 158 | func addCalicoForContainer(req *Request) (int, interface{}) { 159 | type Endpoint struct { 160 | Nid int `json:"nid"` 161 | Profile string `json:"profile"` 162 | IP string `json:"ip"` 163 | Append bool `json:"append"` 164 | } 165 | type Result struct { 166 | Succ int `json:"succ"` 167 | ContainerID string `json:"container_id"` 168 | IP string `json:"ip"` 169 | Err string `json:"err"` 170 | } 171 | 172 | if g.Config.VLan.Calico == "" { 173 | return http.StatusBadRequest, JSON{"message": "Agent not enable calico support"} 174 | } 175 | 176 | cid := req.URL.Query().Get(":container_id") 177 | env := os.Environ() 178 | env = append(env, fmt.Sprintf("ETCD_AUTHORITY=%s", g.Config.VLan.Calico)) 179 | 180 | endpoints := []Endpoint{} 181 | decoder := json.NewDecoder(req.Body) 182 | if err := decoder.Decode(&endpoints); err != nil { 183 | return http.StatusBadRequest, JSON{"message": "wrong JSON format"} 184 | } 185 | 186 | rv := []Result{} 187 | for seq, endpoint := range endpoints { 188 | vethName := fmt.Sprintf("%s%d.%d", common.VLAN_PREFIX, endpoint.Nid, seq) 189 | if err := network.AddCalico(env, endpoint.Append, cid, vethName, endpoint.IP); err != nil { 190 | rv = append(rv, Result{Succ: 0, ContainerID: cid, IP: endpoint.IP, Err: err.Error()}) 191 | logs.Info("API calico add interface failed", err) 192 | continue 193 | } 194 | 195 | //TODO remove when eru-core support ACL 196 | // currently only one profile is used 197 | if err := network.BindCalicoProfile(env, cid, endpoint.Profile); err != nil { 198 | rv = append(rv, Result{Succ: 0, ContainerID: cid, IP: endpoint.IP, Err: err.Error()}) 199 | logs.Info("API calico add profile failed", err) 200 | continue 201 | } 202 | 203 | rv = append(rv, Result{Succ: 1, ContainerID: cid, IP: endpoint.IP}) 204 | } 205 | return http.StatusOK, rv 206 | } 207 | 208 | // URL /api/container/publish/ 209 | func publishContainer(req *Request) (int, interface{}) { 210 | type PublicInfo struct { 211 | EIP string `json:"eip"` 212 | Dest string `json:"dest"` 213 | Ident string `json:"ident"` 214 | } 215 | 216 | info := &PublicInfo{} 217 | decoder := json.NewDecoder(req.Body) 218 | if err := decoder.Decode(info); err != nil { 219 | return http.StatusBadRequest, JSON{"message": "wrong JSON format"} 220 | } 221 | 222 | if err := network.AddPrerouting(info.EIP, info.Dest, info.Ident); err != nil { 223 | logs.Info("Public application failed", err) 224 | return http.StatusBadRequest, JSON{"message": "publish application failed"} 225 | } 226 | return http.StatusOK, JSON{"message": "ok"} 227 | } 228 | 229 | // URL /api/container/unpublish/ 230 | func unpublishContainer(req *Request) (int, interface{}) { 231 | type PublicInfo struct { 232 | EIP string `json:"eip"` 233 | Dest string `json:"dest"` 234 | Ident string `json:"ident"` 235 | } 236 | 237 | info := &PublicInfo{} 238 | decoder := json.NewDecoder(req.Body) 239 | if err := decoder.Decode(info); err != nil { 240 | return http.StatusBadRequest, JSON{"message": "wrong JSON format"} 241 | } 242 | 243 | if err := network.DelPrerouting(info.EIP, info.Dest, info.Ident); err != nil { 244 | logs.Info("Diable application failed", err) 245 | return http.StatusBadRequest, JSON{"message": "disable application failed"} 246 | } 247 | return http.StatusOK, JSON{"message": "ok"} 248 | } 249 | 250 | // URL /api/container/:container_id/addroute/ 251 | func addRouteForContainer(req *Request) (int, interface{}) { 252 | type Entry struct { 253 | CIDR string 254 | Interface string 255 | } 256 | cid := req.URL.Query().Get(":container_id") 257 | data := &Entry{} 258 | 259 | decoder := json.NewDecoder(req.Body) 260 | if err := decoder.Decode(data); err != nil { 261 | return http.StatusBadRequest, JSON{"message": "wrong JSON format"} 262 | } 263 | if !network.AddRoute(cid, data.CIDR, data.Interface) { 264 | logs.Info("Add route failed") 265 | return http.StatusServiceUnavailable, JSON{"message": "add route failed"} 266 | } 267 | return http.StatusOK, JSON{"message": "ok"} 268 | } 269 | 270 | // URL /api/container/:container_id/setroute/ 271 | func setRouteForContainer(req *Request) (int, interface{}) { 272 | type Gateway struct { 273 | IP string 274 | } 275 | cid := req.URL.Query().Get(":container_id") 276 | data := &Gateway{} 277 | 278 | decoder := json.NewDecoder(req.Body) 279 | if err := decoder.Decode(data); err != nil { 280 | return http.StatusBadRequest, JSON{"message": "wrong JSON format"} 281 | } 282 | if !network.SetDefaultRoute(cid, data.IP) { 283 | logs.Info("Set default route failed") 284 | return http.StatusServiceUnavailable, JSON{"message": "set default route failed"} 285 | } 286 | return http.StatusOK, JSON{"message": "ok"} 287 | } 288 | 289 | // URL /api/container/add/ 290 | func addNewContainer(req *Request) (int, interface{}) { 291 | type Data struct { 292 | Control string `json:"control"` 293 | ContainerID string `json:"container_id"` 294 | Meta map[string]interface{} `json:"meta"` 295 | } 296 | 297 | data := &Data{} 298 | decoder := json.NewDecoder(req.Body) 299 | err := decoder.Decode(data) 300 | if err != nil { 301 | return http.StatusBadRequest, JSON{"message": "wrong JSON format"} 302 | } 303 | 304 | switch data.Control { 305 | case "+": 306 | if app.Valid(data.ContainerID) { 307 | break 308 | } 309 | logs.Info("API status watch", data.ContainerID) 310 | container, err := g.Docker.InspectContainer(data.ContainerID) 311 | if err != nil { 312 | logs.Info("API status inspect docker failed", err) 313 | break 314 | } 315 | if eruApp := app.NewEruApp(container, data.Meta); eruApp != nil { 316 | lenz.Attacher.Attach(&eruApp.Meta) 317 | app.Add(eruApp) 318 | } 319 | } 320 | return http.StatusOK, JSON{"message": "ok"} 321 | } 322 | 323 | func HTTPServe() { 324 | restfulAPIServer := pat.New() 325 | 326 | handlers := map[string]map[string]func(*Request) (int, interface{}){ 327 | "GET": { 328 | "/profile/": profile, 329 | "/version/": version, 330 | "/api/app/list/": listEruApps, 331 | }, 332 | "POST": { 333 | "/api/container/add/": addNewContainer, 334 | "/api/container/:container_id/addvlan/": addVlanForContainer, 335 | "/api/container/:container_id/addcalico/": addCalicoForContainer, 336 | "/api/container/:container_id/setroute/": setRouteForContainer, 337 | "/api/container/:container_id/addroute/": addRouteForContainer, 338 | "/api/eip/bind/": bindEIP, 339 | "/api/eip/release/": releaseEIP, 340 | "/api/container/publish/": publishContainer, 341 | "/api/container/unpublish/": unpublishContainer, 342 | }, 343 | } 344 | 345 | for method, routes := range handlers { 346 | for route, handler := range routes { 347 | restfulAPIServer.Add(method, route, http.HandlerFunc(JSONWrapper(handler))) 348 | } 349 | } 350 | 351 | http.Handle("/", restfulAPIServer) 352 | logs.Info("API http server start at", g.Config.API.Addr) 353 | err := http.ListenAndServe(g.Config.API.Addr, nil) 354 | if err != nil { 355 | logs.Assert(err, "ListenAndServe: ") 356 | } 357 | } 358 | --------------------------------------------------------------------------------