├── common ├── compute │ ├── doc.go │ ├── compute.go │ └── functions.go ├── doc.go ├── .DS_Store ├── time.go ├── rpc.go ├── hash.go ├── global.go ├── dataparam.go └── mapdata.go ├── dispatcher ├── doc.go ├── summary.go └── dispatcher.go ├── input ├── .DS_Store ├── testinput │ ├── testinput_test.go │ └── testinput.go ├── csvinput │ ├── csvinput_test.go │ └── csvinput.go └── input.go ├── service ├── .DS_Store ├── tcp │ ├── tcpclient.go │ └── tcp.go ├── http │ ├── httpclient.go │ └── http.go ├── cell │ ├── column.go │ ├── order.go │ ├── cell.go │ └── time.go ├── online │ ├── online.go │ ├── session.go │ └── client.go ├── service.go ├── rpc │ └── rpc.go └── protocol │ └── protocol.go ├── README.md ├── test.csv ├── cache ├── doc.go ├── cache_test.go ├── timecache_test.go ├── redis_test.go ├── cache.go ├── redis.go └── timecache.go ├── main_test.go ├── relation.xml ├── .gitignore ├── log ├── log.xml └── log.go ├── config ├── relation.go ├── rpc.go ├── config_test.go └── config.go ├── config.xml ├── main.go ├── rpc.xml └── LICENSE /common/compute/doc.go: -------------------------------------------------------------------------------- 1 | //公式计算 2 | package compute 3 | -------------------------------------------------------------------------------- /common/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | 公共模块 3 | */ 4 | package common 5 | -------------------------------------------------------------------------------- /dispatcher/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | 还没有想到更好的方法 3 | */ 4 | package dispatcher 5 | -------------------------------------------------------------------------------- /common/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seefan/monitor/HEAD/common/.DS_Store -------------------------------------------------------------------------------- /input/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seefan/monitor/HEAD/input/.DS_Store -------------------------------------------------------------------------------- /service/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seefan/monitor/HEAD/service/.DS_Store -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | monitor 2 | ======= 3 | 对分钟级汇总数据进行处理，组合成不同维度的指标，并支持以TCP或WEBSOCKET方式推送给客户端。 4 | 5 | 实时数据监控 6 | -------------------------------------------------------------------------------- /common/compute/compute.go: -------------------------------------------------------------------------------- 1 | package compute 2 | 3 | import ( 4 | //"github.com/robertkrimen/otto" 5 | ) 6 | -------------------------------------------------------------------------------- /test.csv: -------------------------------------------------------------------------------- 1 | 1,2,3,1,2012-11-11 12:00 2 | 2,4,5,10,2012-11-11 12:00 3 | 1,2,3,3,2012-11-11 12:00 4 | 2,4,5,100,2012-11-11 12:00 -------------------------------------------------------------------------------- /cache/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | System:Period:[数据类型ID]=数据时间粒度 3 | [时间]:[数据类型ID]:[网元ID]=该网元指标 4 | System:Relation:[ID]=该节点下属所有子节点ID的数组 5 | */ 6 | package cache 7 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "monitor/input/testinput" 5 | "testing" 6 | ) 7 | 8 | func Test(t *testing.T) { 9 | testinput.Test_TestInput(t) 10 | } 11 | -------------------------------------------------------------------------------- /relation.xml: -------------------------------------------------------------------------------- 1 | BSC1110100113100 -------------------------------------------------------------------------------- /cache/cache_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | //import ( 4 | // "testing" 5 | // "time" 6 | //) 7 | 8 | //func Test_Set(t *testing.T) { 9 | // Set("test1", time.Now()) 10 | // t.Log("set test1") 11 | //} 12 | 13 | //func Test_Get(t *testing.T) { 14 | // _, ok := Get("test1") 15 | // if !ok { 16 | // t.Error("not found test1") 17 | // } 18 | //} 19 | -------------------------------------------------------------------------------- /common/time.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | //按时间粒度取时间的格式 4 | func GetTimeFormat(period int) string { 5 | switch period { 6 | case 1, 5: 7 | return "2006-01-02 15:04" 8 | case 10, 15, 30: 9 | return "2006-01-02 15:0" 10 | case 60: 11 | return "2006-01-02 15" 12 | case 1440: 13 | return "2006-01-02" 14 | default: 15 | return "2006-01-02 15:04" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | test 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | monitor 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | main_test.go 21 | .DS_Store 22 | *.exe 23 | -------------------------------------------------------------------------------- /common/rpc.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type RPCResponseSql struct { 8 | Cols map[string]int 9 | Rows [][]interface{} 10 | } 11 | type RPCRequestSql struct { 12 | Id string 13 | Param map[string]interface{} 14 | Params []map[string]interface{} 15 | } 16 | 17 | func (this *RPCRequestSql) ToString() string { 18 | return fmt.Sprintf("Sql Id:%s\tParam:%v", this.Id, this.Param) 19 | } 20 | -------------------------------------------------------------------------------- /service/tcp/tcpclient.go: -------------------------------------------------------------------------------- 1 | package tcp 2 | 3 | import ( 4 | "monitor/log" 5 | "monitor/service/online" 6 | "net" 7 | ) 8 | 9 | //tcp client 10 | type TcpClient struct { 11 | online.Client 12 | Connect *net.TCPConn 13 | } 14 | 15 | func (this *TcpClient) Close() { 16 | this.Client.Close() 17 | if this.Connect != nil { 18 | if err := this.Connect.Close(); err != nil { 19 | log.Errorf("client %s close with error:%s", this.Name, err.Error()) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /input/testinput/testinput_test.go: -------------------------------------------------------------------------------- 1 | package testinput 2 | 3 | import ( 4 | "monitor/config" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func Test_TestInput(t *testing.T) { 10 | c, err := config.Read("../../config.xml") 11 | if err != nil { 12 | t.Error(err.Error()) 13 | } 14 | for _, cfg := range c.Inputs.TestInputs { 15 | test := New(cfg) 16 | go func() { 17 | for out := range test.Output { 18 | t.Log(out) 19 | } 20 | }() 21 | test.Start() 22 | time.Sleep(time.Second) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /service/http/httpclient.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "code.google.com/p/go.net/websocket" 5 | "monitor/log" 6 | "monitor/service/online" 7 | ) 8 | 9 | //http client 10 | type HttpClient struct { 11 | online.Client 12 | Connect *websocket.Conn 13 | } 14 | 15 | func (this *HttpClient) Close() { 16 | this.Client.Close() 17 | if this.Connect != nil { 18 | if err := this.Connect.Close(); err != nil { 19 | log.Errorf("client %s close with error:%s", this.Name, err.Error()) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /input/csvinput/csvinput_test.go: -------------------------------------------------------------------------------- 1 | package csvinput 2 | 3 | import ( 4 | "monitor/common" 5 | "monitor/config" 6 | "testing" 7 | ) 8 | 9 | func Test_TestInput(t *testing.T) { 10 | c, err := config.Read("../../config.xml") 11 | if err != nil { 12 | t.Error(err.Error()) 13 | } 14 | 15 | for _, cfg := range c.Inputs.CsvInputs { 16 | test := New(cfg) 17 | common.IsRun = true 18 | test.Config.FileName = "../../test.csv" 19 | test.Start() 20 | for o := range test.Output { 21 | println(o.Time) 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /cache/timecache_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | //import ( 4 | // "monitor/common" 5 | // "monitor/config" 6 | // "monitor/input/testinput" 7 | // "testing" 8 | //) 9 | 10 | //func Test_Add(t *testing.T) { 11 | // mc := NewTimeCache("test", "CID") 12 | // c, err := config.Read("../config.xml") 13 | // if err != nil { 14 | // t.Fatal(err.Error()) 15 | // } 16 | // Init(c, nil) 17 | // input := testinput.New(c.Inputs.TestInputs[0]) 18 | // common.IsRun = true 19 | // input.Start() 20 | // for rows := range input.Output { 21 | // mc.Add(rows) 22 | // } 23 | //} 24 | -------------------------------------------------------------------------------- /common/hash.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | func HashCode(in string) int32 { 8 | // Initialize output 9 | var hash int32 10 | // Empty string has a hashcode of 0 11 | if len(in) == 0 { 12 | return hash 13 | } 14 | // Convert string into slice of bytes 15 | b := []byte(in) 16 | // Build hash 17 | for i := range b { 18 | char := b[i] 19 | hash = ((hash << 5) - hash) + int32(char) 20 | } 21 | return hash 22 | } 23 | func HashString(in string) string { 24 | code := HashCode(in) 25 | return strconv.Itoa(int(code)) 26 | } 27 | -------------------------------------------------------------------------------- /common/global.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "runtime" 5 | "time" 6 | ) 7 | 8 | var ( 9 | MaxCPU int //最大可用的CPU数 10 | StoreMinute time.Duration = 2 11 | StoreCount int = 20 12 | IsRun bool //系统是否正在运行 13 | IsDebug bool //是否为调试状态 14 | ) 15 | 16 | const ( 17 | TimeFormat string = "2006-01-02 15:04" 18 | MiniTimeFormat string = "02:15:04" 19 | NewLine byte = '\n' 20 | BufferSize = 1024 21 | ) 22 | 23 | func init() { 24 | MaxCPU = runtime.NumCPU() - 1 25 | } 26 | -------------------------------------------------------------------------------- /log/log.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /service/cell/column.go: -------------------------------------------------------------------------------- 1 | package cell 2 | 3 | import ( 4 | "monitor/cache" 5 | "monitor/common" 6 | "monitor/common/compute" 7 | // "monitor/log" 8 | "time" 9 | ) 10 | 11 | //单时间点，多指标，单网元 12 | //取一个网元指定时间的多指标，主要用于查询某网元的详细指标 13 | func GetColumnsData(dp *common.DataParam, t time.Time) *common.OutputParam { 14 | tmp := []*common.DataRow{} 15 | period := 1 16 | if p, ok := cache.Get(cache.FormatKey("System", "Period", dp.DataKey)); ok { 17 | period = compute.AsInt(p) 18 | } 19 | //key := cache.FormatKey(t.Format(common.GetTimeFormat(period)), dp.DataKey, dp.LimitId[0]) 20 | //log.Info("id=%s", key) 21 | if r, ok := cache.GetDataRow(dp.DataKey, t.Format(common.GetTimeFormat(period)), dp.LimitId[0]); ok { 22 | tmp = append(tmp, r) 23 | } 24 | 25 | //限制输出 26 | result := new(common.OutputParam) 27 | if len(tmp) > 0 { 28 | result.RowMap = make(map[string]int) 29 | for i, k := range dp.OutputKey { 30 | result.RowMap[k] = i 31 | } 32 | result.Time = time.Now() 33 | result.Rows = append(result.Rows, createRow(dp, tmp[0])) 34 | } 35 | return result 36 | } 37 | -------------------------------------------------------------------------------- /config/relation.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/xml" 5 | "io/ioutil" 6 | "os" 7 | ) 8 | 9 | //数据关系，用于标记一类数据的父子关系 10 | type RelationChild struct { 11 | Id string 12 | Childs []string `xml:"Childs>Child"` 13 | } 14 | 15 | //数据关系配置 16 | type RelationConfig struct { 17 | Relations []*RelationParent `xml:"Relations>Relation"` 18 | } 19 | 20 | //数据关系，用于标记一类数据的父子关系的集合 21 | type RelationParent struct { 22 | Id string 23 | Childs []*RelationChild `xml:"Childs>Child"` 24 | } 25 | 26 | //从文件中读取出Config 27 | func ReadRelation(file string) (*RelationConfig, error) { 28 | content, err := ioutil.ReadFile(file) 29 | if err != nil { 30 | return nil, err 31 | } 32 | var result RelationConfig 33 | err = xml.Unmarshal(content, &result) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return &result, nil 38 | } 39 | 40 | //将Config保存为文件 41 | func (this *RelationConfig) Write(file string) error { 42 | bt, err := xml.Marshal(this) 43 | if err != nil { 44 | return err 45 | } 46 | if err = ioutil.WriteFile(file, bt, os.ModePerm); err != nil { 47 | return err 48 | } 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /input/input.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "monitor/common" 5 | "monitor/config" 6 | "monitor/log" 7 | ) 8 | 9 | type InputStartFace interface { 10 | Start() 11 | GetConfig() *config.Input 12 | GetOutput() chan *common.DataRows 13 | } 14 | type InputFace struct { 15 | Output chan *common.DataRows 16 | RowMap *common.DataRowMap 17 | } 18 | 19 | func (this *InputFace) InitColumn(c *config.Expression, k []string) { 20 | this.RowMap = new(common.DataRowMap) 21 | if c == nil { 22 | log.Infoln("output column is nil") 23 | return 24 | } 25 | i := 0 26 | this.RowMap.Key = make(map[string]int) 27 | if c.Sum != nil { 28 | this.RowMap.Sum = make(map[string]int) 29 | for _, v := range c.Sum { 30 | this.RowMap.Sum[v] = i 31 | this.RowMap.Key[v] = i 32 | i = i + 1 33 | } 34 | } 35 | if c.Max != nil { 36 | this.RowMap.Max = make(map[string]int) 37 | for _, v := range c.Max { 38 | this.RowMap.Key[v] = i 39 | this.RowMap.Max[v] = i 40 | i = i + 1 41 | } 42 | } 43 | if c.Min != nil { 44 | this.RowMap.Min = make(map[string]int) 45 | for _, v := range c.Min { 46 | this.RowMap.Key[v] = i 47 | this.RowMap.Min[v] = i 48 | i = i + 1 49 | } 50 | } 51 | 52 | if k != nil { 53 | for _, v := range k { 54 | this.RowMap.Key[v] = i 55 | i = i + 1 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /config/rpc.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/xml" 5 | "io/ioutil" 6 | "os" 7 | ) 8 | 9 | type RPCConfig struct { 10 | DataBases []*DataBase `xml:"DataBases>DataBase"` //多个数据源 11 | Sqls []*Sql `xml:"Sqls>Sql"` //Sql配置，根据id决定不同的功能 12 | RPC *Service //对外的RPC服务 13 | } 14 | 15 | //数据库配置 16 | type DataBase struct { 17 | Id string `xml:"id,attr"` 18 | DSN string 19 | Host string 20 | Port int 21 | SID string 22 | User string 23 | Password string 24 | IsPrimary bool //是否是主库 25 | } 26 | 27 | //Sql结构 28 | type Sql struct { 29 | Id string // 30 | SqlString string // 31 | Params []string `xml:"Params>Param"` //param name 32 | From string //dbid from database 33 | } 34 | 35 | //从文件中读取出Config 36 | func ReadRPC(file string) (*RPCConfig, error) { 37 | content, err := ioutil.ReadFile(file) 38 | if err != nil { 39 | return nil, err 40 | } 41 | var result RPCConfig 42 | err = xml.Unmarshal(content, &result) 43 | if err != nil { 44 | return nil, err 45 | } 46 | return &result, nil 47 | } 48 | 49 | //将Config保存为文件 50 | func (this *RPCConfig) Write(file string) error { 51 | bt, err := xml.Marshal(this) 52 | if err != nil { 53 | return err 54 | } 55 | if err = ioutil.WriteFile(file, bt, os.ModePerm); err != nil { 56 | return err 57 | } 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /config.xml: -------------------------------------------------------------------------------- 1 | 192.168.8.113637912trueCIDSTART_TIMEN2N3N6falsetrueCID2006-01-02 15:041START_TIMECIDSTART_TIMEN2N3N6truetrueCID2006-01-02 15:041test.csvACELLBSCCIDSTART_TIMEtrue0.0.0.06780true0.0.0.06781true0.0.0.06782true30true -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | // "io/ioutil" 5 | "monitor/cache" 6 | "monitor/common" 7 | "monitor/config" 8 | "monitor/dispatcher" 9 | "runtime" 10 | "time" 11 | ) 12 | 13 | func main() { 14 | runtime.GOMAXPROCS(common.MaxCPU) 15 | println("cpu num is ", common.MaxCPU) 16 | cfg, err := config.Read("config.xml") 17 | if err != nil { 18 | panic(err.Error()) 19 | } 20 | rcfg, err := config.ReadRelation("relation.xml") 21 | if err != nil { 22 | panic(err.Error()) 23 | } 24 | cache.Init(cfg, rcfg) 25 | LoadBaseData() 26 | dispatcher.Start(cfg) 27 | time.Sleep(time.Hour) 28 | //if str, err := ioutil.ReadFile("config/config.xml"); err == nil { 29 | // ioutil.WriteFile("config.xml", str, os.ModePerm) 30 | //} else { 31 | // panic(err.Error()) 32 | //}// 33 | 34 | //test.Test_Dispatcher() 35 | //test.Test_MutiCache() 36 | //test.Test_Push() 37 | //test.Test_WebSocket() 38 | a, err := time.Parse(common.TimeFormat, "2014-02-12 12:59") 39 | b, err := time.Parse(common.TimeFormat, "2014-02-12 13:00") 40 | println(a.Before(b), err) 41 | println(a.After(b)) 42 | println(a.Sub(b)) 43 | println(a.String()) 44 | println(b.String()) 45 | } 46 | func LoadBaseData() { 47 | cache.Set("Relation:1", []string{"1", "2", "3", "4", "5"}) 48 | cache.Set("Relation:11", []string{"1", "2", "3", "4", "5"}) 49 | cache.Set("Relation:2", []string{"1", "2", "3", "4", "5"}) 50 | } 51 | -------------------------------------------------------------------------------- /rpc.xml: -------------------------------------------------------------------------------- 1 | 192.168.8.1131521orcldsamdsamtrueacellSELECT sum(n2) n2,sum(n3) n3,sum(n6) n6,cid, to_char( start_time,'yyyy-mm-dd hh24:mi') start_time from DS_PRD_A_CELL_M t where start_time>=to_date(':start_time','yyyy-mm-dd hh24:mi:ss') and start_time<to_date(':end_time','yyyy-mm-dd hh24:mi:ss') group by cid,to_char( start_time,'yyyy-mm-dd hh24:mi')start_timeend_timedb113loginSELECT USER_PWD FROM CFG_USER_G WHERE USER_NAME = :loginloginall_nodeSELECT SCENEID, NEID FROM CFG_SCENE_NEinsert_nodeinsert into CFG_SCENE_NE (SCENEID, NEID) values(:parentid,:childid) parentidchildiddelete_nodedelete from CFG_SCENE_NE where SCENEID=:parentid and NEID=:childid) parentidchildid0.0.0.06782true -------------------------------------------------------------------------------- /service/cell/order.go: -------------------------------------------------------------------------------- 1 | package cell 2 | 3 | import ( 4 | "monitor/cache" 5 | "monitor/common" 6 | // "monitor/log" 7 | "monitor/common/compute" 8 | "sort" 9 | "time" 10 | ) 11 | 12 | var ( 13 | //Config *config.Config 14 | ) 15 | 16 | //单时间点，单指标，多网元 17 | //主要查多个网元指定时间的排序 18 | func GetOrderData(dp *common.DataParam, t time.Time) *common.OutputParam { 19 | tmp := []*common.DataRow{} 20 | period := 1 21 | if p, ok := cache.Get(cache.FormatKey("System", "Period", dp.DataKey)); ok { 22 | period = compute.AsInt(p) 23 | } 24 | // limit := []string{} 25 | if len(dp.LimitNodeId) > 0 { //双重限制，与LimitId同时起作用 26 | 27 | } else { 28 | for _, v := range dp.LimitId { //取出所有的数据 29 | // key := cache.FormatKey(t.Format(common.GetTimeFormat(period)), dp.DataKey, v) 30 | if r, ok := cache.GetDataRow(dp.DataKey, t.Format(common.GetTimeFormat(period)), v); ok { //都是最新一个时间点的，都在内存 31 | tmp = append(tmp, r) 32 | } 33 | } 34 | } 35 | //排序 36 | sort.Sort(common.ByKey{tmp, dp.OrderKey}) 37 | //限制输出 38 | result := new(common.OutputParam) 39 | if len(tmp) > 0 { 40 | result.RowMap = make(map[string]int) 41 | for i, k := range dp.OutputKey { 42 | result.RowMap[k] = i 43 | } 44 | result.Time = time.Now() 45 | } 46 | if dp.OrderBy == common.Desc { 47 | for i := len(tmp); i > 0 && i > len(tmp)-dp.DataRange; i-- { 48 | result.Rows = append(result.Rows, createRow(dp, tmp[i-1])) 49 | } 50 | } else { 51 | for i := 0; i < len(tmp) && i < dp.DataRange; i++ { 52 | result.Rows = append(result.Rows, createRow(dp, tmp[i])) 53 | } 54 | } 55 | return result 56 | } 57 | -------------------------------------------------------------------------------- /service/cell/cell.go: -------------------------------------------------------------------------------- 1 | package cell 2 | 3 | import ( 4 | "monitor/cache" 5 | "monitor/common" 6 | "monitor/log" 7 | "time" 8 | ) 9 | 10 | //处理单元入口 11 | func GetData(dp *common.DataParam, t time.Time) *common.OutputParam { 12 | if dp.TimeRange == 1 && len(dp.LimitId) == 1 { //单网元，单时间点，用于指标详细 13 | log.Infoln("开始处理过程，单网元，单时间点，用于指标详细") 14 | return GetColumnsData(dp, t) 15 | } else if dp.TimeRange > 1 && len(dp.LimitId) == 1 { //单网元，多时间点，用于走势图 16 | log.Infoln("开始处理过程，单网元，多时间点，用于走势图") 17 | return GetTimeLineData(dp, t) 18 | } else if dp.TimeRange == 1 && (len(dp.LimitId) > 1 || len(dp.LimitNodeId) > 0) { //单时间点，多网元，用于排行榜 19 | log.Infoln("开始处理过程，单时间点，多网元，用于排行榜") 20 | return GetOrderData(dp, t) 21 | } else { //多时间点，多网元，暂不支持 22 | log.Error("开始处理过程，不被支持的过程", dp.ToString()) 23 | return nil 24 | } 25 | } 26 | 27 | //从内部存贮创建一个新行 28 | func createRow(dp *common.DataParam, r *common.DataRow) []interface{} { 29 | var row []interface{} 30 | for _, k := range dp.OutputKey { 31 | if v, ok := r.GetValue(k); ok { 32 | row = append(row, v) 33 | } else { //回填数据，从缓存中按回填key取值,key值在dp中指定 34 | //log.Infoln("key is ", dp.FillKey) 35 | v = getFillValue(dp.FillKey, r, k) 36 | row = append(row, v) 37 | } 38 | } 39 | return row 40 | } 41 | 42 | //回填 43 | func getFillValue(fillKey string, r *common.DataRow, k string) interface{} { 44 | if len(fillKey) > 0 { 45 | key := cache.FormatKey(fillKey, r.GetKey(fillKey), k) 46 | // log.Infoln("key is ", key) 47 | if tv, ok := cache.Get(key); ok { 48 | return tv 49 | } 50 | } 51 | return nil 52 | } 53 | func getStoreCache() *common.DataRow { 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /service/online/online.go: -------------------------------------------------------------------------------- 1 | package online 2 | 3 | import ( 4 | "encoding/json" 5 | "monitor/common" 6 | "monitor/log" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | var ( 12 | onlineClient = make(map[string]*Client) 13 | currentTime time.Time //time from data push 14 | lock sync.RWMutex 15 | ) 16 | 17 | //通知所有客户端有时间消息 18 | func SendTimeMessageToAll(tm *common.TimeMessage) { 19 | log.Infoln("send message is ", tm.Time, tm.Key, len(onlineClient)) 20 | for _, c := range onlineClient { 21 | go c.Processing(tm) 22 | } 23 | } 24 | 25 | //取得当前时间 26 | func GetCurrentTime() time.Time { 27 | lock.RLock() 28 | defer lock.RUnlock() 29 | return currentTime 30 | } 31 | 32 | //设置当前时间 33 | func SetCurrentTime(t time.Time) { 34 | lock.Lock() 35 | defer lock.Unlock() 36 | currentTime = t 37 | } 38 | 39 | //send cmd to all client 40 | func SendToAll(cmd interface{}) { 41 | if jscmd, err := json.Marshal(cmd); err == nil { 42 | for _, c := range onlineClient { 43 | c.SendBytes(jscmd) 44 | } 45 | } 46 | } 47 | 48 | //将客户端加到在线列表 49 | func Set(c *Client) { 50 | if _, ok := onlineClient[c.UUID]; !ok { //说明已存在同名用户 51 | onlineClient[c.UUID] = c 52 | } 53 | 54 | } 55 | 56 | //取指定客户端 57 | func Get(name string) (client *Client, ok bool) { 58 | if v, exists := onlineClient[name]; exists { 59 | client = v 60 | ok = true 61 | } 62 | return nil, false 63 | } 64 | 65 | //将客户端从在线列表中删除 66 | func Delete(name string) { 67 | if c, ok := onlineClient[name]; ok { 68 | var cc ClientClose 69 | cc = c 70 | cc.Close() 71 | delete(onlineClient, name) 72 | log.Infof("client: %s close and remove from online", c.Name) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /service/cell/time.go: -------------------------------------------------------------------------------- 1 | package cell 2 | 3 | import ( 4 | "monitor/cache" 5 | "monitor/common" 6 | "monitor/common/compute" 7 | // "monitor/log" 8 | "sort" 9 | "time" 10 | ) 11 | 12 | //时间线，单指标，单网元 13 | //查询一段时间内指定网元，某指标的走势 14 | func GetTimeLineData(dp *common.DataParam, t time.Time) *common.OutputParam { 15 | tmp := []*common.DataRow{} 16 | //log.Info("timeline @", dp.Id, t) 17 | period := 1 18 | if p, ok := cache.Get(cache.FormatKey("System", "Period", dp.DataKey)); ok { 19 | period = compute.AsInt(p) 20 | } 21 | for i := 0; i < dp.TimeRange; i++ { //取出所有的数据 22 | tmpTime := t.Add(time.Minute * time.Duration(-i)) 23 | // key := cache.FormatKey(tmpTime.Format(common.GetTimeFormat(period)), dp.DataKey, dp.LimitId[0]) 24 | //log.Info("search key is ", key) 25 | if r, ok := cache.GetDataRow(dp.DataKey, tmpTime.Format(common.GetTimeFormat(period)), dp.LimitId[0]); ok { 26 | tmp = append(tmp, r) 27 | //log.Infoln("find cache", r.Row) 28 | } 29 | } 30 | //log.Info("result1") 31 | //排序 32 | sort.Sort(common.ByTime{tmp}) 33 | //log.Info("result2") 34 | result := new(common.OutputParam) 35 | if len(tmp) > 0 { 36 | result.RowMap = make(map[string]int) 37 | for i, k := range dp.OutputKey { 38 | result.RowMap[k] = i 39 | } 40 | result.Time = t 41 | } 42 | //log.Info("result3") 43 | //限制输出 44 | //start := 0 45 | //if len(tmp) >= dp.DataRange { 46 | // start = len(tmp) - dp.DataRange 47 | //} 48 | //for i := start; i < len(tmp); i++ { 49 | // result.Rows = append(result.Rows, createRow(dp, tmp[i])) 50 | //} 51 | for _, r := range tmp { 52 | result.Rows = append(result.Rows, createRow(dp, r)) 53 | } 54 | //log.Info("result") 55 | return result 56 | } 57 | -------------------------------------------------------------------------------- /service/service.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | //"sync" 5 | // "monitor/cache" 6 | "monitor/common" 7 | "monitor/config" 8 | "monitor/service/http" 9 | "monitor/service/online" 10 | "monitor/service/rpc" 11 | "monitor/service/tcp" 12 | "time" 13 | "uway/log" 14 | ) 15 | 16 | var ( 17 | changes = make(chan *common.TimeMessage, 10) 18 | PushDelay float64 = 60 19 | delayTime time.Time = time.Now().Truncate(time.Second) 20 | ) 21 | 22 | //启用服务 23 | func Start(conf *config.Config) { 24 | if conf.Tcp.Enable { 25 | tcp.StartTcpService(conf.Tcp.Host, conf.Tcp.Port) 26 | } 27 | if conf.Http.Enable { 28 | http.StartHttpService(conf.Http.Host, conf.Http.Port) 29 | } 30 | PushDelay = conf.PushDelay 31 | rpc.ServerIP = conf.RPC.Host 32 | rpc.Port = conf.RPC.Port 33 | go Run() 34 | log.Info("service is starting...") 35 | } 36 | 37 | func Run() { 38 | go timeTick() 39 | for tm := range changes { 40 | if len(tm.Key) != 0 && online.GetCurrentTime().Before(tm.Time) { //push time 41 | online.SetCurrentTime(tm.Time) 42 | } 43 | 44 | //两种情况，实时推送（pushdelay==0 ,tm是由数据推送的，key不为空）；定时推送（pushdelay>0 并超时，不考虑key）； 45 | if PushDelay == 0 && len(tm.Key) != 0 || PushDelay > 0 && time.Since(delayTime).Seconds() >= PushDelay { 46 | delayTime = time.Now() 47 | tm.Time = online.GetCurrentTime() 48 | //log.Infof("now push data @ ", tm.Key, tm.Time) 49 | online.SendTimeMessageToAll(tm) 50 | } 51 | } 52 | } 53 | func Close() { 54 | close(changes) 55 | } 56 | 57 | //定时产生时间点 58 | func timeTick() { 59 | times := time.Tick(time.Second * 10) 60 | for common.IsRun { 61 | <-times 62 | tm := &common.TimeMessage{} 63 | changes <- tm 64 | } 65 | } 66 | 67 | //通知时间点变化 68 | func MessageTimeChange(t *common.TimeMessage) { 69 | //log.Info("time change", t.Time) 70 | changes <- t 71 | } 72 | -------------------------------------------------------------------------------- /cache/redis_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "fmt" 5 | "monitor/common" 6 | "monitor/config" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func Test_Redis(t *testing.T) { 12 | cfg, err := config.Read("../config.xml") 13 | if err != nil { 14 | t.Fatal(err.Error()) 15 | } 16 | Init(cfg, nil) 17 | if c, err := GetClient("d"); err != nil { 18 | t.Error(err.Error()) 19 | } else { 20 | RedisStatus() 21 | if result, err := c.Redis.AllKeys(); err == nil { 22 | t.Log("allkey", strings.Join(result, ",")) 23 | } else { 24 | t.Error(err.Error()) 25 | } 26 | if err := c.Redis.Set("test", []byte("dfdsiedskioe")); err != nil { 27 | t.Error(err.Error()) 28 | } 29 | if ss, err := c.Redis.Get("test"); err != nil { 30 | t.Error(err.Error()) 31 | } else { 32 | t.Log("test=", string(ss)) 33 | } 34 | //for i := 0; i < 100000; i++ { 35 | // c.Redis.Hset("tt", fmt.Sprintf("%s", i), []byte("test")) 36 | //} 37 | //for i := 0; i < 100000; i++ { 38 | // c.Redis.Set(fmt.Sprintf("teaaaaaaaaaaaaaaaaaaaaaaaaaaaaaadsafsdafdsfdsafasdfdsafdsafsadfasfasfasfasfaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaast%s", i), []byte("test")) 39 | //} 40 | c.Close() 41 | 42 | for i := 0; i < 100000; i++ { 43 | if cc, err := GetClient(fmt.Sprintf("%s", i)); err != nil { 44 | t.Error(err.Error()) 45 | } else { 46 | cc.Redis.Set(fmt.Sprintf("%s:%d", "teaaaaaaaaaaaaaaaaaaaaaaaaaaaaaadsafsdafdsfdsafasdfdsafdsafsadfasfasfasfasfaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaast", i), []byte("test")) 47 | cc.Close() 48 | } 49 | 50 | } 51 | 52 | } 53 | 54 | } 55 | func Test_Code(t *testing.T) { 56 | row := &common.DataRow{} 57 | row.Row = append(row.Row, 1) 58 | row.RowMap = new(common.DataRowMap) 59 | row.RowMap.Key = make(map[string]int) 60 | row.RowMap.Key["id"] = 0 61 | v, _ := encode(*row) 62 | vr, _ := decode(v) 63 | println("vr=", vr.RowMap.Key) 64 | } 65 | -------------------------------------------------------------------------------- /service/online/session.go: -------------------------------------------------------------------------------- 1 | package online 2 | 3 | import ( 4 | "monitor/log" 5 | "time" 6 | ) 7 | 8 | var ( 9 | sessions = make(map[string]*Session) //存贮在线用户的uid和key 10 | ) 11 | 12 | const Expired = 60 * 30 13 | 14 | //init 15 | func init() { 16 | log.Info("Session is started,expired every %d seconds", Expired) 17 | go func() { 18 | var i int 19 | timer := time.Tick(time.Minute) 20 | for t := range timer { 21 | i = 0 22 | lock.Lock() 23 | for k, s := range sessions { 24 | if s.IsExpired() { 25 | delete(sessions, k) 26 | i = i + 1 27 | } 28 | } 29 | lock.Unlock() 30 | log.Infof("clear %d sessions at %v,used %v", i, t, time.Since(t)) 31 | } 32 | }() 33 | } 34 | 35 | //web专用session 36 | type Session struct { 37 | UID string 38 | Name string 39 | Key string 40 | LastTime time.Time 41 | } 42 | 43 | //是否过期 44 | func (this *Session) IsExpired() bool { 45 | return time.Since(this.LastTime).Seconds() > Expired 46 | } 47 | 48 | //设置值 49 | func SetSession(uid string, name string, key string) { 50 | s := &Session{UID: uid, Key: key, LastTime: time.Now()} 51 | lock.Lock() 52 | defer lock.Unlock() 53 | sessions[uid] = s 54 | } 55 | 56 | //取取，如果过期，串也为空 57 | func GetSession(id string) (name string, key string, ok bool) { 58 | lock.RLock() 59 | defer lock.RUnlock() 60 | 61 | if s, exists := sessions[id]; exists { 62 | key = s.Key 63 | name = s.Name 64 | ok = true 65 | s.LastTime = time.Now() 66 | } 67 | //log.Info("get session id ", id, name, key, ok) 68 | return 69 | } 70 | 71 | //更新Session的过期时间 72 | func FreshSession(id string) { 73 | lock.Lock() 74 | defer lock.Unlock() 75 | if s, ok := sessions[id]; ok { 76 | s.LastTime = time.Now() 77 | } 78 | } 79 | 80 | //删除一个Session 81 | func RemoveSession(id string) { 82 | lock.Lock() 83 | defer lock.Unlock() 84 | if _, ok := sessions[id]; ok { 85 | delete(sessions, id) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | //lg "code.google.com/p/log4go" 5 | "log" 6 | ) 7 | 8 | func init() { 9 | //lg.LoadConfiguration("log.xml") 10 | } 11 | 12 | // Tracef formats message according to format specifier 13 | // and writes to default logger with log level = Trace. 14 | func Tracef(format string, params ...interface{}) { 15 | //lg.Trace(format, params...) 16 | log.Printf(format, params...) 17 | } 18 | 19 | // Debugf formats message according to format specifier 20 | // and writes to default logger with log level = Debug. 21 | func Debugf(format string, params ...interface{}) { 22 | log.Printf(format, params...) 23 | } 24 | 25 | // Infof formats message according to format specifier 26 | // and writes to default logger with log level = Info. 27 | func Infof(format string, params ...interface{}) { 28 | log.Printf(format, params...) 29 | } 30 | 31 | // Warnf formats message according to format specifier and writes to default logger with log level = Warn 32 | func Warnf(format string, params ...interface{}) { 33 | log.Printf(format, params...) 34 | } 35 | 36 | // Errorf formats message according to format specifier and writes to default logger with log level = Error 37 | func Errorf(format string, params ...interface{}) { 38 | log.Printf(format, params...) 39 | } 40 | 41 | // Trace formats message using the default formats for its operands and writes to default logger with log level = Trace 42 | func Trace(v ...interface{}) { 43 | log.Print(v) 44 | } 45 | 46 | // Debug formats message using the default formats for its operands and writes to default logger with log level = Debug 47 | func Debug(v ...interface{}) { 48 | log.Print(v) 49 | } 50 | 51 | // Info formats message using the default formats for its operands and writes to default logger with log level = Info 52 | func Info(v ...interface{}) { 53 | log.Print(v) 54 | } 55 | func Infoln(v ...interface{}) { 56 | log.Println(v) 57 | } 58 | 59 | // Warn formats message using the default formats for its operands and writes to default logger with log level = Warn 60 | func Warn(v ...interface{}) { 61 | log.Print(v) 62 | } 63 | 64 | // Error formats message using the default formats for its operands and writes to default logger with log level = Error 65 | func Error(v ...interface{}) { 66 | log.Print(v) 67 | } 68 | -------------------------------------------------------------------------------- /service/rpc/rpc.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "fmt" 5 | "monitor/log" 6 | // "net/http" 7 | "monitor/common" 8 | "net/rpc" 9 | ) 10 | 11 | var ( 12 | ServerIP string 13 | Port int 14 | ) 15 | 16 | //执行特定id的sql，并返回数据 17 | func BySql(id string) (map[string]int, [][]interface{}, error) { 18 | if client, err := rpc.DialHTTP("tcp", fmt.Sprintf("%s:%d", ServerIP, Port)); err != nil { 19 | var reply = new(common.RPCResponseSql) 20 | args := common.RPCRequestSql{Id: id} 21 | err = client.Call("Oracle.BySql", args, &reply) 22 | if err != nil { 23 | log.Error("bysql error:", err) 24 | } 25 | return reply.Cols, reply.Rows, err 26 | } else { 27 | log.Errorf("rcp %s:%d error", ServerIP, Port, err.Error()) 28 | } 29 | return nil, nil, fmt.Errorf("sql %s not exists", id) 30 | } 31 | 32 | //执行带参数的特定id的sql，并返回数据 33 | func BySqlParamName(id string, p map[string]interface{}) (map[string]int, [][]interface{}, error) { 34 | if client, err := rpc.DialHTTP("tcp", fmt.Sprintf("%s:%d", ServerIP, Port)); err != nil { 35 | var reply = new(common.RPCResponseSql) 36 | args := common.RPCRequestSql{Id: id, Param: p} 37 | err = client.Call("Oracle.BySqlParamName", args, &reply) 38 | if err != nil { 39 | log.Error("bysql error:", err) 40 | } 41 | return reply.Cols, reply.Rows, err 42 | } else { 43 | log.Errorf("rcp %s:%d error", ServerIP, Port, err.Error()) 44 | } 45 | return nil, nil, fmt.Errorf("sql %s not exists", id) 46 | } 47 | 48 | //执行带参数的特定id的sql 49 | func ExecSqlParamName(id string, p map[string]interface{}) error { 50 | if client, err := rpc.DialHTTP("tcp", fmt.Sprintf("%s:%d", ServerIP, Port)); err != nil { 51 | var reply = new(common.RPCResponseSql) 52 | args := common.RPCRequestSql{Id: id, Param: p} 53 | return client.Call("Oracle.ExecuteByParamName", args, &reply) 54 | } else { 55 | return fmt.Errorf("rcp %s:%d error", ServerIP, Port, err.Error()) 56 | } 57 | } 58 | 59 | //批量执行带参数的特定id的sql 60 | func BatchExecSqlParamName(id string, p []map[string]interface{}) error { 61 | if client, err := rpc.DialHTTP("tcp", fmt.Sprintf("%s:%d", ServerIP, Port)); err != nil { 62 | var reply = new(common.RPCResponseSql) 63 | args := common.RPCRequestSql{Id: id, Params: p} 64 | return client.Call("Oracle.BatchExecuteByParamName", args, &reply) 65 | } else { 66 | return fmt.Errorf("rcp %s:%d error", ServerIP, Port, err.Error()) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "fmt" 5 | "monitor/common" 6 | "monitor/config" 7 | "monitor/log" 8 | // "sync" 9 | ) 10 | 11 | var ( 12 | // lock sync.RWMutex 13 | TimeChanged func(*common.TimeMessage) 14 | staticCache = make(map[string]interface{}) 15 | timeCache = make(map[string]*TimeCache) 16 | ) 17 | 18 | //取缓存的数据行，如果在内部缓存中没有就到外部缓存找。前提是时间已过期 19 | func GetDataRow(dataKey string, timeKey string, id string) (*common.DataRow, bool) { 20 | //lock.RLock() 21 | //defer lock.RUnlock() 22 | if data, ok := timeCache[dataKey]; ok { 23 | if v, ok := data.Get(timeKey, id); ok { 24 | //println("neibu cache") 25 | return v.(*common.DataRow), true 26 | } else { //在内部缓存没有找到，在外部缓存查找 27 | //println("waibu cache") 28 | if !data.ExistsTime(timeKey) { //内部时间已过期 29 | //println("waibu store cache") 30 | return getFromStore(FormatStoreKey(dataKey, id), timeKey) 31 | } 32 | } 33 | } 34 | return nil, false 35 | } 36 | 37 | //get key value 38 | func Get(key string) (interface{}, bool) { 39 | if v, ok := staticCache[key]; ok { 40 | return v, true 41 | } 42 | return nil, false 43 | } 44 | func Set(key string, value interface{}) { 45 | staticCache[key] = value 46 | } 47 | 48 | //按固定格式连接关键字，组成key 49 | func FormatKey(inKeyType string, inkeyValue interface{}, outKeyName string) string { 50 | return fmt.Sprintf("%s:%v:%s", inKeyType, inkeyValue, outKeyName) 51 | } 52 | 53 | //外面用key 54 | func FormatStoreKey(inKeyType string, inkeyValue interface{}) string { 55 | return fmt.Sprintf("%s:%v", inKeyType, inkeyValue) 56 | } 57 | 58 | func CloseCache() { 59 | for _, v := range timeCache { 60 | v.Close() 61 | } 62 | } 63 | 64 | //增加一条记录 65 | func AddRowsToCache(nodeName string, rows *common.DataRows) { 66 | //log.Infof("receive %s--%s", nodeName, rows.Time) 67 | if c, ok := timeCache[nodeName]; ok { 68 | c.Add(rows) 69 | } 70 | //log.Infof("end receive %s--%s", nodeName, rows.Time) 71 | } 72 | 73 | //初始化数据关系 74 | func initRelation(cfg *config.RelationConfig) { 75 | for _, r := range cfg.Relations { 76 | Set(FormatKey("System", "Relation", r.Id), r.Childs) 77 | } 78 | } 79 | 80 | //初始化缓存 81 | func Init(cfg *config.Config, rcfg *config.RelationConfig) { 82 | log.Info("cache initing") 83 | initRedis(cfg.Redises) 84 | initTimeCache(cfg) 85 | if rcfg != nil { 86 | initRelation(rcfg) 87 | } 88 | } 89 | func addTimeCache(input *config.Input) { 90 | if input.Enable && input.EnableStore { 91 | c := NewTimeCache(input.Id, input.PrimaryKey.Value) 92 | timeCache[input.Id] = c 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /input/csvinput/csvinput.go: -------------------------------------------------------------------------------- 1 | package csvinput 2 | 3 | import ( 4 | "io/ioutil" 5 | "monitor/common" 6 | "monitor/config" 7 | "monitor/input" 8 | "monitor/log" 9 | "strconv" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | type CsvInput struct { 15 | input.InputFace 16 | Config *config.CsvInput 17 | TIME time.Time 18 | } 19 | 20 | //start oracle face 21 | func New(cfg *config.CsvInput) (db *CsvInput) { 22 | db = new(CsvInput) 23 | db.Config = cfg 24 | db.TIME = time.Now().Truncate(time.Minute) 25 | db.Output = make(chan *common.DataRows, 100) 26 | db.InitColumn(cfg.OutputColumn, cfg.OutputKey) 27 | return 28 | } 29 | 30 | func (this *CsvInput) Start() { 31 | go this.Run() 32 | } 33 | func (this *CsvInput) GetOutput() chan *common.DataRows { 34 | return this.Output 35 | } 36 | func (this *CsvInput) GetConfig() *config.Input { 37 | return &this.Config.Input 38 | } 39 | 40 | //run service 41 | func (this *CsvInput) Run() { 42 | log.Infoln("CsvInput is starting") 43 | log.Infof("open file %s", this.Config.FileName) 44 | 45 | g := float64(this.Config.Period) 46 | if bts, err := ioutil.ReadFile(this.Config.FileName); err == nil { 47 | strs := strings.Split(string(bts), "\n") 48 | for common.IsRun { 49 | beginget := time.Now() 50 | mod := this.TIME.Minute() % 3 51 | // log.Infof("w count is %d", mod+1) 52 | for i := 0; i <= mod; i++ { 53 | row := new(common.DataRows) 54 | row.Time = time.Now().Format(this.Config.TimeFormat) 55 | row.RowMap = this.RowMap 56 | for _, str := range strs { 57 | if this.Config.OutputColumn == nil { 58 | log.Infoln("output column is nil") 59 | break 60 | } 61 | cols := strings.Split(str, ",") 62 | var r []interface{} 63 | for _, c := range cols { 64 | if d, err := strconv.ParseFloat(c, 10); err == nil { 65 | r = append(r, d) 66 | } else { 67 | r = append(r, 0) 68 | } 69 | } 70 | row.Rows = append(row.Rows, r) 71 | } 72 | this.Output <- row 73 | //log.Infof("push row is %v", row.Time) 74 | } 75 | //计算下一次运行的时间 76 | 77 | this.TIME = this.TIME.Add(time.Minute * time.Duration(g)) 78 | log.Infof("next time is %v", this.TIME) 79 | sec := time.Since(beginget).Seconds() 80 | 81 | //计算下一次运行的时间 82 | if sec < g*60 { //如果一个处理周期内处理完，就延时，没有处理完就立即执行,如果正式运行期间还需要进行时间纠正 83 | log.Infof("sleep seconds %f", (g*60)-sec) 84 | time.Sleep(time.Second * (time.Duration(g*60 - sec))) //sleep one minute-sec 85 | } 86 | } 87 | 88 | } else { 89 | log.Error(err.Error()) 90 | } 91 | close(this.Output) 92 | } 93 | -------------------------------------------------------------------------------- /common/dataparam.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "encoding/json" 5 | "monitor/common/compute" 6 | "monitor/log" 7 | "time" 8 | ) 9 | 10 | const ( 11 | Desc = iota 12 | Asc 13 | ) 14 | 15 | func (this *RequestData) GetString(key string) string { 16 | if v, ok := this.Param[key]; ok { 17 | return compute.AsString(v) 18 | } 19 | return "" 20 | } 21 | func (this *RequestData) GetStringArray(key string) []string { 22 | if v, ok := this.Param[key]; ok { 23 | if v, ok := v.([]interface{}); ok { 24 | re := []string{} 25 | for _, s := range v { 26 | re = append(re, compute.AsString(s)) 27 | } 28 | return re 29 | } else { 30 | log.Infof("param %s value error", key, v) 31 | } 32 | } 33 | return nil 34 | } 35 | func (this *RequestData) GetFloat(key string) float64 { 36 | if v, ok := this.Param[key]; ok { 37 | compute.AsFloat64(v) 38 | } 39 | return -1 40 | } 41 | func (this *RequestData) GetInt64(key string) int64 { 42 | if v, ok := this.Param[key]; ok { 43 | return compute.AsInt64(v) 44 | } 45 | return -1 46 | } 47 | func (this *RequestData) GetInt(key string) int { 48 | if v, ok := this.Param[key]; ok { 49 | return compute.AsInt(v) 50 | } 51 | return -1 52 | } 53 | 54 | //input data 55 | type RequestData struct { 56 | Pid int 57 | Param map[string]interface{} 58 | } 59 | 60 | // response data 61 | type ResponseData struct { 62 | Pid int //协议号 63 | Status int //状态 64 | Output interface{} //输出内容 65 | Param interface{} 66 | } 67 | 68 | //数据推送请求参数 69 | type DataParam struct { 70 | FillKey string //回填from的key，不指定就不进行回填 71 | TimeRange int //时间点个数 72 | DataRange int //最多显示多少条数据，多时间点时，此项无效 73 | DataKey string //取哪类数据，与数据节点id对应 74 | LimitId []string //限制只取哪几个id的数据 75 | LimitNodeId string //限制父节点id，与LimitId互斥 [排行榜专用] 76 | OrderKey string //排序指标 [排行榜专用] 77 | OrderBy int //排序方式， [排行榜专用] 78 | OutputKey []string //输出的字段 79 | Id string //请求ID，在客户端要区分开，不同的请求使用不同的id，在不需要时要删除该推送请求 80 | } 81 | 82 | func (this *DataParam) ToString() string { 83 | if s, err := json.Marshal(this); err == nil { 84 | return string(s) 85 | } else { 86 | return err.Error() 87 | } 88 | 89 | } 90 | 91 | //数据输出结构 92 | type OutputParam struct { 93 | Time time.Time 94 | RowMap map[string]int 95 | Rows [][]interface{} 96 | } 97 | 98 | //func (this *DataParam) Hashcode() int32 { 99 | // tostring := fmt.Sprint(this.GetType, this.DataRange, this.DataType, strings.Join(this.LimitId, ":"), this.PrimaryKey, this.Order, strings.Join(this.OutputKey, ":"), this.Id) 100 | // return hash.HashCode(tostring) 101 | //} 102 | 103 | //用于提示时间消息的结构 104 | type TimeMessage struct { 105 | Key string 106 | Time time.Time 107 | } 108 | -------------------------------------------------------------------------------- /service/online/client.go: -------------------------------------------------------------------------------- 1 | package online 2 | 3 | import ( 4 | "encoding/json" 5 | "monitor/common" 6 | "monitor/log" 7 | "monitor/service/cell" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | //client 13 | type Client struct { 14 | Name string 15 | OutputChan chan []byte 16 | IsRun bool //是否正在运行 17 | IsLogin bool //是否已登陆 18 | IsPlayBack bool //是否是处于回放状态 19 | PlayBackTime time.Time //回放时下发数据时间点 20 | dataParams map[string]*common.DataParam 21 | lock sync.RWMutex 22 | UUID string 23 | LastUpdateTime time.Time 24 | Key string 25 | } 26 | 27 | func (this *Client) UpdateTime() { 28 | this.LastUpdateTime = time.Now() 29 | FreshSession(this.UUID) 30 | } 31 | 32 | //客户端关闭接口 33 | type ClientClose interface { 34 | Close() 35 | } 36 | 37 | //send cmd to client 38 | func (this *Client) Send(cmd *common.ResponseData) { 39 | if jscmd, err := json.Marshal(cmd); err == nil { 40 | this.OutputChan <- jscmd 41 | } 42 | } 43 | 44 | //read cmd from byte 45 | func (this *Client) Read(data []byte, cmd *common.RequestData) { 46 | if err := json.Unmarshal(data, cmd); err != nil { 47 | log.Error("cmd format error,cmd string is %s,error is %s", string(data), err.Error()) 48 | } 49 | } 50 | 51 | //注册一个新的推送类型 52 | func (this *Client) AddRequest(req *common.DataParam) { 53 | this.lock.Lock() 54 | if this.dataParams == nil { 55 | this.dataParams = make(map[string]*common.DataParam) 56 | } 57 | this.dataParams[req.Id] = req 58 | this.lock.Unlock() 59 | //println("register request", req.Id) 60 | } 61 | 62 | //删除一个推送类型 63 | func (this *Client) RemoveRequest(reqId string) { 64 | //println("delete request", reqId) 65 | this.lock.Lock() 66 | delete(this.dataParams, reqId) 67 | this.lock.Unlock() 68 | } 69 | 70 | //send cmd to client 71 | func (this *Client) SendBytes(jscmd []byte) { 72 | this.OutputChan <- jscmd 73 | } 74 | 75 | //新收到一个时间点变化，收集数据并下发 76 | func (this *Client) Processing(tm *common.TimeMessage) { 77 | //首先看本类数据是需要下发，查字典 78 | //println("get time message is ", tm.Time.String(), tm.Key) 79 | this.lock.RLock() 80 | for _, v := range this.dataParams { 81 | //log.Info("dataparams is ", v.DataKey == tm.Key, len(tm.Key) == 0) 82 | if v.DataKey == tm.Key || len(tm.Key) == 0 { //同类数据 83 | //log.Info("celling is ", v.Id) 84 | go this.celling(v, tm.Time) 85 | } 86 | } 87 | this.lock.RUnlock() 88 | } 89 | 90 | //处理各类推送数据 91 | func (this *Client) celling(dp *common.DataParam, t time.Time) { 92 | re := cell.GetData(dp, t) 93 | //log.Info("get dp %s is ", dp.Id, re != nil) 94 | if re != nil { 95 | out := common.ResponseData{2, 0, re, dp} 96 | this.Send(&out) 97 | } 98 | } 99 | 100 | //客户端关闭方法 101 | func (this *Client) Close() { 102 | close(this.OutputChan) 103 | } 104 | -------------------------------------------------------------------------------- /input/testinput/testinput.go: -------------------------------------------------------------------------------- 1 | package testinput 2 | 3 | import ( 4 | "math/rand" 5 | "monitor/common" 6 | "monitor/config" 7 | "monitor/input" 8 | "monitor/log" 9 | "time" 10 | ) 11 | 12 | type TestInput struct { 13 | input.InputFace 14 | Config *config.TestInput 15 | TIME time.Time 16 | } 17 | 18 | //start oracle face 19 | func New(cfg *config.TestInput) (db *TestInput) { 20 | db = new(TestInput) 21 | db.Config = cfg 22 | db.Output = make(chan *common.DataRows, 100) 23 | db.TIME = time.Now().Truncate(time.Minute) 24 | db.InitColumn(cfg.OutputColumn, cfg.OutputKey) 25 | return 26 | } 27 | func (this *TestInput) Start() { 28 | go this.Run() 29 | } 30 | func (this *TestInput) GetOutput() chan *common.DataRows { 31 | return this.Output 32 | } 33 | func (this *TestInput) GetConfig() *config.Input { 34 | return &this.Config.Input 35 | } 36 | 37 | //run service 38 | func (this *TestInput) Run() { 39 | log.Infoln("TestInput is starting") 40 | g := float64(this.Config.Period) 41 | for common.IsRun { 42 | beginget := time.Now() 43 | for i := 0; i < 60; i++ { 44 | row := new(common.DataRows) 45 | row.RowMap = this.RowMap 46 | if this.Config.OutputColumn == nil { 47 | log.Infoln("output column is nil") 48 | break 49 | } 50 | rnd := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) 51 | for j := 0; j < 100000; j++ { 52 | var r []interface{} 53 | if this.Config.OutputColumn.Sum != nil { 54 | for k := 0; k < len(this.Config.OutputColumn.Sum); k++ { 55 | r = append(r, rnd.Float32()) 56 | } 57 | } 58 | if this.Config.OutputColumn.Max != nil { 59 | for k := 0; k < len(this.Config.OutputColumn.Max); k++ { 60 | r = append(r, rnd.Float32()) 61 | } 62 | } 63 | if this.Config.OutputColumn.Min != nil { 64 | for k := 0; k < len(this.Config.OutputColumn.Min); k++ { 65 | r = append(r, rnd.Float32()) 66 | } 67 | } 68 | 69 | if this.Config.OutputKey != nil { 70 | for _, k := range this.Config.OutputKey { 71 | var v interface{} 72 | if k == this.Config.TimeKey { 73 | v = time.Now() 74 | } else { 75 | v = i * j 76 | } 77 | r = append(r, v) 78 | } 79 | } 80 | row.Rows = append(row.Rows, r) 81 | } 82 | 83 | row.Time = this.TIME.Format(this.Config.TimeFormat) 84 | 85 | //log.Infof("testinput 正在输出数据...%v", row.Time) 86 | this.Output <- row 87 | 88 | } 89 | //计算下一次运行的时间 90 | 91 | this.TIME = this.TIME.Add(time.Minute * time.Duration(g)) 92 | log.Infof("next time is %v", this.TIME) 93 | //计算时间间隔 94 | sec := time.Since(beginget).Seconds() 95 | 96 | //计算下一次运行的时间 97 | if sec < g*60 { //如果一个处理周期内处理完，就延时，没有处理完就立即执行,如果正式运行期间还需要进行时间纠正 98 | log.Infof("sleep seconds %f", (g*60)-sec) 99 | time.Sleep(time.Second * (time.Duration(g*60 - sec))) //sleep one minute-sec 100 | } 101 | 102 | } 103 | close(this.Output) 104 | } 105 | -------------------------------------------------------------------------------- /dispatcher/summary.go: -------------------------------------------------------------------------------- 1 | package dispatcher 2 | 3 | import ( 4 | "monitor/cache" 5 | "monitor/common" 6 | "monitor/config" 7 | "monitor/log" 8 | "sync" 9 | ) 10 | 11 | //运算结点，每类数据在一个节点运算，这样需要进行读写锁。 12 | type Summary struct { 13 | Config *config.Summary //配置 14 | Output chan *common.DataRows //数据输出流，根据配置由Summary创建 15 | dict map[string][]string //父子关系的逆向字典 16 | Id string 17 | NeedReloadRelation bool //是否需要重新加载父子关系 18 | lock sync.Mutex 19 | } 20 | 21 | //创建一个新的节点 22 | func NewSummary(cfg *config.Summary) *Summary { 23 | log.Infof("node %s is created...", cfg.Id) 24 | n := &Summary{Config: cfg} 25 | n.Id = cfg.Id 26 | n.Output = make(chan *common.DataRows, 100) 27 | n.init() 28 | return n 29 | } 30 | 31 | func (this *Summary) init() { 32 | //处理字典 33 | this.dict = make(map[string][]string) 34 | this.ReloadRelation() 35 | } 36 | 37 | /* 38 | 输入数据(小区)根据逆向字典生成输出(场景)数据(也可以使用协程)，每输出(场景)一个协程。(优先，效率高，云计算) 39 | 节点间流转数据结构:包括sum，max，min，count，key等的数组为一条记录。按顺序对应字段名。 40 | 节点只处理单数据源，无需字段描述，输出时带上数据结构或标明数据结构。 41 | */ 42 | 43 | //开始一个新的处理过程 44 | func (this *Summary) Write(row *common.DataRows) { 45 | if common.IsRun { 46 | if this.NeedReloadRelation { 47 | this.ReloadRelation() 48 | this.NeedReloadRelation = false 49 | } 50 | // log.Infof("get row count", len(row.Rows)) 51 | go this.RunIt(row) 52 | } 53 | } 54 | 55 | //刷新节点关系树 56 | func (this *Summary) ReloadRelation() { 57 | this.lock.Lock() 58 | defer this.lock.Unlock() 59 | if rs, ok := cache.Get("System:Relation:" + this.Id); ok { 60 | if vs, ok := rs.([]*config.RelationChild); ok { 61 | this.dict = make(map[string][]string) 62 | for _, r := range vs { //遍历所有关系表 63 | for _, key := range r.Childs { //遍历所有子key 64 | if v, ok := this.dict[key]; ok { //如果字典中存在就增加 65 | this.dict[key] = append(v, r.Id) 66 | } else { 67 | this.dict[key] = []string{r.Id} 68 | } 69 | } 70 | } 71 | } 72 | } 73 | } 74 | 75 | //close node 76 | func (this *Summary) Close() { 77 | close(this.Output) 78 | } 79 | func (this *Summary) RunIt(row *common.DataRows) { 80 | //log.Infoln("收到一批数据", row.Time) 81 | //将每个输入对应到特定的父级数据 82 | rmap := make(map[string]*common.DataRow) 83 | cmap := row.CloneMap() 84 | 85 | for _, v := range row.Rows { 86 | //find key 87 | key := row.GetKey(this.Config.Relation.ChildKey, v) 88 | if pkeys, ok := this.dict[key]; ok { 89 | for _, pkey := range pkeys { 90 | if r, ok := rmap[pkey]; ok { 91 | r.Merge(v) 92 | } else { 93 | rrow := new(common.DataRow) 94 | rrow.RowMap = cmap 95 | rrow.Time = row.Time 96 | rrow.Row = v 97 | rmap[pkey] = rrow 98 | } 99 | } 100 | } 101 | } 102 | //合并输出 103 | result := new(common.DataRows) 104 | result.Time = row.Time 105 | result.RowMap = row.CloneMap() 106 | result.RowMap.Key[this.Config.Relation.PrimaryKey] = cmap.ColumnCount 107 | result.RowMap.ColumnCount = cmap.ColumnCount + 1 108 | for k, v := range rmap { 109 | v.Row = append(v.Row, k) 110 | result.Rows = append(result.Rows, v.Row) 111 | } 112 | if common.IsRun { 113 | log.Info(result) 114 | this.Output <- result 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /service/tcp/tcp.go: -------------------------------------------------------------------------------- 1 | //每个请求会登陆一个Client，如果登陆成功就保存起来，继续运行。每个Client有两个协程，分别读取和写入。 2 | //写入使用一个chanel，保证同步 3 | //每个读写请求一但出错，立即抛弃Client，等待客户端再次登陆。连接会自动关闭 4 | // 5 | // 6 | // 7 | 8 | package tcp 9 | 10 | import ( 11 | "bufio" 12 | // "encoding/json" 13 | "fmt" 14 | "net" 15 | // "time" 16 | "code.google.com/p/go-uuid/uuid" 17 | "monitor/common" 18 | "monitor/log" 19 | "monitor/service/online" 20 | "monitor/service/protocol" 21 | ) 22 | 23 | var ( 24 | maxRead = 1024 25 | 26 | // Message = make(chan time.Time) 27 | ) 28 | 29 | //tcp service 30 | type TcpService struct { 31 | Host string 32 | Port int 33 | IsRun bool 34 | } 35 | 36 | //init tcp service 37 | func (this *TcpService) init(host string, port int) { 38 | this.Host = host 39 | this.Port = port 40 | } 41 | 42 | //create new http service 43 | //with host and port 44 | func StartTcpService(host string, port int) { 45 | h := new(TcpService) 46 | h.init(host, port) 47 | go h.Run() 48 | 49 | } 50 | 51 | //tcp service start 52 | func (this *TcpService) Run() { 53 | tcpAddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", this.Host, this.Port)) 54 | if err != nil { 55 | panic("resolve tcp address error") 56 | } 57 | link, err := net.ListenTCP("tcp", tcpAddr) 58 | if err != nil { 59 | panic("start tcp service error") 60 | } 61 | defer link.Close() 62 | this.IsRun = true 63 | log.Infof("tcp service started at %s:%d", this.Host, this.Port) 64 | for { 65 | conn, err := link.AcceptTCP() 66 | if err != nil { 67 | log.Error(err) 68 | continue 69 | } 70 | 71 | connFrom := conn.RemoteAddr().String() 72 | log.Infof("accept new client from %s\n", connFrom) 73 | c := new(TcpClient) 74 | c.Connect = conn 75 | c.UUID = uuid.New() 76 | c.IsRun = true 77 | c.OutputChan = make(chan []byte, 10) 78 | go this.HandleRequest(c) 79 | go this.HandleResponse(c) 80 | } 81 | } 82 | 83 | //send cmd to client 84 | func (this *TcpService) HandleResponse(c *TcpClient) { 85 | defer online.Delete(c.Name) 86 | c.Connect.SetWriteBuffer(common.BufferSize) 87 | //c.Connect.SetNoDelay(true) 88 | var err error 89 | for cmd := range c.OutputChan { 90 | //cmd := <-c.OutputChan 91 | cmd = append(cmd, '\n') 92 | if _, err = c.Connect.Write(cmd); err != nil { 93 | log.Error("send cmd error and login out,%s\n", err.Error()) 94 | c.IsRun = false 95 | break //结束协程 96 | } 97 | } 98 | } 99 | 100 | //handle tcp request 101 | func (this *TcpService) HandleRequest(c *TcpClient) { 102 | defer online.Delete(c.Name) 103 | nr := bufio.NewReader(c.Connect) 104 | var err error 105 | var data []byte 106 | cmd := new(common.RequestData) 107 | for this.IsRun && c.IsRun { 108 | data, err = nr.ReadBytes(common.NewLine) //读取一行数据 109 | if err != nil { 110 | c.IsRun = false 111 | log.Error("tcp service read error", err.Error()) 112 | break 113 | } 114 | c.Read(data, cmd) 115 | //if err := json.Unmarshal(data, &cmd); err != nil { 116 | // log.Error("cmd format error,", err.Error()) 117 | //} 118 | //log.Info("server cmd", cmd) 119 | protocol.Exec(cmd, &c.Client) 120 | } 121 | 122 | } 123 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func Test_config(t *testing.T) { 8 | cfg := new(Config) 9 | exp := new(Expression) 10 | exp.Sum = []string{"N2", "N3", "N6"} 11 | baseInput := Input{ 12 | Id: "ACELL", 13 | Enable: true, 14 | OutputKey: []string{"CID", "START_TIME"}, 15 | OutputColumn: exp, 16 | TimeFormat: "2006-01-02 15:04", 17 | PrimaryKey: &Key{Type: "number", Value: "CID"}, 18 | EnableStore: true, 19 | Period: 1, 20 | } 21 | cfg.Inputs = new(DataFace) 22 | 23 | rs := []*Redis{&Redis{Host: "192.168.8.113", Port: 6379, DB: 12, Id: "redis113", Enable: true}} 24 | cfg.Redises = rs 25 | 26 | ga := new(Summary) 27 | ga.Id = "BSC" 28 | ga.From = "ACELL" 29 | ga.TimeName = "START_TIME" 30 | ga.EnableStore = true 31 | ga.Relation = &Relation{ChildKey: "CID", PrimaryKey: "BSC"} 32 | cfg.Summarys = []*Summary{ga} 33 | test := &TestInput{ 34 | Input: baseInput, 35 | TimeKey: "START_TIME", 36 | } 37 | test.Enable = false 38 | csv := &CsvInput{ 39 | Input: baseInput, 40 | FileName: "test.csv", 41 | } 42 | csv.Enable = true 43 | csv.EnableStore = true 44 | cfg.Inputs.TestInputs = []*TestInput{test} 45 | cfg.Inputs.CsvInputs = []*CsvInput{csv} 46 | cfg.Http = &Service{Host: "0.0.0.0", Port: 6780, Enable: true} 47 | cfg.Tcp = &Service{Host: "0.0.0.0", Port: 6781, Enable: true} 48 | cfg.RPC = &Service{Host: "0.0.0.0", Port: 6782, Enable: true} 49 | cfg.PushDelay = 30 50 | cfg.IsDebug = true 51 | if err := cfg.Write("../config.xml"); err != nil { 52 | t.Error(err.Error()) 53 | } 54 | if cfg, err := Read("../config.xml"); err != nil { 55 | t.Error(err.Error()) 56 | } else { 57 | t.Log(cfg) 58 | } 59 | } 60 | func Test_rpc(t *testing.T) { 61 | cfg := new(RPCConfig) 62 | cfg.DataBases = []*DataBase{&DataBase{Id: "db113", Host: "192.168.8.113", Port: 1521, User: "dsam", Password: "dsam", SID: "orcl", IsPrimary: true}} 63 | cfg.Sqls = []*Sql{ 64 | &Sql{Id: "acell", SqlString: "SELECT sum(n2) n2,sum(n3) n3,sum(n6) n6,cid, to_char( start_time,'yyyy-mm-dd hh24:mi') start_time from DS_PRD_A_CELL_M t where start_time>=to_date(':start_time','yyyy-mm-dd hh24:mi:ss') and start_time 0 { 10 | result = 0 11 | for _, d := range params { 12 | result += AsFloat64(d) 13 | } 14 | } 15 | return 16 | } 17 | 18 | func Div(params ...interface{}) (result float64) { 19 | if len(params) > 0 { 20 | result = AsFloat64(params[0]) 21 | for i := 1; i < len(params); i++ { 22 | result -= AsFloat64(params[i]) 23 | } 24 | } 25 | return 26 | } 27 | func Max(params ...interface{}) (result float64) { 28 | if len(params) > 0 { 29 | result = AsFloat64(params[0]) 30 | for i := 1; i < len(params); i++ { 31 | if d := AsFloat64(params[i]); d > result { 32 | result = d 33 | } 34 | } 35 | } 36 | return 37 | } 38 | func Min(params ...interface{}) (result float64) { 39 | if len(params) > 0 { 40 | result = AsFloat64(params[0]) 41 | for i := 1; i < len(params); i++ { 42 | if d := AsFloat64(params[i]); d < result { 43 | result = d 44 | } 45 | } 46 | } 47 | return 48 | } 49 | func AsString(src interface{}) string { 50 | switch v := src.(type) { 51 | case string: 52 | return v 53 | case []byte: 54 | return string(v) 55 | case int8, int16, int32, int64, int, uint, uint16, uint32, uint64, uint8: 56 | return fmt.Sprintf("%d", v) 57 | case float32, float64: 58 | return fmt.Sprintf("%f", v) 59 | } 60 | return fmt.Sprintf("%v", src) 61 | } 62 | func AsFloat64(src interface{}) (result float64) { 63 | result = 0 64 | switch src.(type) { 65 | case float64: 66 | result = src.(float64) 67 | case float32: 68 | result = float64(src.(float32)) 69 | case int8: 70 | result = float64(src.(int8)) 71 | case int16: 72 | result = float64(src.(int16)) 73 | case int32: 74 | result = float64(src.(int32)) 75 | case int64: 76 | result = float64(src.(int64)) 77 | case uint8: 78 | result = float64(src.(uint8)) 79 | case uint16: 80 | result = float64(src.(uint16)) 81 | case uint32: 82 | result = float64(src.(uint32)) 83 | case uint64: 84 | result = float64(src.(uint64)) 85 | default: 86 | if dc, err := strconv.ParseFloat(AsString(src), 10); err == nil { 87 | result = dc 88 | } 89 | } 90 | return 91 | } 92 | func AsInt64(src interface{}) (result int64) { 93 | result = 0 94 | switch src.(type) { 95 | case int8: 96 | result = int64(src.(int8)) 97 | case int16: 98 | result = int64(src.(int16)) 99 | case int32: 100 | result = int64(src.(int32)) 101 | case int64: 102 | result = int64(src.(int64)) 103 | case uint8: 104 | result = int64(src.(uint8)) 105 | case uint16: 106 | result = int64(src.(uint16)) 107 | case uint32: 108 | result = int64(src.(uint32)) 109 | case uint64: 110 | result = int64(src.(uint64)) 111 | default: 112 | if dc, err := strconv.ParseInt(AsString(src), 10, 64); err == nil { 113 | result = dc 114 | } 115 | } 116 | return 117 | } 118 | func AsInt(src interface{}) (result int) { 119 | result = 0 120 | switch v := src.(type) { 121 | case int8: 122 | result = int(v) 123 | case int16: 124 | result = int(v) 125 | case int32: 126 | result = int(v) 127 | case int64: 128 | result = int(v) 129 | case uint8: 130 | result = int(v) 131 | case uint16: 132 | result = int(v) 133 | case uint32: 134 | result = int(v) 135 | case uint64: 136 | result = int(v) 137 | case float32: 138 | result = int(v) 139 | case float64: 140 | result = int(v) 141 | default: 142 | if dc, err := strconv.ParseInt(AsString(src), 10, 64); err == nil { 143 | result = int(dc) 144 | } 145 | } 146 | return 147 | } 148 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/xml" 5 | "io/ioutil" 6 | "os" 7 | ) 8 | 9 | //配置 10 | type Config struct { 11 | Redises []*Redis `xml:"Redises>Redis"` //多个缓存 12 | Inputs *DataFace //多个输入 13 | Summarys []*Summary `xml:"Summarys>Summary"` //多个计算单元 14 | Http *Service //对外的HTTP服务 15 | Tcp *Service //对外的TCP服务 16 | RPC *Service //外部的RPC服务 17 | PushDelay float64 //发送数据的时间间隔，如果设置为0,就进行实时推送，单位为秒 18 | IsDebug bool //是否为调试状态，[调试时直接登陆，不验证密码；] 19 | } 20 | 21 | //将Config转成xml格式 22 | func (this *Config) String() string { 23 | if bt, err := xml.Marshal(this); err == nil { 24 | return string(bt) 25 | } else { 26 | return err.Error() 27 | } 28 | } 29 | 30 | //将Config直接转成字节数组 31 | func (this *Config) Bytes() []byte { 32 | if bt, err := xml.Marshal(this); err == nil { 33 | return bt 34 | } 35 | return nil 36 | } 37 | 38 | //从文件中读取出Config 39 | func Read(file string) (*Config, error) { 40 | content, err := ioutil.ReadFile(file) 41 | if err != nil { 42 | return nil, err 43 | } 44 | var result Config 45 | err = xml.Unmarshal(content, &result) 46 | if err != nil { 47 | return nil, err 48 | } 49 | return &result, nil 50 | } 51 | 52 | //将Config保存为文件 53 | func (this *Config) Write(file string) error { 54 | bt, err := xml.Marshal(this) 55 | if err != nil { 56 | return err 57 | } 58 | if err = ioutil.WriteFile(file, bt, os.ModePerm); err != nil { 59 | return err 60 | } 61 | return nil 62 | } 63 | 64 | //支持的数据输入接口 65 | type DataFace struct { 66 | TcpInputs []*TcpInput 67 | TestInputs []*TestInput 68 | CsvInputs []*CsvInput 69 | RedisInputs []*RedisInput 70 | } 71 | 72 | //一个数据库的输入项 73 | type SqlInput struct { 74 | Input 75 | SqlId string //sql语句，用于查询数据库，注意与输出列对应 76 | } 77 | 78 | //一个测试用的输入项，用于测试系统运行状况 79 | type TestInput struct { 80 | Input 81 | TimeKey string //时间字段的名称，默认为START_TIME，用于推动数据流动 82 | } 83 | 84 | //csv文件输入，用于测试数据准确性 85 | type CsvInput struct { 86 | Input 87 | FileName string //文件名 88 | } 89 | 90 | //从Redis实时接收数据 91 | type RedisInput struct { 92 | Input 93 | From string //redis的id 94 | SubId string //订阅的id 95 | } 96 | 97 | //基础输入参数结构 98 | type Input struct { 99 | Id string `xml:"id,attr"` 100 | OutputKey []string `xml:"OutputKeys>Key"` //输出的关键字段key 101 | OutputColumn *Expression `xml:"OutputColumns>Column"` //输出列 102 | Enable bool //是否启用 103 | EnableStore bool //是否进行存贮 104 | PrimaryKey *Key //主键字段名 105 | TimeFormat string //时间的格式串 106 | Period int //粒度，以分钟为单位，重复输入的时间间隔。主要用于处理时间字段。 107 | } 108 | 109 | //tcp的输入参数 110 | type TcpInput struct { 111 | Input 112 | InputKey []string //输入字段的列名，按位置对应 113 | } 114 | 115 | //通用key，value 116 | type Key struct { 117 | Type string `xml:"type,attr"` //类型，包括int float time string 118 | Value string 119 | } 120 | 121 | //汇总单元，对数据按指定规则进行汇聚 122 | type Summary struct { 123 | Id string `xml:"id,attr"` 124 | From string 125 | Outputs []*Key `xml:"Outputs>Output"` //输出的key 126 | Relation *Relation //该计算单元所有需要计算的，类似bsc->cell 127 | TimeName string //时间字段的名称，默认为START_TIME，用于推动数据流动 128 | EnableStore bool //是否进行存贮 129 | } 130 | 131 | //对应的父子关系 132 | type Relation struct { 133 | PrimaryKey string //主键名 134 | ChildKey string //对应子级的字段名，与PrimaryKey联合使用 135 | } 136 | 137 | //指标计算方式 138 | type Expression struct { 139 | Sum []string 140 | Max []string 141 | Min []string 142 | } 143 | 144 | //Redis配置 145 | type Redis struct { 146 | Id string `xml:"id,attr"` 147 | Host string 148 | Port int 149 | Password string 150 | DB int //DB号 151 | Enable bool //是否启用 152 | } 153 | 154 | //Tcp服务 155 | type Service struct { 156 | Host string //ip地址 157 | Port int //端口 158 | Enable bool //是否启用 159 | } 160 | -------------------------------------------------------------------------------- /dispatcher/dispatcher.go: -------------------------------------------------------------------------------- 1 | package dispatcher 2 | 3 | import ( 4 | "monitor/cache" 5 | "monitor/common" 6 | "monitor/config" 7 | "monitor/input" 8 | "monitor/input/csvinput" 9 | "monitor/input/testinput" 10 | "monitor/log" 11 | "monitor/service" 12 | ) 13 | 14 | var ( 15 | inputs = make(map[string]input.InputStartFace) 16 | summarys = make(map[string]*Summary) 17 | link = make(map[string][]string) 18 | ) 19 | 20 | //启动处理引擎 21 | func Start(conf *config.Config) { 22 | log.Infof("dispatcher is starting") 23 | common.IsRun = true 24 | common.IsDebug = conf.IsDebug 25 | createDataPeriod(conf) 26 | service.Start(conf) 27 | createInput(conf) 28 | createSummary(conf) 29 | //cron.Start(conf) 30 | Run() 31 | } 32 | 33 | //创建数据类型与时间粒度的关系 34 | func createDataPeriod(conf *config.Config) { 35 | for _, c := range conf.Inputs.CsvInputs { 36 | createPeriodFromInput(&c.Input) 37 | } 38 | for _, c := range conf.Inputs.RedisInputs { 39 | createPeriodFromInput(&c.Input) 40 | } 41 | for _, c := range conf.Inputs.TcpInputs { 42 | createPeriodFromInput(&c.Input) 43 | } 44 | for _, c := range conf.Inputs.TestInputs { 45 | createPeriodFromInput(&c.Input) 46 | } 47 | } 48 | func createPeriodFromInput(conf *config.Input) { 49 | cache.Set(cache.FormatKey("System", "Period", conf.Id), conf.Period) 50 | } 51 | 52 | //关闭处理引擎 53 | func Close() { 54 | common.IsRun = false 55 | for k, _ := range inputs { 56 | delete(inputs, k) 57 | } 58 | for k, _ := range summarys { 59 | delete(summarys, k) 60 | } 61 | cache.CloseCache() 62 | } 63 | 64 | //按配置把输入和输出连接起来 65 | func Run() { 66 | //接收时间通知事件，将时间变化通知给服务 67 | cache.TimeChanged = service.MessageTimeChange 68 | 69 | for k, v := range inputs { 70 | go runInput(k, v) 71 | } 72 | for k, v := range summarys { 73 | go runSummary(k, v) 74 | } 75 | 76 | } 77 | 78 | //处理数据输入流 79 | func runInput(inputName string, in input.InputStartFace) { 80 | begin: 81 | out := in.GetOutput() 82 | for r := range out { //收到一条输出数据，两种选择，一是传给需要的节点，二是保存到缓存 83 | go func(r *common.DataRows) { 84 | if ns, ok := link[inputName]; ok { //如果有节点需要本节点的输出 85 | for _, fname := range ns { //给所有节点 86 | summarys[fname].Write(r) //写数据 87 | } 88 | } 89 | //如果需要保存到CACHE，这里处理 90 | //log.Infof("input is %v", in.GetConfig().EnableStore, inputName, r.Time) 91 | cache.AddRowsToCache(inputName, r) 92 | }(r) 93 | } 94 | if common.IsRun { //如果程序还没有结束，就重新启动INPUT 95 | in.Start() 96 | goto begin 97 | } 98 | } 99 | 100 | //处理节点去运算 101 | func runSummary(inputName string, node *Summary) { 102 | for rows := range node.Output { //接收数据 103 | if ns, ok := link[inputName]; ok { //如果有节点需要本节点的输出 104 | for _, fname := range ns { //给所有节点 105 | summarys[fname].Write(rows) //写数据 106 | } 107 | } 108 | //如果需要保存到CACHE，这里处理 109 | // log.Infof("node is %v", node.Config.EnableStore, inputName, rows.Time) 110 | 111 | cache.AddRowsToCache(inputName, rows) 112 | 113 | } 114 | } 115 | 116 | //按参数创建所有的输入，并加到一个字典里 117 | func createSummary(conf *config.Config) { 118 | for _, n := range conf.Summarys { 119 | summarys[n.Id] = NewSummary(n) //创建一个节点 120 | makeLink(n.Id) 121 | } 122 | } 123 | 124 | //创建节点间，数据输入流单的映射关系 125 | func makeLink(id string) { 126 | for _, fn := range summarys { //处理节点之间的引用关系 127 | if fn.Config.From == id { //找到引用当前节点的节点 128 | if ns, ok := link[id]; ok { 129 | ns = append(ns, fn.Id) 130 | link[id] = ns 131 | } else { 132 | link[id] = []string{fn.Id} 133 | } 134 | } 135 | } 136 | } 137 | 138 | //按参数创建所有的输入，并加到一个字典里 139 | func createInput(conf *config.Config) { 140 | for _, s := range conf.Inputs.TestInputs { 141 | if !s.Enable { 142 | continue 143 | } 144 | in := testinput.New(s) 145 | in.Start() 146 | 147 | inputs[s.Id] = in 148 | makeLink(s.Id) 149 | } 150 | for _, s := range conf.Inputs.CsvInputs { 151 | if !s.Enable { 152 | continue 153 | } 154 | in := csvinput.New(s) 155 | in.Start() 156 | 157 | inputs[s.Id] = in 158 | makeLink(s.Id) 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /cache/redis.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "bytes" 5 | "container/ring" 6 | "encoding/gob" 7 | "fmt" 8 | "github.com/alphazero/Go-Redis" 9 | "monitor/common" 10 | "monitor/config" 11 | "monitor/log" 12 | "sync" 13 | "time" 14 | ) 15 | 16 | var ( 17 | pool = make(map[string]chan *RedisClient) 18 | errRedis = make(map[string]*config.Redis) //重连机制，定时重试创建全新的chan，成功后从这里删除 19 | keyToClient = make(map[byte]string) 20 | ids *ring.Ring 21 | redislock sync.RWMutex 22 | ) 23 | 24 | func RedisStatus() { 25 | fmt.Println("pool size:", len(pool)) 26 | } 27 | 28 | type RedisClient struct { 29 | Redis redis.Client 30 | Id string 31 | } 32 | 33 | //id是否可用 34 | type IdAvailable struct { 35 | Id string 36 | IsAvailable bool 37 | } 38 | 39 | func (this *RedisClient) Close() { 40 | //println("close client at %s", this.Id) 41 | pool[this.Id] <- this 42 | } 43 | 44 | const ( 45 | PoolSize = 4 46 | ) 47 | 48 | //初始化redis连接池 49 | func initRedis(confs []*config.Redis) { 50 | ids = ring.New(len(confs)) 51 | for _, conf := range confs { 52 | if !conf.Enable { 53 | continue 54 | } 55 | id := &IdAvailable{Id: conf.Id} 56 | ids.Value = id 57 | spec := redis.DefaultSpec().Db(conf.DB).Password(conf.Password).Host(conf.Host).Port(conf.Port) 58 | //log.Info("redis init at ", spec) 59 | c := make(chan *RedisClient, PoolSize) 60 | for j := 0; j < PoolSize; j++ { 61 | if client, err := redis.NewSynchClientWithSpec(spec); err != nil { 62 | goto err 63 | } else { 64 | c <- &RedisClient{Id: conf.Id, Redis: client} 65 | } 66 | } 67 | pool[conf.Id] = c 68 | id.IsAvailable = true 69 | ids = ids.Next() 70 | err: //如果创建CLIENT时出错，就抛弃这个台机器 71 | errRedis[conf.Id] = conf 72 | ids = ids.Next() 73 | } 74 | } 75 | 76 | //取一个client的id，如果没有映射就要新增 77 | func getId(key string) (string, error) { 78 | bytes := []byte(key) 79 | if len(bytes) > 0 { 80 | redislock.Lock() 81 | defer redislock.Unlock() 82 | b := bytes[0] 83 | //log.Info("client b is %s", b) 84 | if id, ok := keyToClient[b]; ok { //如果有记录有对应关系 85 | //log.Info("client id is %s", id) 86 | return id, nil 87 | } else { 88 | //首先检查是否 89 | for tid := ids.Value; tid != nil && tid.(*IdAvailable).IsAvailable; ids = ids.Next() { 90 | id = tid.(*IdAvailable).Id 91 | //log.Info("client id is %s", id) 92 | break 93 | } 94 | return id, nil 95 | } 96 | } 97 | return "", fmt.Errorf("error get id with %s", key) 98 | } 99 | 100 | //创建一个连接 101 | func GetClient(key string) (*RedisClient, error) { 102 | if id, err := getId(key); err == nil { 103 | if c, ok := pool[id]; ok { 104 | timeout := time.After(time.Second * 10) 105 | //client := <-c 106 | 107 | select { 108 | case client := <-c: 109 | //log.Info("get client id is %s", client.Id) 110 | return client, nil 111 | case <-timeout: 112 | return nil, fmt.Errorf("get connection time out") 113 | } 114 | return <-c, nil 115 | } 116 | } 117 | return nil, fmt.Errorf("get client error ") 118 | } 119 | 120 | //将数据保存到外部缓存中，同时也支持从外部缓存取数据，在这里进行屏蔽，其它地方不操作外部缓存 121 | //保存到外部存贮 122 | func saveToStore(key string, value *common.DataRow, expired int) { 123 | //begin := time.Now() 124 | if len(pool) > 0 { 125 | if c, err := GetClient(value.Time); err == nil { 126 | defer c.Close() 127 | if bs, err := encode(*value); err == nil { 128 | c.Redis.Hset(key, value.Time, bs) 129 | c.Redis.Expire(key, int64(expired*60)) //秒 130 | } 131 | } else { 132 | log.Error("get client error,begin recycle", err.Error()) 133 | //回收连接，过段时间再试 134 | } 135 | } 136 | //diff := time.Since(begin) 137 | //log.Infof("save to store at %v seconds", diff.Seconds()) 138 | } 139 | 140 | //从外部取数据 141 | func getFromStore(key string, keyTime string) (*common.DataRow, bool) { 142 | if c, err := GetClient(keyTime); err == nil { 143 | defer c.Close() 144 | if bts, err := c.Redis.Hget(key, keyTime); err == nil { 145 | if r, err := decode(bts); err == nil { 146 | return r, true 147 | } 148 | } 149 | } 150 | return nil, false 151 | } 152 | 153 | //数据编码 154 | func encode(vs common.DataRow) ([]byte, error) { 155 | buff := new(bytes.Buffer) 156 | enc := gob.NewEncoder(buff) 157 | err := enc.Encode(vs) 158 | return buff.Bytes(), err 159 | } 160 | 161 | //数据解码 162 | func decode(vs []byte) (*common.DataRow, error) { 163 | buff := bytes.NewBuffer(vs) 164 | dec := gob.NewDecoder(buff) 165 | result := new(common.DataRow) 166 | err := dec.Decode(result) 167 | return result, err 168 | } 169 | -------------------------------------------------------------------------------- /cache/timecache.go: -------------------------------------------------------------------------------- 1 | /* 2 | 针对不同的数据类型，采用不同的缓存方式 3 | 1、普通不过期缓存，使用Set和Get方法统计处理，三段式Key值 4 | 2、定时过期数据，按StoreMinute进行存贮，过期清理 5 | */ 6 | package cache 7 | 8 | import ( 9 | "monitor/common" 10 | "monitor/common/compute" 11 | "monitor/config" 12 | "monitor/log" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | type TimeCache struct { 18 | Id string 19 | KeyName string //数据列字段名 20 | input chan *common.DataRows //数据输入窗口 21 | caches map[string]*TimeCacheItem //所有的缓存数据 22 | lock sync.RWMutex 23 | } 24 | type TimeCacheItem struct { 25 | Cache map[string]interface{} 26 | ExpiredTime time.Time 27 | } 28 | 29 | func (this *TimeCache) init() { 30 | this.input = make(chan *common.DataRows, 100) 31 | this.caches = make(map[string]*TimeCacheItem) 32 | } 33 | func (this *TimeCache) ExistsTime(timeKey string) bool { 34 | this.lock.RLock() 35 | defer this.lock.RUnlock() 36 | _, ok := this.caches[timeKey] 37 | return ok 38 | } 39 | func (this *TimeCache) Get(timeKey string, id string) (interface{}, bool) { 40 | this.lock.RLock() 41 | defer this.lock.RUnlock() 42 | if item, ok := this.caches[timeKey]; ok { 43 | if value, ok := item.Cache[id]; ok { 44 | return value, true 45 | } 46 | } 47 | return nil, false 48 | } 49 | 50 | //创建一个新的缓存实例 51 | func NewTimeCache(id string, keyName string) *TimeCache { 52 | c := &TimeCache{Id: id, KeyName: keyName} 53 | c.Start() 54 | return c 55 | } 56 | 57 | //启动实例 58 | func (this *TimeCache) Start() { 59 | this.init() 60 | go this.Run() 61 | go this.Clean(30) 62 | } 63 | 64 | //清理过期数据 65 | func (this *TimeCache) Clean(s time.Duration) { 66 | tc := time.Tick(time.Second * s) 67 | for t := range tc { 68 | ts := []string{} 69 | for k, v := range this.caches { 70 | if v.ExpiredTime.Before(t) { 71 | // log.Infof("data %s is expired %v,now is %s", k, v.ExpiredTime, t.String()) 72 | ts = append(ts, k) 73 | } 74 | } 75 | this.lock.Lock() 76 | for _, k := range ts { 77 | delete(this.caches, k) 78 | } 79 | this.lock.Unlock() 80 | } 81 | } 82 | 83 | //close it 84 | func (this *TimeCache) Close() { 85 | close(this.input) 86 | } 87 | 88 | //增加一条数据 89 | func (this *TimeCache) Add(rows *common.DataRows) { 90 | // log.Info("receive new rows@", rows.Time) 91 | this.input <- rows 92 | } 93 | 94 | //运行计算 95 | func (this *TimeCache) Run() { 96 | log.Infoln("cache is started", this.Id) 97 | period := 1 98 | if p, ok := Get(FormatKey("System", "Period", this.Id)); ok { 99 | period = compute.AsInt(p) 100 | } 101 | timeFormat := common.GetTimeFormat(period) 102 | 103 | for rs := range this.input { //接收数据 104 | //log.Infof("time changed ", rs.Time) 105 | currTime, err := time.Parse(timeFormat, rs.Time) 106 | if err != nil { 107 | log.Error("cache time format is error", err.Error()) 108 | currTime = time.Now() 109 | } 110 | cacheItem, ok := this.caches[rs.Time] //这个时间点是否已记录 111 | if !ok { 112 | cacheItem = &TimeCacheItem{Cache: make(map[string]interface{})} 113 | this.caches[rs.Time] = cacheItem 114 | cacheItem.ExpiredTime = time.Now().Add(time.Minute * common.StoreMinute) 115 | } 116 | 117 | for _, vs := range rs.Rows { //分析每一条数据 118 | row := rs.CreateDataRow(vs) 119 | key := row.GetKey(this.KeyName) 120 | //log.Infoln("add to cache", key) 121 | this.lock.Lock() 122 | if r, ok := cacheItem.Cache[key]; ok { //如果同类数据已存在，就进行数据合并 123 | row.Merge(r.(*common.DataRow).Row) 124 | } 125 | cacheItem.Cache[key] = row 126 | this.lock.Unlock() 127 | //log.Infoln(key, row.Row) 128 | // log.Info("key=%s", key) 129 | go saveToStore(FormatStoreKey(this.Id, key), row, period*common.StoreCount) //保存到外部存贮，并保存粒度*保存个数分钟 130 | } 131 | 132 | //通知外到有新数据到来 133 | if TimeChanged != nil { 134 | TimeChanged(&common.TimeMessage{Key: this.Id, Time: currTime}) 135 | } 136 | } 137 | } 138 | 139 | func CountCache() int { 140 | re := 0 141 | for _, v := range timeCache { 142 | for _, cv := range v.caches { 143 | re += len(cv.Cache) 144 | } 145 | } 146 | return re 147 | } 148 | 149 | //初始化时间缓存节点 150 | func initTimeCache(cfg *config.Config) { 151 | for _, input := range cfg.Inputs.CsvInputs { 152 | addTimeCache(&input.Input) 153 | } 154 | for _, input := range cfg.Inputs.RedisInputs { 155 | addTimeCache(&input.Input) 156 | } 157 | for _, input := range cfg.Inputs.TcpInputs { 158 | addTimeCache(&input.Input) 159 | } 160 | for _, input := range cfg.Inputs.TestInputs { 161 | addTimeCache(&input.Input) 162 | } 163 | for _, input := range cfg.Summarys { 164 | c := NewTimeCache(input.Id, input.Relation.PrimaryKey) 165 | timeCache[input.Id] = c 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /common/mapdata.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "monitor/common/compute" 6 | "strconv" 7 | ) 8 | 9 | //数据集，批处理用 10 | type DataRows struct { 11 | Time string 12 | RowMap *DataRowMap 13 | Rows [][]interface{} 14 | } 15 | type DataRow struct { 16 | Time string 17 | RowMap *DataRowMap 18 | Row []interface{} 19 | } 20 | 21 | //创建一个数据行结构 22 | func (this *DataRows) CreateDataRow(v []interface{}) *DataRow { 23 | return &DataRow{ 24 | Time: this.Time, 25 | RowMap: this.RowMap, 26 | Row: v, 27 | } 28 | } 29 | 30 | //节点中流转的数据结构 31 | type DataRowMap struct { 32 | Key map[string]int 33 | Sum map[string]int 34 | Max map[string]int 35 | Min map[string]int 36 | Count map[string]int 37 | ColumnCount int //列统计 38 | } 39 | 40 | func (this *DataRows) CloneMap() (rowmap *DataRowMap) { 41 | rowmap = new(DataRowMap) 42 | rowmap.Key = make(map[string]int) 43 | rowmap.Sum = make(map[string]int) 44 | rowmap.Max = make(map[string]int) 45 | rowmap.Min = make(map[string]int) 46 | rowmap.Count = make(map[string]int) 47 | if this.RowMap == nil { 48 | println("column map is not found") 49 | return 50 | } 51 | i := 0 52 | for k, v := range this.RowMap.Key { 53 | rowmap.Key[k] = v 54 | i = i + 1 55 | } 56 | for k, v := range this.RowMap.Sum { 57 | rowmap.Sum[k] = v 58 | i = i + 1 59 | } 60 | for k, v := range this.RowMap.Max { 61 | rowmap.Max[k] = v 62 | i = i + 1 63 | } 64 | for k, v := range this.RowMap.Min { 65 | rowmap.Min[k] = v 66 | i = i + 1 67 | } 68 | for k, v := range this.RowMap.Count { 69 | rowmap.Count[k] = v 70 | i = i + 1 71 | } 72 | rowmap.ColumnCount = i 73 | return 74 | } 75 | 76 | //取得指标的key值 77 | func (this *DataRows) GetKey(key string, row []interface{}) string { 78 | if this.RowMap == nil { 79 | return fmt.Sprintf("column map is not found") 80 | } 81 | if i, ok := this.RowMap.Key[key]; ok { 82 | return compute.AsString(row[i]) 83 | } 84 | //log.Infoln(this.RowMap) 85 | return fmt.Sprintf("column %s is not found", key) 86 | } 87 | 88 | //取得特定的值 89 | func (this *DataRow) GetValue(key string) (interface{}, bool) { 90 | if i, ok := this.RowMap.Key[key]; ok { 91 | return this.Row[i], true 92 | } 93 | return nil, false 94 | } 95 | 96 | //合并数据 97 | func (this *DataRow) Merge(row []interface{}) { 98 | if this.RowMap.Sum != nil { 99 | this.Sum(row) 100 | } 101 | if this.RowMap.Sum != nil { 102 | this.Max(row) 103 | } 104 | if this.RowMap.Sum != nil { 105 | this.Min(row) 106 | } 107 | if this.RowMap.Sum != nil { 108 | this.Count(row) 109 | } 110 | } 111 | 112 | //按sum计算指标 113 | func (this *DataRow) Sum(row []interface{}) { 114 | for _, i := range this.RowMap.Sum { 115 | this.Row[i] = compute.Add(this.Row[i], row[i]) 116 | } 117 | } 118 | 119 | //按max计算指标 120 | func (this *DataRow) Max(row []interface{}) { 121 | for _, i := range this.RowMap.Max { 122 | this.Row[i] = compute.Add(this.Row[i], row[i]) 123 | } 124 | } 125 | 126 | //按min计算指标 127 | func (this *DataRow) Min(row []interface{}) { 128 | for _, i := range this.RowMap.Min { 129 | this.Row[i] = compute.Add(this.Row[i], row[i]) 130 | } 131 | } 132 | 133 | //按count计算指标 134 | func (this *DataRow) Count(row []interface{}) { 135 | for _, i := range this.RowMap.Count { 136 | this.Row[i] = compute.Add(this.Row[i], 1) 137 | } 138 | } 139 | 140 | //取得指标的key值 141 | func (this *DataRow) GetKey(key string) string { 142 | if this.RowMap == nil { 143 | return fmt.Sprintf("column map is not found") 144 | } 145 | if i, ok := this.RowMap.Key[key]; ok { 146 | switch v := this.Row[i].(type) { 147 | case string: 148 | return v 149 | case []byte: 150 | return string(v) 151 | case int8, int16, int32, int64, int, uint, uint16, uint32, uint64, uint8: 152 | return fmt.Sprintf("%d", v) 153 | case float32: 154 | return strconv.FormatFloat(float64(v), 'f', 0, 32) 155 | case float64: 156 | return strconv.FormatFloat(v, 'f', 0, 64) 157 | } 158 | return fmt.Sprintf("%v", this.Row[i]) 159 | } 160 | //log.Infoln(this.RowMap) 161 | return fmt.Sprintf("column %s is not found", key) 162 | } 163 | 164 | type DataRowSorter []*DataRow 165 | 166 | func (this DataRowSorter) Len() int { 167 | return len(this) 168 | } 169 | 170 | func (this DataRowSorter) Swap(i, j int) { 171 | this[i], this[j] = this[j], this[i] 172 | } 173 | 174 | type ByTime struct { 175 | DataRowSorter 176 | } 177 | 178 | func (this ByTime) Less(i, j int) bool { 179 | return this.DataRowSorter[i].Time < this.DataRowSorter[j].Time 180 | } 181 | 182 | type ByKey struct { 183 | DataRowSorter 184 | Key string 185 | } 186 | 187 | func (this ByKey) Less(i, j int) bool { 188 | if v1, ok := this.DataRowSorter[i].GetValue(this.Key); ok { 189 | if v2, ok := this.DataRowSorter[j].GetValue(this.Key); ok { 190 | return compute.AsFloat64(v1) < compute.AsFloat64(v2) 191 | } 192 | } 193 | return true 194 | } 195 | -------------------------------------------------------------------------------- /service/http/http.go: -------------------------------------------------------------------------------- 1 | /* 2 | 用cookie保持连接，每10分钟换一个auth值。 3 | 与tcp服务不同，websocket的连接断开后不需要重新登陆，只需要验证Auth的登陆信息即可。 4 | 不同的单元可以保持不同的连接，简化客户端的代码 5 | 不同的连接产生不同的Client，同一个连接内不允许多次登陆。多次登陆也不会重新生成Client 6 | 登陆后生成key，用于同一用户多连接,and reset key 7 | */ 8 | package http 9 | 10 | import ( 11 | "code.google.com/p/go-uuid/uuid" 12 | "code.google.com/p/go.net/websocket" 13 | "fmt" 14 | "monitor/common" 15 | "monitor/log" 16 | "monitor/service/online" 17 | "monitor/service/protocol" 18 | "net/http" 19 | "time" 20 | ) 21 | 22 | var ( 23 | session = make(map[string]string) //在线用户 24 | ) 25 | 26 | const ( 27 | Sec_Auth_Uid = "Sec_Auth_Uid" 28 | Sec_Auth_Key = "Sec_Auth_Key" 29 | ) 30 | 31 | //http service 32 | type HttpService struct { 33 | Host string 34 | Port int 35 | IsRun bool 36 | } 37 | 38 | //init tcp service 39 | func (this *HttpService) init(host string, port int) { 40 | this.Host = host 41 | this.Port = port 42 | this.IsRun = true 43 | } 44 | 45 | //create new http service 46 | //with host and port 47 | func StartHttpService(host string, port int) { 48 | h := new(HttpService) 49 | h.init(host, port) 50 | go h.Run() 51 | 52 | } 53 | 54 | //处理单独的Http请求 55 | func (this *HttpService) HandleHttp(w http.ResponseWriter, req *http.Request) { 56 | req.ParseForm() 57 | uid := req.Header.Get(Sec_Auth_Uid) 58 | key := req.Header.Get(Sec_Auth_Key) 59 | c := this.getClient(uid, key) 60 | defer c.Close() 61 | 62 | if data := req.FormValue("json"); len(data) > 0 { 63 | cmd := new(common.RequestData) 64 | c.Read([]byte(data), cmd) 65 | protocol.Exec(cmd, &c.Client) 66 | } else { 67 | cmd := common.ResponseData{0, -1, "json string is empty", ""} 68 | c.Send(&cmd) 69 | } 70 | for { 71 | timeout := time.After(time.Nanosecond * 10) //10ns就超时 72 | select { 73 | case data := <-c.OutputChan: 74 | if _, err := w.Write(data); err != nil { 75 | log.Error("%s send cmd error and login out,%s\n", c.Name, err.Error()) 76 | } 77 | break 78 | case <-timeout: 79 | goto end 80 | } 81 | } 82 | end: 83 | } 84 | 85 | //按cookei取在线用户 86 | func (this *HttpService) getClient(uid string, key string) *HttpClient { 87 | c := new(HttpClient) 88 | if s, _, ok := online.GetSession(uid); ok && s == key { //key相同时认为已登陆过的 89 | if tc, ok := online.Get(uid); ok { //第一个登陆的Client的uuid被当作了uid 90 | c.Name = tc.Name 91 | c.IsLogin = tc.IsLogin 92 | c.IsPlayBack = c.IsPlayBack 93 | } 94 | } 95 | if c == nil { 96 | c = new(HttpClient) 97 | } 98 | c.UUID = uuid.New() 99 | c.IsRun = true 100 | c.OutputChan = make(chan []byte, 10) 101 | return c 102 | } 103 | 104 | ////设置cookie 105 | //func (this *HttpService) setCookie(c *HttpClient, w http.ResponseWriter, age int) { 106 | // id := &http.Cookie{Name: "monitor_http_key", Value: common.HashString(c.Name + ":monitor"), Path: "/", MaxAge: age, HttpOnly: true, Secure: true} 107 | // name := &http.Cookie{Name: "monitor_http_name", Value: c.Name, Path: "/", MaxAge: age, HttpOnly: true, Secure: true} 108 | // http.SetCookie(w, id) 109 | // http.SetCookie(w, name) 110 | //} 111 | 112 | //处理Socket请求 113 | func (this *HttpService) HandleSocket(ws *websocket.Conn) { 114 | connFrom := ws.RemoteAddr() 115 | log.Infof("accept new http client from %s\n", connFrom) 116 | uid := ws.Request().Header.Get(Sec_Auth_Uid) 117 | key := ws.Request().Header.Get(Sec_Auth_Key) 118 | c := this.getClient(uid, key) 119 | c.Connect = ws 120 | go this.HandleResponse(c) 121 | this.HandleRequest(c) 122 | } 123 | 124 | //主运行方法 125 | //根目录为标准Http请求 126 | //socket为Socket请求 127 | func (this *HttpService) Run() { 128 | this.IsRun = true 129 | http.Handle("/", http.FileServer(http.Dir("."))) // <-- note this line 130 | http.HandleFunc("/http", this.HandleHttp) 131 | http.Handle("/socket", websocket.Handler(this.HandleSocket)) 132 | log.Infof("http service started at %s:%d", this.Host, this.Port) 133 | log.Info("http socket host on /http and /socket ") 134 | if err := http.ListenAndServe(fmt.Sprintf("%s:%d", this.Host, this.Port), nil); err != nil { 135 | log.Error("http service start error", err.Error()) 136 | } 137 | } 138 | 139 | //send cmd to client 140 | func (this *HttpService) HandleResponse(c *HttpClient) { 141 | var err error 142 | defer online.Delete(c.UUID) 143 | for cmd := range c.OutputChan { 144 | //cmd := <-c.OutputChan 145 | cmd = append(cmd, '\n') 146 | if err = websocket.Message.Send(c.Connect, string(cmd)); err != nil { 147 | log.Error("%s send cmd error and login out,%s\n", c.Name, err.Error()) 148 | c.IsRun = false 149 | break 150 | } 151 | c.UpdateTime() 152 | } 153 | } 154 | 155 | //handle tcp request 156 | func (this *HttpService) HandleRequest(c *HttpClient) { 157 | var err error 158 | defer online.Delete(c.UUID) 159 | cmd := new(common.RequestData) 160 | var data []byte 161 | for this.IsRun && c.IsRun { 162 | if err = websocket.Message.Receive(c.Connect, &data); err != nil { 163 | log.Infof("%s can't received cmd", c.Name, err.Error()) 164 | c.IsRun = false 165 | break 166 | } 167 | c.Read(data, cmd) 168 | protocol.Exec(cmd, &c.Client) 169 | c.UpdateTime() 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /service/protocol/protocol.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "fmt" 5 | "monitor/common" 6 | "monitor/log" 7 | "monitor/service/online" 8 | "monitor/service/rpc" 9 | "time" 10 | ) 11 | 12 | //pid process 13 | type PidProcess func(cmd *common.RequestData, c *online.Client) 14 | 15 | var ( 16 | pid map[int]PidProcess //all pid 17 | now time.Time //start time 18 | ) 19 | 20 | //init pid 21 | func init() { 22 | pid = map[int]PidProcess{ //以0为区分，0以下的id都不需要进行登陆 23 | 0: pid0, //运行状态测试 24 | -1: pid_1, //登陆 25 | -2: pid_2, //退出 26 | -3: pid_3, //用临时key重新登陆，用户登陆后会临时产生 27 | 2: pid2, //数据推送注册 28 | 3: pid3, //数据库的读操作 29 | 4: pid4, //数据库的写操作 30 | 5: pid5, //更新节点内的父子关系 31 | } 32 | now = time.Now() 33 | } 34 | 35 | //exec pid function 36 | func Exec(cmd *common.RequestData, c *online.Client) { 37 | 38 | //没有登陆就必须登陆系统，小于等于0的协议不需要登陆。 39 | if cmd.Pid <= 0 { 40 | pid[cmd.Pid](cmd, c) 41 | } else if !c.IsLogin { 42 | cmd := common.ResponseData{1, 0, "need login", nil} 43 | c.Send(&cmd) 44 | } else { 45 | if p, ok := pid[cmd.Pid]; ok { 46 | p(cmd, c) 47 | } else { 48 | pidok(cmd, c) 49 | } 50 | } 51 | } 52 | 53 | //数据库read操作 54 | func pid3(cmd *common.RequestData, c *online.Client) { 55 | if checkParamError(cmd, c, "Id", "Param") { 56 | return 57 | } 58 | id := cmd.GetString("Id") 59 | m := make(map[string]interface{}) 60 | col, rows, err := rpc.BySqlParamName(id, m) 61 | if err != nil { 62 | log.Error(err.Error()) 63 | writePrror("db error", cmd, c) 64 | return 65 | } 66 | dp := new(common.OutputParam) 67 | dp.RowMap = col 68 | dp.Rows = rows 69 | dp.Time = time.Now() 70 | writepid(c, cmd, 0, dp) 71 | } 72 | 73 | //数据库write操作 74 | func pid4(cmd *common.RequestData, c *online.Client) { 75 | if checkParamError(cmd, c, "Id", "Param") { 76 | return 77 | } 78 | id := cmd.GetString("Id") 79 | m := make(map[string]interface{}) 80 | err := rpc.ExecSqlParamName(id, m) 81 | if err != nil { 82 | log.Error(err.Error()) 83 | writePrror("db error", cmd, c) 84 | return 85 | } 86 | writepid(c, cmd, 0, nil) 87 | } 88 | 89 | //注册推送数据的请求 90 | func pid2(cmd *common.RequestData, c *online.Client) { 91 | if checkParamError(cmd, c, "DataKey", "DataRange", "Id", "LimitId", "OrderBy", "OrderKey", "OutputKey", "TimeRange") { 92 | return 93 | } 94 | dp := new(common.DataParam) 95 | dp.DataKey = cmd.GetString("DataKey") 96 | dp.DataRange = cmd.GetInt("DataRange") 97 | dp.Id = cmd.GetString("Id") 98 | dp.LimitId = cmd.GetStringArray("LimitId") 99 | dp.OrderBy = cmd.GetInt("OrderBy") 100 | dp.OrderKey = cmd.GetString("OrderKey") 101 | dp.OutputKey = cmd.GetStringArray("OutputKey") 102 | dp.FillKey = cmd.GetString("FillKey") 103 | dp.TimeRange = cmd.GetInt("TimeRange") 104 | c.AddRequest(dp) 105 | //同时直接返回一次数据，否则界面会有空白期 106 | tm := &common.TimeMessage{Time: online.GetCurrentTime()} 107 | c.Processing(tm) 108 | } 109 | 110 | //更新节点内的父子关系，有3类变化，新增、删除 111 | func pid5(cmd *common.RequestData, c *online.Client) { 112 | if checkParamError(cmd, c, "DataKey", "Insert", "Delete", "Id") { 113 | return 114 | } 115 | } 116 | 117 | //尚未支持 118 | func pidok(cmd *common.RequestData, c *online.Client) { 119 | writepid(c, cmd, -1, "not surport") 120 | } 121 | 122 | // test service 123 | func pid0(cmd *common.RequestData, c *online.Client) { 124 | if checkParamError(cmd, c, "Test") { 125 | return 126 | } 127 | 128 | test := cmd.GetInt("Test") 129 | var result interface{} 130 | switch test { 131 | case 0: 132 | result = true 133 | case 1: 134 | result = time.Now().Format(common.TimeFormat) 135 | case 2: 136 | result = time.Since(now).String() 137 | case 3: 138 | result = now.Format(common.TimeFormat) 139 | default: 140 | result = fmt.Sprintf("not surport %d", test) 141 | } 142 | writepid(c, cmd, 0, result) 143 | } 144 | 145 | //check param exists 146 | func checkParamError(cmd *common.RequestData, c *online.Client, keys ...string) bool { 147 | for _, k := range keys { 148 | if _, ok := cmd.Param[k]; !ok { 149 | writePrror(fmt.Sprintf("param '%s' not exists", k), cmd, c) 150 | return true 151 | } 152 | } 153 | return false 154 | } 155 | 156 | //return error status=-1 157 | func writePrror(msg string, cmd *common.RequestData, c *online.Client) { 158 | writepid(c, cmd, -1, msg) 159 | } 160 | 161 | //key login in 162 | func pid_3(cmd *common.RequestData, c *online.Client) { 163 | if checkParamError(cmd, c, "UID", "Key") { 164 | return 165 | } 166 | 167 | login := cmd.GetString("UID") 168 | pwd := cmd.GetString("Key") 169 | 170 | loginSuccess := false 171 | //log.Info("pid -3 uid=key", login, pwd) 172 | //check session 173 | if name, key, ok := online.GetSession(login); ok && key == pwd { 174 | c.Name = name 175 | c.Key = key 176 | loginSuccess = true 177 | //log.Info("get session success", key) 178 | } 179 | 180 | //check 181 | re := make(map[string]interface{}) 182 | if loginSuccess { 183 | c.IsLogin = true 184 | re["LoginState"] = 0 185 | re["Message"] = "login success" 186 | re["UID"] = c.UUID 187 | re["Key"] = c.Key 188 | } else { 189 | re["LoginState"] = -1 190 | re["Message"] = "login failed" 191 | } 192 | online.Set(c) 193 | writepid(c, cmd, 0, re) 194 | } 195 | 196 | // login in 197 | func pid_1(cmd *common.RequestData, c *online.Client) { 198 | if checkParamError(cmd, c, "Login", "Password") { 199 | return 200 | } 201 | 202 | login := cmd.GetString("Login") 203 | pwd := cmd.GetString("Password") 204 | c.Name = login 205 | log.Infof("user %s login", c.Name, common.IsDebug) 206 | loginSuccess := false 207 | if common.IsDebug { //调试时直接登陆，不验证密码 208 | loginSuccess = true 209 | } else { 210 | param := make(map[string]interface{}) 211 | param["login"] = login 212 | _, epwd, err := rpc.BySqlParamName("login", param) 213 | if err != nil { 214 | log.Error(err.Error()) 215 | } 216 | if err == nil && len(epwd) > 0 { 217 | loginSuccess = epwd[0][0] == pwd //需要对密码进行加密再进行比较 218 | } 219 | } 220 | 221 | //check 222 | re := make(map[string]interface{}) 223 | if loginSuccess { 224 | c.IsLogin = true 225 | c.Key = common.HashString(c.UUID + ":" + c.Name) 226 | online.SetSession(c.UUID, c.Name, c.Key) 227 | re["LoginState"] = 0 228 | re["Message"] = "login success" 229 | re["UID"] = c.UUID 230 | re["Key"] = c.Key 231 | online.Set(c) 232 | } else { 233 | re["LoginState"] = -1 234 | re["Message"] = "login failed" 235 | } 236 | writepid(c, cmd, 0, re) 237 | } 238 | 239 | // //senc data to online.Client 240 | // func pid2(cmd *common.RequestData, c *online.Client) { 241 | // log.Println("pid 2") 242 | // //if param, ok := cmd.Param.(string); ok { 243 | // if start_time, err := time.Parse(common.TimeFormat, cmd.Param["time"]); err == nil { 244 | // log.Println(start_time) 245 | // out := online.ResponseData{1, 0, nil, cmd.Param} 246 | // //get from redis and send to all online.Client 247 | // this.SendToAll(out) 248 | // } 249 | // } 250 | 251 | //login out 252 | func pid_2(cmd *common.RequestData, c *online.Client) { 253 | //if param, ok := cmd.Param.(string); ok { 254 | log.Debug("login out ", c.Name) 255 | writepid(c, cmd, 0, nil) 256 | c.IsRun = false 257 | online.RemoveSession(c.UUID) 258 | //} 259 | } 260 | 261 | //write pid to responseout ResponseData 262 | func writepid(c *online.Client, cmd *common.RequestData, status int, result interface{}) { 263 | out := common.ResponseData{cmd.Pid, status, result, cmd.Param} 264 | c.Send(&out) 265 | } 266 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | {description} 294 | Copyright (C) {year} {fullname} 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | {signature of Ty Coon}, 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | --------------------------------------------------------------------------------