├── .gitattributes ├── .gitignore ├── COPYING ├── README.md ├── api ├── api.cpp ├── api.go ├── api.php └── api_test.go ├── conf ├── conf.go └── conf.json ├── main.go └── procs ├── alarm_handler.go ├── file_handler.go └── tubemap.go /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | clog 27 | logs/ 28 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016-2020, 353474225@qq.com 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [中文 README](#中文) 2 | 3 | 4 | # [clog](http://github.com/simplejia/clog) (Centralized log collection service) 5 | ## Original Intention 6 | * In actual projects, the service will be deplored to many servers and it is not convenient to view local logs. But it is convenient to view logs via collecting logs to one or two computers. The logs exist in the form of files and are respectively stored according to their service names, ip addresses, dates and types. 7 | * In our service, we often need add some features that have nothing to do with our business logic, such as alarm logs, reporting data for statistics and so on. It is really not necessary mix these features and business logic. While having clog, we just need send effective data and leave clog to deal with the data. 8 | 9 | ## Functions 10 | * Sending logs to remote host which can be equipped with one or two servers. 11 | * Api currently offers golang, c, php. 12 | * According to the configuration (server/conf/conf.json), clog will execute relevant log handlers. And log output and alarm process have been implemented so far. 13 | * The output log files is named according to server/logs/{module name}/log{dbg|err|info|war}/{day}/log{ip}{+}{sub}. And the logs can be saved at most 30 days. 14 | 15 | ## Usage 16 | * server 17 | > Deploring server service: server/server, configuring files: server/conf/conf.json 18 | 19 | * Recommending using [cmonitor](http://github.com/simplejia/cmonitor) to manage 20 | 21 | ## Notice 22 | * api.go has defined AddrFunc variable to get server address, you can redefine it. 23 | * In server/conf/conf.json files, tpl defines template and then is cited via $xxx. Currently supported handlers are filehandler and alarmhandler. Filehandler is used to record local logs and alarmhandler is used to send alarms. you can redefine the parameters in configuration files via passing custom env and conf parameter, for example: ./server -env dev -conf='port=8080;clog.mode=1'(multiple parameters are seperated by `;` 24 | * For alarmhandler, relevant parameter configuration can be referred to params field. In current alarms, it just provides output logs. In actual use, we should replace it with our own alarm logic by redefine procs.AlarmFunc. We can create a new go file under the directory of server/procs, as following: 25 | ``` 26 | package procs 27 | 28 | import ( 29 | "encoding/json" 30 | "os" 31 | ) 32 | 33 | func init() { 34 | // replace it with your own alarm function 35 | AlarmFunc = func(sender string, receivers []string, text string) { 36 | params := map[string]interface{}{ 37 | "Sender": sender, 38 | "Receivers": receivers, 39 | "Text": text, 40 | } 41 | json.NewEncoder(os.Stdout).Encode(params) 42 | } 43 | } 44 | ``` 45 | * Alarmhandler has anti disturbance control logic, the same content will not be alarmed again within one minute. The interval between two alarms is not less than 30 seconds. 46 | * If you want to add a new handler, you just need create a new go file under the directory of server/procs, as following: 47 | ``` 48 | package procs 49 | 50 | func XxxHandler(cate, subcate string, content []byte, params map[string]interface{}) { 51 | } 52 | 53 | func init() { 54 | RegisterHandler("xxxhandler", XxxHandler) 55 | } 56 | ``` 57 | 58 | > Handler used in product is as following:(having implemented the function of sending data to multiple subscribers after receiving it) 59 | 60 | ``` 61 | package procs 62 | 63 | import ( 64 | "encoding/json" 65 | "fmt" 66 | "net/url" 67 | "strings" 68 | "time" 69 | 70 | "github.com/simplejia/clog/api" 71 | "github.com/simplejia/utils" 72 | ) 73 | 74 | type TransParam struct { 75 | Nodes []*struct { 76 | Addr string 77 | AddrType string 78 | Retry int 79 | Host string 80 | Cgi string 81 | Params string 82 | Method string 83 | Timeout string 84 | } 85 | } 86 | 87 | func TransHandler(cate, subcate, body string, params map[string]interface{}) { 88 | clog.Info("TransHandler() Begin Trans: %s, %s, %s", cate, subcate, body) 89 | 90 | var transParam *TransParam 91 | bs, _ := json.Marshal(params) 92 | json.Unmarshal(bs, &transParam) 93 | if transParam == nil { 94 | clog.Error("TransHandler() params not right: %v", params) 95 | return 96 | } 97 | 98 | arrs := []string{body} 99 | json.Unmarshal([]byte(body), &arrs) 100 | for pos, str := range arrs { 101 | arrs[pos] = url.QueryEscape(str) 102 | } 103 | 104 | for _, node := range transParam.Nodes { 105 | addr := node.Addr 106 | ps := map[string]string{} 107 | values, _ := url.ParseQuery(fmt.Sprintf(node.Params, utils.Slice2Interface(arrs)...)) 108 | for k, vs := range values { 109 | ps[k] = vs[0] 110 | } 111 | 112 | timeout, _ := time.ParseDuration(node.Timeout) 113 | 114 | headers := map[string]string{ 115 | "Host": node.Host, 116 | } 117 | 118 | gpp := &utils.GPP{ 119 | Uri: fmt.Sprintf("http://%s/%s", addr, strings.TrimPrefix(node.Cgi, "/")), 120 | Timeout: timeout, 121 | Headers: headers, 122 | Params: ps, 123 | } 124 | 125 | for step := -1; step < node.Retry; step++ { 126 | var ( 127 | body []byte 128 | err error 129 | ) 130 | switch node.Method { 131 | case "get": 132 | body, err = utils.Get(gpp) 133 | case "post": 134 | body, err = utils.Post(gpp) 135 | } 136 | 137 | if err != nil { 138 | clog.Error("TransHandler() http error, err: %v, body: %s, gpp: %v, step: %d", err, body, gpp, step) 139 | continue 140 | } else { 141 | clog.Info("TransHandler() http success, body: %s, gpp: %v", body, gpp) 142 | break 143 | } 144 | } 145 | } 146 | 147 | return 148 | } 149 | 150 | func init() { 151 | RegisterHandler("transhandler", TransHandler) 152 | } 153 | ``` 154 | 155 | > Corresponding configuration is as following（configuring a template field in server/conf/conf.json）： 156 | 157 | ``` 158 | "tpl": { 159 | "trans": [ 160 | { 161 | "handler": "transhandler", 162 | "params": { 163 | "nodes": [ 164 | { 165 | "addr": "127.0.0.1:80", 166 | "addrType": "ip", 167 | "host": "xx.xx.com", 168 | "cgi": "/c/a", 169 | "params": "a=1&b=%s&c=%s", 170 | "method": "post", 171 | "retry": 2, 172 | "timeout": "50ms" 173 | } 174 | ] 175 | } 176 | } 177 | ] 178 | }, 179 | "procs": { 180 | "demo/logbusi_trans": "$trans" 181 | } 182 | ``` 183 | 184 | > the clog.Busi() function in api.go will be used: clog.Busi("trans", "xxx", "xxx", ...) 185 | 186 | ## demo 187 | * [api_test.go](http://github.com/simplejia/clog/tree/master/api_test.go) 188 | * [demo](http://github.com/simplejia/wsp/tree/master/demo) (has examples for using clog) 189 | 190 | --- 191 | 中文 192 | === 193 | 194 | # [clog](http://github.com/simplejia/clog) (集中式日志收集服务) 195 | ## 实现初衷 196 | * 实际项目中，服务会部署到多台服务器上去，机器本地日志不方便查看，通过集中收集日志到一台或两台机器上，日志以文件形式存在，按服务名，ip，日期，日志类型分别存储，这样查看日志时就方便多了 197 | * 我们做服务时，经常需要添加一些跟业务逻辑无关的功能，比如按错误日志报警，上报数据用于统计等等，这些功能和业务逻辑混在一起，实在没有必要，有了clog，我们只需要发送有效的数据，然后就可把数据处理的工作留给clog去做 198 | 199 | ## 功能 200 | * 发送日志至远程server主机，server可以配多台机器，api目前提供golang，c，php支持 201 | * 根据配置(server/conf/conf.json)运行相关日志分析程序，目前已实现：日志输出，报警 202 | * 输出日志文件按server/logs/{模块名}/log{dbg|err|info|war}/{day}/log{ip}{+}{sub}规则命名，最多保存30天日志 203 | 204 | ## 使用方法 205 | * server机器 206 | > 布署server服务：server/server，配置文件：server/conf/conf.json 207 | 208 | * server服务建议用[cmonitor](http://github.com/simplejia/cmonitor)启动管理 209 | 210 | ## 注意 211 | * api.go文件里定义了获取server服务addr方法 212 | * server/conf/conf.json文件里，tpl定义模板，然后通过`$xxx`方式引用，目前支持的handler有：filehandler和alarmhandler，filehandler用来记录本地日志，alarmhandler用来发报警，可以通过传入自定义的env及conf参数来重定义配置文件里的参数，如：./server -env dev -conf='port=8080;clog.mode=1'，多个参数用`;`分隔 213 | * 对于alarmhandler，相关参数配置见params，目前的报警只是打印日志，实际实用，应替换成自己的报警处理逻辑，重新赋值procs.AlarmFunc就可以了，可以在server/procs目录下新建一个go文件，如下示例： 214 | ``` 215 | package procs 216 | 217 | import ( 218 | "encoding/json" 219 | "os" 220 | ) 221 | 222 | func init() { 223 | // 请替换成你自己的报警处理函数 224 | AlarmFunc = func(sender string, receivers []string, text string) { 225 | params := map[string]interface{}{ 226 | "Sender": sender, 227 | "Receivers": receivers, 228 | "Text": text, 229 | } 230 | json.NewEncoder(os.Stdout).Encode(params) 231 | } 232 | } 233 | ``` 234 | * alarmhandler有防骚扰控制逻辑，相同内容，一分钟内不再报，两次报警不少于30秒，以上限制和日志文件一一对应 235 | * 如果想添加新的handler，只需在server/procs目录下新建一个go文件，如下示例： 236 | ``` 237 | package procs 238 | 239 | func XxxHandler(cate, subcate string, content []byte, params map[string]interface{}) { 240 | } 241 | 242 | func init() { 243 | RegisterHandler("xxxhandler", XxxHandler) 244 | } 245 | ``` 246 | 247 | > 一个实际生产环境使用到的handler如下：（实现了接收数据后分发给多个订阅者的功能） 248 | 249 | ``` 250 | package procs 251 | 252 | import ( 253 | "encoding/json" 254 | "fmt" 255 | "net/url" 256 | "strings" 257 | "time" 258 | 259 | "github.com/simplejia/clog/api" 260 | "github.com/simplejia/utils" 261 | ) 262 | 263 | type TransParam struct { 264 | Nodes []*struct { 265 | Addr string 266 | AddrType string 267 | Retry int 268 | Host string 269 | Cgi string 270 | Params string 271 | Method string 272 | Timeout string 273 | } 274 | } 275 | 276 | func TransHandler(cate, subcate, body string, params map[string]interface{}) { 277 | clog.Info("TransHandler() Begin Trans: %s, %s, %s", cate, subcate, body) 278 | 279 | var transParam *TransParam 280 | bs, _ := json.Marshal(params) 281 | json.Unmarshal(bs, &transParam) 282 | if transParam == nil { 283 | clog.Error("TransHandler() params not right: %v", params) 284 | return 285 | } 286 | 287 | arrs := []string{body} 288 | json.Unmarshal([]byte(body), &arrs) 289 | for pos, str := range arrs { 290 | arrs[pos] = url.QueryEscape(str) 291 | } 292 | 293 | for _, node := range transParam.Nodes { 294 | addr := node.Addr 295 | ps := map[string]string{} 296 | values, _ := url.ParseQuery(fmt.Sprintf(node.Params, utils.Slice2Interface(arrs)...)) 297 | for k, vs := range values { 298 | ps[k] = vs[0] 299 | } 300 | 301 | timeout, _ := time.ParseDuration(node.Timeout) 302 | 303 | headers := map[string]string{ 304 | "Host": node.Host, 305 | } 306 | 307 | gpp := &utils.GPP{ 308 | Uri: fmt.Sprintf("http://%s/%s", addr, strings.TrimPrefix(node.Cgi, "/")), 309 | Timeout: timeout, 310 | Headers: headers, 311 | Params: ps, 312 | } 313 | 314 | for step := -1; step < node.Retry; step++ { 315 | var ( 316 | body []byte 317 | err error 318 | ) 319 | switch node.Method { 320 | case "get": 321 | body, err = utils.Get(gpp) 322 | case "post": 323 | body, err = utils.Post(gpp) 324 | } 325 | 326 | if err != nil { 327 | clog.Error("TransHandler() http error, err: %v, body: %s, gpp: %v, step: %d", err, body, gpp, step) 328 | continue 329 | } else { 330 | clog.Info("TransHandler() http success, body: %s, gpp: %v", body, gpp) 331 | break 332 | } 333 | } 334 | } 335 | 336 | return 337 | } 338 | 339 | func init() { 340 | RegisterHandler("transhandler", TransHandler) 341 | } 342 | ``` 343 | 344 | > 相应配置如下（server/conf/conf.json里配上一个模板）： 345 | 346 | ``` 347 | "tpl": { // 模板定义 348 | "trans": [ 349 | { 350 | "handler": "transhandler", 351 | "params": { 352 | "nodes": [ 353 | { 354 | "addr": "127.0.0.1:80", 355 | "addrType": "ip", 356 | "host": "xx.xx.com", 357 | "cgi": "/c/a", 358 | "params": "a=1&b=%s&c=%s", 359 | "method": "post", 360 | "retry": 2, 361 | "timeout": "50ms" 362 | } 363 | ] 364 | } 365 | } 366 | ] 367 | }, 368 | "procs": { 369 | "demo/logbusi_trans": "$trans" 370 | } 371 | ``` 372 | 373 | > 使用时调用api.go里提供的clog.Busi函数: clog.Busi("trans", "xxx", "xxx", ...) 374 | 375 | ## demo 376 | * [api_test.go](http://github.com/simplejia/clog/tree/master/api_test.go) 377 | * [demo](http://github.com/simplejia/wsp/tree/master/demo) (demo项目里有clog的使用例子) 378 | 379 | -------------------------------------------------------------------------------- /api/api.cpp: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | using namespace std; 16 | 17 | const string g_module = "demo"; 18 | const string g_server_ip = "127.0.0.1"; 19 | const int g_server_port = 28702; 20 | const string local_ip = "127.0.0.1"; // TODO: replace it with your localip 21 | 22 | struct Conf 23 | { 24 | struct Clog { 25 | int mode; 26 | int level; 27 | Clog(): mode(INT_MAX), level(INT_MAX) {} 28 | } clog; 29 | } CONF; 30 | 31 | string datetime() 32 | { 33 | time_t now; 34 | time(&now); 35 | struct tm* t=localtime(&now); 36 | char timestr[32]; 37 | strftime(timestr, sizeof(timestr), "%Y-%m-%d %H:%M:%S", t); 38 | return timestr; 39 | } 40 | 41 | #define VAR_DATA(buf, format) \ 42 | do { \ 43 | va_list arg; \ 44 | va_start(arg, format); \ 45 | (vasprintf(&buf, format, arg) < 0) && (buf = NULL); \ 46 | va_end(arg); \ 47 | } while (false) 48 | 49 | class Clog { 50 | public: 51 | Clog(string module, string subcate) { 52 | dbgcate = module + "," + "logdbg" + "," + local_ip + "," + subcate; 53 | warcate = module + "," + "logwar" + "," + local_ip + "," + subcate; 54 | errcate = module + "," + "logerr" + "," + local_ip + "," + subcate; 55 | infocate = module + "," + "loginfo" + "," + local_ip + "," + subcate; 56 | busicate = module + "," + "logbusi_%s" + "," + local_ip + "," + subcate; 57 | } 58 | 59 | void Do(const string& cate, const char *content) { 60 | int sockfd=socket(AF_INET, SOCK_DGRAM, 0); 61 | if (sockfd < 0) { 62 | cerr<<"Clog:Do() socket "<Debug("dbg msg"); 174 | log_main->Warn("war msg"); 175 | log_main->Error("err msg"); 176 | log_main->Info("info msg"); 177 | log_main->Busi("push", "busi msg"); 178 | return 0; 179 | } 180 | -------------------------------------------------------------------------------- /api/api.go: -------------------------------------------------------------------------------- 1 | package clog 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net" 7 | "reflect" 8 | "strings" 9 | 10 | "github.com/simplejia/utils" 11 | ) 12 | 13 | var ( 14 | Level int 15 | Mode int 16 | cate_dbg string 17 | cate_war string 18 | cate_err string 19 | cate_info string 20 | cate_busi string 21 | ) 22 | 23 | // 请赋值成自己的获取master addr的函数 24 | var AddrFunc = func() (string, error) { 25 | return "127.0.0.1:28702", nil 26 | } 27 | 28 | // 请赋值成自己的split content的函数 29 | var SplitFunc = func(content string) (contents []string) { 30 | for bpos, epos, seg, l := 0, 0, 65000, len(content); bpos < l; bpos += seg { 31 | epos = bpos + seg 32 | if epos > l { 33 | epos = l 34 | } 35 | contents = append(contents, content[bpos:epos]) 36 | } 37 | 38 | return 39 | } 40 | 41 | func sendAgent(tube, content string) { 42 | addr, err := AddrFunc() 43 | if err != nil { 44 | return 45 | } 46 | conn, err := net.Dial("udp", addr) 47 | if err != nil { 48 | return 49 | } 50 | defer conn.Close() 51 | 52 | for _, c := range SplitFunc(content) { 53 | out := tube + "," + c 54 | conn.Write([]byte(out)) 55 | } 56 | } 57 | 58 | func iprint(params []interface{}) { 59 | for pos, param := range params { 60 | switch param.(type) { 61 | case nil, string, []byte: 62 | continue 63 | } 64 | 65 | typ := reflect.TypeOf(param) 66 | if kind := typ.Kind(); kind <= reflect.Complex128 { 67 | continue 68 | } 69 | 70 | v := reflect.ValueOf(param) 71 | switch typ.Kind() { 72 | case reflect.Ptr, reflect.Map, reflect.Slice: 73 | if v.IsNil() { 74 | continue 75 | } 76 | } 77 | 78 | if typ.Implements(reflect.TypeOf((*error)(nil)).Elem()) || typ.Implements(reflect.TypeOf((*fmt.Stringer)(nil)).Elem()) { 79 | params[pos] = fmt.Sprintf("%v", param) 80 | continue 81 | } 82 | 83 | params[pos] = utils.Iprint(param) 84 | } 85 | } 86 | 87 | func Init(module, subcate string, level int, mode int) { 88 | if strings.Contains(module, ",") || strings.Contains(subcate, ",") { 89 | panic("clog Init error, module or subcate contains ','") 90 | } 91 | 92 | cate_dbg = strings.Join([]string{module, "logdbg", utils.LocalIp, subcate}, ",") 93 | cate_war = strings.Join([]string{module, "logwar", utils.LocalIp, subcate}, ",") 94 | cate_err = strings.Join([]string{module, "logerr", utils.LocalIp, subcate}, ",") 95 | cate_info = strings.Join([]string{module, "loginfo", utils.LocalIp, subcate}, ",") 96 | cate_busi = strings.Join([]string{module, "logbusi_%s", utils.LocalIp, subcate}, ",") 97 | 98 | Level = level 99 | Mode = mode 100 | } 101 | 102 | func Debug(format string, params ...interface{}) { 103 | if Level&1 != 0 { 104 | iprint(params) 105 | content := fmt.Sprintf(format, params...) 106 | if Mode&1 != 0 { 107 | log.Println("[DEBUG]", content) 108 | } 109 | if Mode&2 != 0 { 110 | sendAgent(cate_dbg, content) 111 | } 112 | } 113 | } 114 | 115 | func Warn(format string, params ...interface{}) { 116 | if Level&2 != 0 { 117 | iprint(params) 118 | content := fmt.Sprintf(format, params...) 119 | if Mode&1 != 0 { 120 | log.Println("[WARN]", content) 121 | } 122 | if Mode&2 != 0 { 123 | sendAgent(cate_war, content) 124 | } 125 | } 126 | } 127 | 128 | func Error(format string, params ...interface{}) { 129 | if Level&4 != 0 { 130 | iprint(params) 131 | content := fmt.Sprintf(format, params...) 132 | if Mode&1 != 0 { 133 | log.Println("[ERROR]", content) 134 | } 135 | if Mode&2 != 0 { 136 | sendAgent(cate_err, content) 137 | } 138 | } 139 | } 140 | 141 | func Info(format string, params ...interface{}) { 142 | if Level&8 != 0 { 143 | iprint(params) 144 | content := fmt.Sprintf(format, params...) 145 | if Mode&1 != 0 { 146 | log.Println("[INFO]", content) 147 | } 148 | if Mode&2 != 0 { 149 | sendAgent(cate_info, content) 150 | } 151 | } 152 | } 153 | 154 | func Busi(sub string, format string, params ...interface{}) { 155 | iprint(params) 156 | content := fmt.Sprintf(format, params...) 157 | if Mode&1 != 0 { 158 | log.Println("[BUSI]", sub, content) 159 | } 160 | if Mode&2 != 0 { 161 | sendAgent(fmt.Sprintf(cate_busi, sub), content) 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /api/api.php: -------------------------------------------------------------------------------- 1 | dbgcate = implode(',', array($module, 'logdbg', $localip, $subcate)); 16 | $this->warcate = implode(',', array($module, 'logwar', $localip, $subcate)); 17 | $this->errcate = implode(',', array($module, 'logerr', $localip, $subcate)); 18 | $this->infocate = implode(',', array($module, 'loginfo', $localip, $subcate)); 19 | $this->busicate = implode(',', array($module, 'logbusi_%s', $localip, $subcate)); 20 | $this->ip = $ip; 21 | $this->port = $port; 22 | $this->level = $level; 23 | } 24 | 25 | public function Log($cate, $content) { 26 | $sock = socket_create(AF_INET, SOCK_DGRAM, 0); 27 | if (!$sock) { 28 | $errorcode = socket_last_error(); 29 | $errormsg = socket_strerror($errorcode); 30 | fprintf(STDERR, "CLog Log() [$errorcode] $errormsg\n"); 31 | return; 32 | } 33 | 34 | $seg = 65000; 35 | for ($pos=0; $posip, $this->port); 38 | } 39 | socket_close($sock); 40 | } 41 | 42 | public function Debug($content) { 43 | if (($this->level & 1) != 0) { 44 | $this->Log($this->dbgcate, $content); 45 | } 46 | } 47 | 48 | public function Warn($content) { 49 | if (($this->level & 2) != 0) { 50 | $this->Log($this->warcate, $content); 51 | } 52 | } 53 | 54 | public function Error($content) { 55 | if (($this->level & 4) != 0) { 56 | $this->Log($this->errcate, $content); 57 | } 58 | } 59 | 60 | public function Info($content) { 61 | if (($this->level & 8) != 0) { 62 | $this->Log($this->infocate, $content); 63 | } 64 | } 65 | 66 | public function Busi($sub, $content) { 67 | $this->Log(sprintf($this->busicate, $sub), $content); 68 | } 69 | } 70 | 71 | $module = "demo"; 72 | $ip = "127.0.0.1"; 73 | $port = 28702; 74 | $level = 15; 75 | $localip = '127.0.0.1'; // TODO replace with your localip 76 | 77 | $clog = new CLog($ip, $port, $level, $localip, $module, ""); 78 | $clog->Debug("dbg msg"); 79 | $clog->Warn("war msg"); 80 | $clog->Error("err msg"); 81 | $clog->Info("info msg"); 82 | $clog->Busi("sub", "busi msg"); 83 | 84 | -------------------------------------------------------------------------------- /api/api_test.go: -------------------------------------------------------------------------------- 1 | package clog 2 | 3 | import "testing" 4 | 5 | func TestClog(t *testing.T) { 6 | Init("demo", "", 15, 3) 7 | Debug("debug msg") 8 | Warn("warn msg") 9 | Error("error msg") 10 | Info("info msg") 11 | Busi("sub", "busi msg") 12 | } 13 | -------------------------------------------------------------------------------- /conf/conf.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | "sync/atomic" 14 | "time" 15 | 16 | "github.com/simplejia/clog/api" 17 | "github.com/simplejia/utils" 18 | ) 19 | 20 | type ProcAction struct { 21 | Handler string 22 | Params map[string]interface{} 23 | } 24 | 25 | type Conf struct { 26 | Port int 27 | Tpl map[string]json.RawMessage 28 | Procs map[string]interface{} 29 | Clog *struct { 30 | Name string 31 | Mode int 32 | Level int 33 | } 34 | } 35 | 36 | func Get() *Conf { 37 | return C.Load().(*Conf) 38 | } 39 | 40 | func Set(c *Conf) { 41 | C.Store(c) 42 | } 43 | 44 | var ( 45 | Env string 46 | C atomic.Value 47 | ) 48 | 49 | func replaceTpl(src string, tpl map[string]json.RawMessage) (dst []byte) { 50 | oldnew := []string{} 51 | for k, v := range tpl { 52 | oldnew = append(oldnew, "\"$"+k+"\"", string(v)) 53 | } 54 | 55 | r := strings.NewReplacer(oldnew...) 56 | dst = []byte(r.Replace(src)) 57 | return 58 | } 59 | 60 | func reloadConf(content []byte) { 61 | lastbody := content 62 | 63 | for { 64 | time.Sleep(time.Second * 3) 65 | 66 | body, err := getcontents() 67 | if err != nil || len(body) == 0 { 68 | clog.Error("getcontents err: %v, body: %s", err, body) 69 | continue 70 | } 71 | 72 | if bytes.Compare(lastbody, body) == 0 { 73 | continue 74 | } 75 | 76 | if err := parse(body); err != nil { 77 | clog.Error("parse err: %v, body: %s", err, body) 78 | continue 79 | } 80 | 81 | if err := savecontents(body); err != nil { 82 | clog.Error("savecontents err: %v, body: %s", err, body) 83 | continue 84 | } 85 | 86 | lastbody = body 87 | } 88 | } 89 | 90 | func getcontents() (fcontent []byte, err error) { 91 | dir := "conf" 92 | for i := 0; i < 3; i++ { 93 | if info, err := os.Stat(dir); err == nil && info.IsDir() { 94 | break 95 | } 96 | dir = filepath.Join("..", dir) 97 | } 98 | fcontent, err = ioutil.ReadFile(filepath.Join(dir, "conf.json")) 99 | if err != nil { 100 | return 101 | } 102 | return 103 | } 104 | 105 | func savecontents(fcontent []byte) (err error) { 106 | dir := "conf" 107 | for i := 0; i < 3; i++ { 108 | if info, err := os.Stat(dir); err == nil && info.IsDir() { 109 | break 110 | } 111 | dir = filepath.Join("..", dir) 112 | } 113 | err = ioutil.WriteFile(filepath.Join(dir, "conf.json"), fcontent, 0644) 114 | if err != nil { 115 | return 116 | } 117 | return 118 | } 119 | 120 | func parse(fcontent []byte) (err error) { 121 | fcontent = utils.RemoveAnnotation(fcontent) 122 | 123 | var envs map[string]*Conf 124 | if err = json.Unmarshal(fcontent, &envs); err != nil { 125 | return 126 | } 127 | 128 | c := envs[Env] 129 | if c == nil { 130 | return fmt.Errorf("env not right: %s", Env) 131 | } 132 | 133 | cs, _ := json.Marshal(c) 134 | cs = replaceTpl(string(cs), c.Tpl) 135 | if err := json.Unmarshal(cs, &c); err != nil { 136 | return fmt.Errorf("conf.json wrong format:", err) 137 | } 138 | 139 | for k, proc := range c.Procs { 140 | var new_proc []*ProcAction 141 | _proc, _ := json.Marshal(proc) 142 | if err := json.Unmarshal(_proc, &new_proc); err != nil { 143 | return fmt.Errorf("conf.json wrong format(procs):", err) 144 | } 145 | c.Procs[k] = new_proc 146 | } 147 | 148 | Set(c) 149 | 150 | log.Printf("Env: %s\nC: %s\n", Env, utils.Iprint(c)) 151 | return 152 | } 153 | 154 | func init() { 155 | flag.StringVar(&Env, "env", "prod", "set env") 156 | flag.Parse() 157 | 158 | fcontent, err := getcontents() 159 | if err != nil { 160 | log.Printf("get conf file contents error: %v\n", err) 161 | os.Exit(-1) 162 | } 163 | 164 | err = parse(fcontent) 165 | if err != nil { 166 | log.Printf("parse conf file error: %v\n", err) 167 | os.Exit(-1) 168 | } 169 | 170 | go reloadConf(fcontent) 171 | } 172 | -------------------------------------------------------------------------------- /conf/conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "prod": { 3 | "port": 28702, 4 | "tpl": { // 模板定义 5 | "logerr_demo": [ 6 | {"handler": "filehandler"}, 7 | { 8 | "handler": "alarmhandler", 9 | "params": { 10 | "sender": "demo", // 发送者id 11 | "receivers": ["simplejia"], // 接收者 12 | "excludes": [], // strstr，满足条件者不报警 13 | "": null // placeholder 14 | } 15 | } 16 | ], 17 | "logwar": [ 18 | {"handler": "filehandler"} 19 | ], 20 | "logdbg": [ 21 | {"handler": "filehandler"} 22 | ], 23 | "loginfo": [ 24 | {"handler": "filehandler"} 25 | ] 26 | }, 27 | "procs": { 28 | "clog/logerr": "$logerr_demo", 29 | "clog/logdbg": "$logdbg", 30 | "clog/logwar": "$logwar", 31 | "clog/loginfo": "$loginfo", 32 | 33 | "demo/logerr": "$logerr_demo", 34 | "demo/logdbg": "$logdbg", 35 | "demo/logwar": "$logwar", 36 | "demo/loginfo": "$loginfo", 37 | "demo/logbusi_sub": [ 38 | {"handler": "filehandler"} 39 | ], 40 | 41 | "": null // placeholder 42 | }, 43 | "clog": { 44 | "name": "clog", 45 | "mode": 3, // 0: none, 1: localfile, 2: clog (数字代表bit位) 46 | "level": 14 // 0: none, 1: debug, 2: warn 4: error 8: info (数字代表bit位) 47 | }, 48 | 49 | "": null // placeholder 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // 数据收集服务. 2 | // author: simplejia 3 | // date: 2014/12/01 4 | package main 5 | 6 | import ( 7 | "encoding/json" 8 | "fmt" 9 | "io/ioutil" 10 | "log" 11 | "net" 12 | "net/http" 13 | "strings" 14 | 15 | "github.com/simplejia/clog/api" 16 | "github.com/simplejia/clog/conf" 17 | "github.com/simplejia/clog/procs" 18 | "github.com/simplejia/lc" 19 | ) 20 | 21 | type s struct { 22 | cate string 23 | subcate string 24 | body string 25 | } 26 | 27 | var tubes = make(map[string]chan *s) 28 | 29 | func init() { 30 | lc.Init(1e5) 31 | 32 | clog.AddrFunc = func() (string, error) { 33 | return fmt.Sprintf("127.0.0.1:%d", conf.Get().Port), nil 34 | } 35 | c := conf.Get() 36 | clog.Init(c.Clog.Name, "", c.Clog.Level, c.Clog.Mode) 37 | } 38 | 39 | func main() { 40 | log.Println("main()") 41 | 42 | go udp() 43 | go ws() 44 | select {} 45 | } 46 | 47 | func hget(w http.ResponseWriter, r *http.Request) { 48 | fun := "main.hget" 49 | 50 | body, err := ioutil.ReadAll(r.Body) 51 | if err != nil { 52 | log.Printf("%s ReadAll err: %v\n", err) 53 | w.WriteHeader(http.StatusInternalServerError) 54 | return 55 | } 56 | 57 | var p *struct { 58 | Cate string 59 | Subcate string 60 | Body string 61 | } 62 | err = json.Unmarshal(body, &p) 63 | if err != nil || p == nil { 64 | log.Printf("%s Unmarshal err: %v, req: %s\n", fun, err, body) 65 | w.WriteHeader(http.StatusBadRequest) 66 | return 67 | } 68 | 69 | subcate := p.Subcate 70 | if subcate == "" { 71 | host, _, _ := net.SplitHostPort(r.RemoteAddr) 72 | subcate = host 73 | } 74 | 75 | add(p.Cate, subcate, p.Body) 76 | 77 | w.WriteHeader(http.StatusOK) 78 | return 79 | } 80 | 81 | func ws() { 82 | http.HandleFunc("/clog/api", hget) 83 | 84 | addr := fmt.Sprintf(":%d", conf.Get().Port) 85 | err := http.ListenAndServe(addr, nil) 86 | if err != nil { 87 | log.Fatalln("net.ListenAndServe error:", err) 88 | } 89 | } 90 | 91 | func udp() { 92 | udpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf(":%d", conf.Get().Port)) 93 | if err != nil { 94 | log.Fatalln("net.ResolveUDPAddr error:", err) 95 | } 96 | 97 | conn, err := net.ListenUDP("udp", udpAddr) 98 | if err != nil { 99 | log.Fatalln("net.ListenUDP error:", err) 100 | } 101 | defer conn.Close() 102 | 103 | if err := conn.SetReadBuffer(50 * 1024 * 1024); err != nil { 104 | log.Fatalln("conn.SetReadBuffer error:", err) 105 | } 106 | 107 | request := make([]byte, 1024*64) 108 | for { 109 | readLen, raddr, err := conn.ReadFrom(request) 110 | if err != nil || readLen <= 0 { 111 | continue 112 | } 113 | 114 | ss := strings.SplitN(string(request[:readLen]), ",", 5) 115 | if len(ss) != 5 { 116 | continue 117 | } 118 | cate, subcate := "", "" 119 | cate = strings.Join(ss[:2], "/") // module+level 120 | if str := ss[2]; str == "" { // localip 121 | subcate, _, _ = net.SplitHostPort(raddr.String()) 122 | } else { 123 | subcate = str 124 | } 125 | if str := ss[3]; str != "" { // subcate 126 | subcate += "+" + str 127 | } 128 | body := ss[4] 129 | 130 | add(cate, subcate, body) 131 | } 132 | } 133 | 134 | func add(cate, subcate, body string) { 135 | k := cate + "," + subcate 136 | tube, ok := tubes[k] 137 | if !ok { 138 | tube = make(chan *s, 1e5) 139 | tubes[k] = tube 140 | go proc(tube) 141 | } 142 | 143 | select { 144 | case tube <- &s{ 145 | cate: cate, 146 | subcate: subcate, 147 | body: body, 148 | }: 149 | default: 150 | log.Println("add data full") 151 | } 152 | } 153 | 154 | func proc(tube chan *s) { 155 | for d := range tube { 156 | procs.Doit(d.cate, d.subcate, d.body) 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /procs/alarm_handler.go: -------------------------------------------------------------------------------- 1 | package procs 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "os" 8 | "strings" 9 | 10 | "github.com/simplejia/lc" 11 | 12 | "time" 13 | ) 14 | 15 | // 请赋值成自己的报警处理函数 16 | var AlarmFunc = func(sender string, receivers []string, text string) { 17 | params := map[string]interface{}{ 18 | "Sender": sender, 19 | "Receivers": receivers, 20 | "Text": text, 21 | } 22 | json.NewEncoder(os.Stdout).Encode(params) 23 | } 24 | 25 | type AlarmStat struct { 26 | LastTime time.Time 27 | LastTexts []string 28 | } 29 | 30 | type AlarmParam struct { 31 | Sender string 32 | Receivers []string 33 | Excludes []string 34 | } 35 | 36 | func AlarmSplitWord(body string) (m map[string]bool) { 37 | m = map[string]bool{} 38 | 39 | for _, word := range strings.FieldsFunc(body, func(r rune) bool { 40 | switch r { 41 | case ' ', ',', ':', '{', '}', '"', '&': 42 | return true 43 | } 44 | return false 45 | }) { 46 | m[word] = true 47 | } 48 | 49 | return 50 | } 51 | 52 | func AlarmIsSimilar(src, dst string) bool { 53 | if src == dst { 54 | return true 55 | } 56 | 57 | shortM, longM := AlarmSplitWord(src), AlarmSplitWord(dst) 58 | if len(shortM) > len(longM) { 59 | shortM, longM = longM, shortM 60 | } 61 | 62 | if float64(len(shortM))/float64(len(longM)) < 0.8 { 63 | return false 64 | } 65 | 66 | l := 0 67 | for word := range shortM { 68 | if longM[word] { 69 | l++ 70 | } 71 | } 72 | 73 | if l == 0 { 74 | return false 75 | } 76 | 77 | if float64(l)/float64(len(shortM)) > 0.8 { 78 | return true 79 | } 80 | 81 | return false 82 | } 83 | 84 | func AlarmHandler(cate, subcate, body string, params map[string]interface{}) { 85 | var alarmParam *AlarmParam 86 | bs, _ := json.Marshal(params) 87 | json.Unmarshal(bs, &alarmParam) 88 | if alarmParam == nil { 89 | log.Printf("AlarmHandler() params not right: %v\n", params) 90 | return 91 | } 92 | 93 | for _, exclude := range alarmParam.Excludes { 94 | if strings.Contains(body, exclude) { 95 | return 96 | } 97 | } 98 | 99 | tube := cate + "|" + subcate 100 | var alarmstat *AlarmStat 101 | if alarmstatLc, ok := lc.Get(tube); ok && alarmstatLc != nil { 102 | alarmstat = alarmstatLc.(*AlarmStat) 103 | } else { 104 | alarmstat = &AlarmStat{} 105 | lc.Set(tube, alarmstat, time.Hour) 106 | } 107 | 108 | diff := time.Since(alarmstat.LastTime) 109 | if diff < time.Second*30 { 110 | return 111 | } 112 | 113 | if diff < time.Minute*5 { 114 | for _, lastText := range alarmstat.LastTexts { 115 | if AlarmIsSimilar(lastText, body) { 116 | return 117 | } 118 | } 119 | } 120 | 121 | alarmstat.LastTime = time.Now() 122 | alarmstat.LastTexts = append(alarmstat.LastTexts, body) 123 | if curNum, maxNum := len(alarmstat.LastTexts), 5; curNum > maxNum { 124 | alarmstat.LastTexts = alarmstat.LastTexts[curNum-maxNum:] 125 | } 126 | 127 | AlarmFunc(alarmParam.Sender, alarmParam.Receivers, fmt.Sprintf("%s:%s", tube, body)) 128 | return 129 | } 130 | 131 | func init() { 132 | RegisterHandler("alarm_handler", AlarmHandler) 133 | } 134 | -------------------------------------------------------------------------------- /procs/file_handler.go: -------------------------------------------------------------------------------- 1 | package procs 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "os" 8 | "path" 9 | "strconv" 10 | "strings" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | const ( 16 | ROOT_DIR = "logs" 17 | ) 18 | 19 | type FileParam struct { 20 | Excludes []string 21 | } 22 | 23 | var ( 24 | fileMutex sync.RWMutex 25 | hourUses = map[string]int64{} 26 | logfps = map[string]*os.File{} 27 | loggers = map[string]*log.Logger{} 28 | ) 29 | 30 | func FileHandler(cate, subcate, body string, params map[string]interface{}) { 31 | var fileParam *FileParam 32 | bs, _ := json.Marshal(params) 33 | json.Unmarshal(bs, &fileParam) 34 | if fileParam != nil { 35 | for _, exclude := range fileParam.Excludes { 36 | if strings.Contains(body, exclude) { 37 | return 38 | } 39 | } 40 | } 41 | 42 | key := cate + "," + subcate 43 | 44 | fileMutex.RLock() 45 | logger := loggers[key] 46 | hourUse := hourUses[key] 47 | 48 | now := time.Now() 49 | if now.Unix() >= hourUse { 50 | fileMutex.RUnlock() 51 | fileMutex.Lock() 52 | 53 | logfp := logfps[key] 54 | if logfp != nil { 55 | logfp.Close() 56 | delete(logfps, key) 57 | } 58 | 59 | nx := now.Unix() + 3600 60 | hourUse = time.Unix(nx-nx%3600, 0).Unix() 61 | hourUses[key] = hourUse 62 | 63 | dir := path.Join(ROOT_DIR, cate, strconv.Itoa(now.Day())) 64 | if _, err := os.Stat(dir); os.IsNotExist(err) { 65 | err := os.MkdirAll(dir, 0755) 66 | if err != nil { 67 | log.Printf("FileHandler() mkdir error %v\n", err) 68 | fileMutex.Unlock() 69 | return 70 | } 71 | } 72 | 73 | filename := fmt.Sprintf("log%s_%d", subcate, now.Hour()) 74 | logfp, err := os.OpenFile( 75 | path.Join(dir, filename), 76 | os.O_CREATE|os.O_APPEND|os.O_WRONLY, 77 | 0644, 78 | ) 79 | if err != nil { 80 | log.Printf("FileHandler() openfile error %v\n", err) 81 | fileMutex.Unlock() 82 | return 83 | } 84 | 85 | logger = log.New(logfp, "", log.Ldate|log.Ltime|log.Lmicroseconds) 86 | logfps[key] = logfp 87 | loggers[key] = logger 88 | fileMutex.Unlock() 89 | fileMutex.RLock() 90 | } 91 | 92 | logger.Println(body) 93 | fileMutex.RUnlock() 94 | return 95 | } 96 | 97 | func init() { 98 | RegisterHandler("file_handler", FileHandler) 99 | } 100 | -------------------------------------------------------------------------------- /procs/tubemap.go: -------------------------------------------------------------------------------- 1 | package procs 2 | 3 | import ( 4 | "log" 5 | "runtime/debug" 6 | 7 | "github.com/simplejia/clog/conf" 8 | ) 9 | 10 | var Handlers = map[string]HandlerFunc{} 11 | 12 | type HandlerFunc func(cate, subcate, body string, params map[string]interface{}) 13 | 14 | func Doit(cate, subcate, body string) { 15 | defer func() { 16 | if err := recover(); err != nil { 17 | log.Printf("Doit() recover err: %v, cate: %s, subcate: %s, body: %s, stack: %s\n", err, cate, subcate, body, debug.Stack()) 18 | } 19 | }() 20 | 21 | if handlers, ok := conf.Get().Procs[cate]; ok { 22 | for _, p := range handlers.([]*conf.ProcAction) { 23 | if p.Handler == "" { 24 | continue 25 | } 26 | if handler, ok := Handlers[p.Handler]; ok { 27 | handler(cate, subcate, body, p.Params) 28 | } else { 29 | log.Printf("Doit() p.Handler not right: %s\n", p.Handler) 30 | } 31 | } 32 | } else { 33 | // default 34 | FileHandler("default__"+cate, subcate, body, nil) 35 | } 36 | return 37 | } 38 | 39 | func RegisterHandler(name string, handler HandlerFunc) { 40 | Handlers[name] = handler 41 | } 42 | --------------------------------------------------------------------------------