├── .gitignore ├── LICENSE ├── README.md ├── level.go ├── level_test.go ├── log.go └── log_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 IBBD R&D Center 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 异步日志库 2 | 3 | ## Install 4 | 5 | ```sh 6 | go get -u github.com/ibbd-dev/go-async-log 7 | ``` 8 | 9 | ## 实现的功能及说明 10 | 11 | - 多个日志文件写入：例如错误信息一个文件，测试信息一个文件等 12 | - 日志自动切割：前期支持按小时或者天切割 13 | - 支持批量写入：最小单位为100毫秒，不同的文件可以设置不同的写入频率（周期性写入，程序挂掉的时候最多可能会丢一个周期的数据，重要数据不能采用该方式 14 | - 同时支持实时写入文件，使用文件系统缓存（只要系统不挂，就不会有问题） 15 | - 错误等级实现 16 | - 可以写入json数据 17 | - 时间格式采用`RFC3339`，格式如`2006-01-02T15:04:05Z07:00` 18 | - 支持按概率写log 19 | 20 | ## 配置项 21 | 22 | - 文件名 23 | - 日志记录的等级 24 | - 自动切割周期：默认按小时 25 | - 批量写入周期：默认每秒写入一次 26 | - 异常等级： 27 | - 是否需要Flags：默认需要 28 | 29 | ## Example 30 | 31 | 普通写入日志文件 32 | 33 | ```go 34 | lf := asyncLog.NewLogFile("/tmp/test.log") 35 | 36 | // 设置按天切割文件，如果默认则是按小时 37 | lf.SetRotate(asyncLog.RotateDate) 38 | 39 | lf.SetProbability(0.5) // 设置写log的概率，默认全部都写入 40 | 41 | _ = lf.Write("lf: hello world") 42 | 43 | // 注意：因为是每秒写入一次，所以这里需要暂停一下 44 | time.Sleep(time.Second * 2) 45 | 46 | ``` 47 | 48 | 写入错误等级文件 49 | 50 | ```go 51 | infoFile := asyncLog.NewLevelLog("/tmp/test-info.log", asyncLog.LevelInfo) // 只有Info级别或者以上级别的日志才会被记录 52 | infoFile.SetProbability(0.5) // 设置写log的概率，默认全部都写入 53 | infoFile.Debug("hello world") // 该日志不会写入文件 54 | infoFile.Info("hello world") 55 | infoFile.Error("hello world") 56 | 57 | // 需要改变日志写入等级时，例如测试阶段 58 | infoFile.SetLevel(asyncLog.LevelDebug) 59 | 60 | time.Sleep(time.Second * 2) 61 | ``` 62 | 63 | ## 性能数据 64 | 65 | 不缓存内容的时候，如果不对文件句柄进行缓存重用，性能是比较低的，如下：（这是旧版本） 66 | 67 | ```sh 68 | # go test -bench=".*" 69 | BenchmarkWrite-4 3000000 444 ns/op 70 | BenchmarkWriteNoCache-4 300000 4400 ns/op 71 | ``` 72 | 73 | 对句柄进行缓存重用之后，性能如下： 74 | 75 | ```sh 76 | BenchmarkWrite-4 3000000 570 ns/op 77 | BenchmarkWriteNoCache-4 1000000 2304 ns/op 78 | ``` 79 | 80 | 结论：对句柄进行缓存，是能大大提升效率的。 81 | 82 | ------- 83 | 84 | 性能比`github.com/ibbd-dev/go-tools/logfile`至少提升一个数量级 85 | 86 | -------------------------------------------------------------------------------- /level.go: -------------------------------------------------------------------------------- 1 | package asyncLog 2 | 3 | import ( 4 | "math/rand" 5 | "fmt" 6 | ) 7 | 8 | // 日志优先级 9 | type Priority int 10 | 11 | const ( 12 | LevelAll Priority = iota 13 | LevelDebug 14 | LevelInfo 15 | LevelWarn 16 | LevelError 17 | LevelFatal 18 | LevelOff 19 | ) 20 | 21 | var ( 22 | // 日志等级 23 | levelTitle = map[Priority]string{ 24 | LevelDebug: "[DEBUG]", 25 | LevelInfo: "[INFO]", 26 | LevelWarn: "[WARN]", 27 | LevelError: "[ERROR]", 28 | LevelFatal: "[FATAL]", 29 | } 30 | ) 31 | 32 | // NewLevelLog 写入等级日志 33 | // 级别高于logLevel才会被写入 34 | func NewLevelLog(filename string, logLevel Priority) *LogFile { 35 | lf := NewLogFile(filename) 36 | lf.level = logLevel 37 | 38 | return lf 39 | } 40 | 41 | func (lf *LogFile) SetLevel(logLevel Priority) { 42 | lf.level = logLevel 43 | } 44 | 45 | func (lf *LogFile) Debug(format string, a ...interface{}) error { 46 | return lf.writeLevelMsg(LevelDebug, format, a...) 47 | } 48 | 49 | func (lf *LogFile) Info(format string, a ...interface{}) error { 50 | return lf.writeLevelMsg(LevelInfo, format, a...) 51 | } 52 | 53 | func (lf *LogFile) Warn(format string, a ...interface{}) error { 54 | return lf.writeLevelMsg(LevelWarn, format, a...) 55 | } 56 | 57 | func (lf *LogFile) Error(format string, a ...interface{}) error { 58 | return lf.writeLevelMsg(LevelError, format, a...) 59 | } 60 | 61 | func (lf *LogFile) Fatal(format string, a ...interface{}) error { 62 | return lf.writeLevelMsg(LevelFatal, format, a...) 63 | } 64 | 65 | func (lf *LogFile) writeLevelMsg(level Priority, format string, a ...interface{}) error { 66 | if lf.probability < 1.0 && rand.Float32() > lf.probability { 67 | // 按照概率写入 68 | return nil 69 | } 70 | 71 | if level >= lf.level { 72 | msg := fmt.Sprintf(format, a...) 73 | return lf.Write(levelTitle[level] + " " + msg) 74 | } 75 | 76 | return nil 77 | } 78 | 79 | -------------------------------------------------------------------------------- /level_test.go: -------------------------------------------------------------------------------- 1 | package asyncLog 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestLevelWrite(t *testing.T) { 9 | infoFile := NewLevelLog("/tmp/test-info.log", LevelInfo) 10 | infoFile.Debug("hello %s", "world") 11 | infoFile.Info("hello %d", 123) 12 | infoFile.Error("hello %x", &t) 13 | 14 | time.Sleep(time.Second * 2) 15 | } 16 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package asyncLog 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "math/rand" 7 | "os" 8 | "strings" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | type asyncLogType struct { 14 | files map[string]*LogFile // 下标是文件名 15 | 16 | // 避免并发new对象 17 | sync.RWMutex 18 | } 19 | 20 | type LogFile struct { 21 | filename string // 原始文件名（包含完整目录） 22 | flag int // 默认为log.LstdFlags 23 | 24 | // 同步设置 25 | sync struct { 26 | duration time.Duration // 同步数据到文件的周期，默认为1秒 27 | beginTime time.Time // 开始同步的时间，判断同步的耗时 28 | status syncStatus // 同步状态 29 | } 30 | 31 | // 日志的等级 32 | level Priority 33 | 34 | // 缓存 35 | cache struct { 36 | use bool // 是否使用缓存 37 | data []string // 缓存数据 38 | 39 | // 写cache时的互斥锁 40 | mutex sync.Mutex 41 | } 42 | 43 | // 文件切割 44 | logRotate struct { 45 | rotate LogRotate // 默认按小时切割 46 | file *os.File // 文件操作对象 47 | suffix string // 切割后的文件名后缀 48 | mutex sync.Mutex // 文件名锁 49 | } 50 | 51 | // 日志写入概率 52 | probability float32 53 | } 54 | 55 | // log同步的状态 56 | type syncStatus int 57 | 58 | const ( 59 | statusInit syncStatus = iota // 初始状态 60 | statusDoing // 同步中 61 | statusDone // 同步已经完成 62 | ) 63 | 64 | // 日志切割的方式 65 | type LogRotate int 66 | 67 | const ( 68 | RotateHour LogRotate = iota // 按小时切割 69 | RotateDate // 按日期切割 70 | ) 71 | 72 | const ( 73 | // 写日志时前缀的时间格式 74 | // "2006-01-02T15:04:05Z07:00" 75 | logTimeFormat string = time.RFC3339 76 | 77 | // 文件写入mode 78 | fileOpenMode = 0666 79 | 80 | // 文件Flag 81 | fileFlag = os.O_WRONLY | os.O_CREATE | os.O_APPEND 82 | 83 | // 换行符 84 | newlineStr = "\n" 85 | newlineChar = '\n' 86 | 87 | // 缓存切片的初始容量 88 | cacheInitCap = 64 89 | ) 90 | 91 | // 是否需要Flag信息 92 | const ( 93 | NoFlag = 0 94 | StdFlag = log.LstdFlags 95 | ) 96 | 97 | // 异步日志变量 98 | var asyncLog *asyncLogType 99 | 100 | var nowFunc = time.Now 101 | 102 | func init() { 103 | asyncLog = &asyncLogType{ 104 | files: make(map[string]*LogFile), 105 | } 106 | 107 | timer := time.NewTicker(time.Millisecond * 100) 108 | //timer := time.NewTicker(time.Second) 109 | go func() { 110 | for { 111 | select { 112 | case <-timer.C: 113 | //now := nowFunc() 114 | asyncLog.RLock() 115 | for _, file := range asyncLog.files { 116 | if file.sync.status != statusDoing { 117 | go file.flush() 118 | } 119 | } 120 | asyncLog.RUnlock() 121 | } 122 | } 123 | 124 | }() 125 | } 126 | 127 | func NewLogFile(filename string) *LogFile { 128 | asyncLog.Lock() 129 | defer asyncLog.Unlock() 130 | 131 | if lf, ok := asyncLog.files[filename]; ok { 132 | return lf 133 | } 134 | 135 | lf := &LogFile{ 136 | filename: filename, 137 | flag: StdFlag, 138 | } 139 | 140 | asyncLog.files[filename] = lf 141 | 142 | // 默认按小时切割文件 143 | lf.logRotate.rotate = RotateHour 144 | 145 | // 默认开启缓存 146 | lf.cache.use = true 147 | 148 | // 日志写入概率，默认为1.1, 就是全部写入 149 | lf.probability = 1.1 150 | 151 | // TODO 同步的时间周期，缓存开启才有效 152 | lf.sync.duration = time.Second 153 | return lf 154 | } 155 | 156 | func (lf *LogFile) SetFlags(flag int) { 157 | lf.flag = flag 158 | } 159 | 160 | func (lf *LogFile) SetRotate(rotate LogRotate) { 161 | lf.logRotate.rotate = rotate 162 | } 163 | 164 | func (lf *LogFile) SetUseCache(useCache bool) { 165 | lf.cache.use = useCache 166 | } 167 | 168 | func (lf *LogFile) SetProbability(probability float32) { 169 | lf.probability = probability 170 | } 171 | 172 | // Write 写缓存 173 | func (lf *LogFile) Write(msg string) error { 174 | if lf.flag == StdFlag { 175 | msg = nowFunc().Format(logTimeFormat) + " " + msg + newlineStr 176 | } else { 177 | msg = msg + newlineStr 178 | } 179 | 180 | if lf.cache.use { 181 | lf.appendCache(msg) 182 | return nil 183 | } 184 | 185 | return lf.directWrite([]byte(msg)) 186 | } 187 | 188 | // WriteJson 写入json数据 189 | func (lf *LogFile) WriteJson(data interface{}) error { 190 | if lf.probability < 1.0 && rand.Float32() > lf.probability { 191 | // 按照概率写入 192 | return nil 193 | } 194 | 195 | bts, err := json.Marshal(data) 196 | if err != nil { 197 | return err 198 | } 199 | bts = append(bts, newlineChar) 200 | 201 | if lf.cache.use { 202 | lf.appendCache(string(bts)) 203 | return nil 204 | } 205 | 206 | return lf.directWrite(bts) 207 | } 208 | 209 | //*********************** 以下是私有函数 ************************************ 210 | 211 | func (lf *LogFile) appendCache(msg string) { 212 | lf.cache.mutex.Lock() 213 | lf.cache.data = append(lf.cache.data, msg) 214 | lf.cache.mutex.Unlock() 215 | } 216 | 217 | // 同步缓存到文件中 218 | func (lf *LogFile) flush() error { 219 | lf.sync.status = statusDoing 220 | defer func() { 221 | lf.sync.status = statusDone 222 | }() 223 | 224 | // 写入log文件 225 | file, err := lf.openFileNoCache() 226 | if err != nil { 227 | panic(err) 228 | } 229 | defer file.Close() 230 | 231 | // 获取缓存数据 232 | lf.cache.mutex.Lock() 233 | cache := lf.cache.data 234 | lf.cache.data = make([]string, 0, cacheInitCap) 235 | lf.cache.mutex.Unlock() 236 | 237 | if len(cache) == 0 { 238 | return nil 239 | } 240 | 241 | _, err = file.WriteString(strings.Join(cache, "")) 242 | if err != nil { 243 | // 重试 244 | _, err = file.WriteString(strings.Join(cache, "")) 245 | if err != nil { 246 | panic(err) 247 | } 248 | } 249 | 250 | return nil 251 | } 252 | 253 | // 获取文件名的后缀 254 | func (lf *LogFile) getFilenameSuffix() string { 255 | if lf.logRotate.rotate == RotateDate { 256 | return nowFunc().Format("20060102") 257 | } 258 | return nowFunc().Format("2006010215") 259 | } 260 | 261 | // 直接写入日志文件 262 | func (lf *LogFile) directWrite(msg []byte) error { 263 | file, err := lf.openFile() 264 | //file, err := lf.openFileNoCache() 265 | if err != nil { 266 | panic(err) 267 | } 268 | //defer file.Close() 269 | 270 | lf.logRotate.mutex.Lock() 271 | _, err = file.Write(msg) 272 | lf.logRotate.mutex.Unlock() 273 | 274 | return err 275 | } 276 | 277 | // 打开日志文件 278 | func (lf *LogFile) openFile() (*os.File, error) { 279 | suffix := lf.getFilenameSuffix() 280 | 281 | lf.logRotate.mutex.Lock() 282 | defer lf.logRotate.mutex.Unlock() 283 | 284 | if suffix == lf.logRotate.suffix { 285 | return lf.logRotate.file, nil 286 | } 287 | 288 | logFilename := lf.filename + "." + suffix 289 | file, err := os.OpenFile(logFilename, fileFlag, fileOpenMode) 290 | if err != nil { 291 | // 重试 292 | file, err = os.OpenFile(logFilename, fileFlag, fileOpenMode) 293 | if err != nil { 294 | return file, err 295 | } 296 | } 297 | 298 | // 关闭旧的文件 299 | if lf.logRotate.file != nil { 300 | lf.logRotate.file.Close() 301 | } 302 | 303 | lf.logRotate.file = file 304 | lf.logRotate.suffix = suffix 305 | return file, nil 306 | } 307 | 308 | // 打开日志文件(不缓存句柄) 309 | func (lf *LogFile) openFileNoCache() (*os.File, error) { 310 | logFilename := lf.filename + "." + lf.getFilenameSuffix() 311 | 312 | lf.logRotate.mutex.Lock() 313 | defer lf.logRotate.mutex.Unlock() 314 | 315 | file, err := os.OpenFile(logFilename, fileFlag, fileOpenMode) 316 | if err != nil { 317 | // 重试 318 | file, err = os.OpenFile(logFilename, fileFlag, fileOpenMode) 319 | if err != nil { 320 | return file, err 321 | } 322 | } 323 | 324 | return file, nil 325 | } 326 | -------------------------------------------------------------------------------- /log_test.go: -------------------------------------------------------------------------------- 1 | package asyncLog 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestNewLogFile(t *testing.T) { 9 | n := len(asyncLog.files) 10 | 11 | lf1 := NewLogFile("/tmp/test1.log") 12 | if len(asyncLog.files) != n+1 { 13 | t.Fatalf("NewLogFile num != 1") 14 | } 15 | 16 | _ = NewLogFile("/tmp/test1.log") 17 | if len(asyncLog.files) != n+1 { 18 | t.Fatalf("NewLogFile num != 1 repeat") 19 | } 20 | 21 | lf2 := NewLogFile("/tmp/test2.log") 22 | if len(asyncLog.files) != n+2 { 23 | t.Fatalf("NewLogFile num != 2") 24 | } 25 | lf2.SetRotate(RotateDate) 26 | 27 | _ = lf1.Write("lf1: hello world1") 28 | _ = lf1.Write("lf1: hello world2") 29 | _ = lf2.Write("lf2: hello world1") 30 | _ = lf2.Write("lf2: hello world2") 31 | _ = lf1.Write("lf1: hello world3") 32 | _ = lf1.Write("lf1: hello world4") 33 | 34 | time.Sleep(time.Second * 2) 35 | 36 | _ = lf1.Write("lf1: ---hello world1") 37 | _ = lf1.Write("lf1: ---hello world2") 38 | _ = lf2.Write("lf2: ---hello world1") 39 | _ = lf2.Write("lf2: ---hello world2") 40 | _ = lf1.Write("lf1: ---hello world3") 41 | _ = lf1.Write("lf1: ---hello world4") 42 | 43 | time.Sleep(time.Second * 2) 44 | } 45 | 46 | func TestJson(t *testing.T) { 47 | lf1 := NewLogFile("/tmp/test-json.log") 48 | lf1.SetFlags(NoFlag) 49 | lf1.SetUseCache(false) 50 | 51 | var hello = struct { 52 | Hello string 53 | World int 54 | }{ 55 | Hello: "test", 56 | World: 12, 57 | } 58 | _ = lf1.WriteJson(hello) 59 | _ = lf1.WriteJson(hello) 60 | _ = lf1.WriteJson(hello) 61 | 62 | time.Sleep(time.Second * 2) 63 | } 64 | 65 | func TestProbability(t *testing.T) { 66 | lf := NewLogFile("/tmp/test-probability.log") 67 | lf.SetProbability(0.5) 68 | for i := 0; i < 20; i++ { 69 | lf.Write("probability") 70 | } 71 | 72 | time.Sleep(time.Second) 73 | } 74 | 75 | func BenchmarkWrite(b *testing.B) { 76 | lf := NewLogFile("/tmp/bench-test.log") 77 | b.RunParallel(func(pb *testing.PB) { 78 | for pb.Next() { 79 | _ = lf.Write("hello world") 80 | } 81 | }) 82 | } 83 | 84 | func BenchmarkWriteNoCache(b *testing.B) { 85 | lf := NewLogFile("/tmp/bench-nocache-test.log") 86 | lf.SetUseCache(false) 87 | b.RunParallel(func(pb *testing.PB) { 88 | for pb.Next() { 89 | _ = lf.Write("hello world") 90 | } 91 | }) 92 | } 93 | --------------------------------------------------------------------------------