├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── doc.go ├── example └── main.go ├── go.mod ├── go.sum ├── line.go ├── line_test.go ├── options.go ├── rate_limiter.go ├── rate_limiter_channel.go ├── rate_limiter_channel_test.go ├── seeker_test.go ├── tailor.go └── tailor_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.10.x 5 | - 1.11.x 6 | - 1.12.x 7 | 8 | matrix: 9 | fast_finish: true 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Yegor Myskin 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Tailor, the library for tailing logs under logrotate 2 | ----- 3 | [![Go Doc](https://godoc.org/github.com/un000/tailor?status.svg)](https://godoc.org/github.com/un000/tailor) 4 | [![Sourcegraph](https://sourcegraph.com/github.com/un000/tailor/-/badge.svg)](https://sourcegraph.com/github.com/un000/tailor?badge) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/un000/tailor)](https://goreportcard.com/report/github.com/un000/tailor) 6 | 7 | Tailor provides the functionality of tailing for e. g. nginx logs under logrotate. 8 | Tailor will follow a selected log file and reopen it if it's been rotated. Now, tailor doesn't require inotify, because it polls logs 9 | with a tiny delay. So the library can achieve cross-platform. 10 | 11 | There is no plan to implement truncate detection. 12 | 13 | Currently this library is used in production, handling 5k of opened files with a load over 100k rps per instance, 14 | without such overhead. 15 | ![Actual usage](https://i.imgur.com/G4QICfk.png) 16 | 17 | ## Install 18 | ```Bash 19 | go get github.com/un000/tailor 20 | ``` 21 | 22 | ## Features 23 | - Tail files from any offsets 24 | - Reopening on logrotate 25 | - Rate limiter + support custom rate limiters 26 | - Leaky bucket 27 | - Performant helpers to trim or convert bytes to string 28 | - Lag monitoring 29 | 30 | ## Example 31 | ```Go 32 | package main 33 | 34 | import ( 35 | "context" 36 | "fmt" 37 | "io" 38 | "time" 39 | 40 | "github.com/un000/tailor" 41 | ) 42 | 43 | func main() { 44 | t := tailor.New( 45 | "./github.com_access.log", 46 | tailor.WithSeekOnStartup(0, io.SeekStart), 47 | tailor.WithPollerTimeout(10*time.Millisecond), 48 | ) 49 | 50 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 51 | defer cancel() 52 | 53 | err := t.Run(ctx) 54 | if err != nil { 55 | panic(err) 56 | } 57 | 58 | fmt.Println("Tailing file:", t.FileName()) 59 | for { 60 | select { 61 | case line, ok := <-t.Lines(): 62 | if !ok { 63 | return 64 | } 65 | 66 | fmt.Println(line.StringTrimmed()) 67 | case err, ok := <-t.Errors(): 68 | if !ok { 69 | return 70 | } 71 | 72 | panic(err) 73 | } 74 | } 75 | } 76 | 77 | ``` 78 | 79 | ## Contributions are appreciated, feel free ✌️ 80 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Yegor Myskin. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package tailor provides the functionality of tailing nginx_access.log under logrotate. 6 | // Tailor can follow a selected log file and reopen it. Now, tailor doesn't require inotify, because it polls logs 7 | // with a tiny delay. So the library can achieve cross-platform. 8 | package tailor 9 | -------------------------------------------------------------------------------- /example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "time" 8 | 9 | "github.com/un000/tailor" 10 | ) 11 | 12 | func main() { 13 | t := tailor.New( 14 | "./github.com_access.log", 15 | tailor.WithSeekOnStartup(0, io.SeekStart), 16 | tailor.WithPollerTimeout(10*time.Millisecond), 17 | ) 18 | 19 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 20 | defer cancel() 21 | 22 | err := t.Run(ctx) 23 | if err != nil { 24 | panic(err) 25 | } 26 | 27 | fmt.Println("Tailing file:", t.FileName()) 28 | for { 29 | select { 30 | case line, ok := <-t.Lines(): 31 | if !ok { 32 | return 33 | } 34 | 35 | fmt.Println(line.StringTrimmed()) 36 | case err, ok := <-t.Errors(): 37 | if !ok { 38 | return 39 | } 40 | 41 | panic(err) 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/un000/tailor 2 | 3 | go 1.12 4 | 5 | require github.com/pkg/errors v0.8.1 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 2 | -------------------------------------------------------------------------------- /line.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Yegor Myskin. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tailor 6 | 7 | import ( 8 | "unsafe" 9 | ) 10 | 11 | // Line represents a returned line from the tailed file. 12 | type Line struct { 13 | fileName string 14 | line []byte 15 | } 16 | 17 | // FileName returns the file name of the line. 18 | func (l Line) FileName() string { 19 | return l.fileName 20 | } 21 | 22 | // String returns an untrimmed string. 23 | func (l Line) String() string { 24 | return *(*string)(unsafe.Pointer(&l.line)) 25 | } 26 | 27 | // StringTrimmed returns a trimmed string. 28 | func (l Line) StringTrimmed() string { 29 | trimmedString := l.BytesTrimmed() 30 | return *(*string)(unsafe.Pointer(&trimmedString)) 31 | } 32 | 33 | // Bytes returns a line. 34 | func (l Line) Bytes() []byte { 35 | return l.line 36 | } 37 | 38 | // BytesTrimmed trims the line from the \r and \n sequences. Not unicode safe. 39 | func (l Line) BytesTrimmed() []byte { 40 | // inline optimization with goto instead of for 41 | Loop: 42 | if len(l.line) == 0 { 43 | return l.line 44 | } 45 | 46 | last := l.line[len(l.line)-1] 47 | if last == '\n' || last == '\r' || last == ' ' { 48 | l.line = l.line[:len(l.line)-1] 49 | } else { 50 | return l.line 51 | } 52 | goto Loop 53 | } 54 | -------------------------------------------------------------------------------- /line_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Yegor Myskin. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tailor 6 | 7 | import ( 8 | "bytes" 9 | "testing" 10 | ) 11 | 12 | func TestLine(t *testing.T) { 13 | b := []byte("") 14 | l := Line{ 15 | line: b, 16 | fileName: "file.txt", 17 | } 18 | 19 | if l.FileName() != "file.txt" { 20 | t.Fail() 21 | return 22 | } 23 | 24 | if !bytes.Equal(b, l.Bytes()) { 25 | t.FailNow() 26 | return 27 | } 28 | 29 | if !bytes.Equal(b, l.BytesTrimmed()) { 30 | t.FailNow() 31 | return 32 | } 33 | 34 | if l.String() != "" { 35 | t.FailNow() 36 | return 37 | } 38 | 39 | if l.StringTrimmed() != "" { 40 | t.FailNow() 41 | return 42 | } 43 | 44 | l.line = []byte("abc\nde\n\r \n") 45 | 46 | if !bytes.Equal([]byte("abc\nde\n\r \n"), l.Bytes()) { 47 | t.FailNow() 48 | return 49 | } 50 | 51 | if !bytes.Equal([]byte("abc\nde"), l.BytesTrimmed()) { 52 | t.FailNow() 53 | return 54 | } 55 | 56 | if l.String() != "abc\nde\n\r \n" { 57 | t.FailNow() 58 | return 59 | } 60 | 61 | if l.StringTrimmed() != "abc\nde" { 62 | t.FailNow() 63 | return 64 | } 65 | 66 | l.line = []byte("\n") 67 | 68 | if !bytes.Equal([]byte(""), l.BytesTrimmed()) { 69 | t.FailNow() 70 | return 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Yegor Myskin. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tailor 6 | 7 | import ( 8 | "io" 9 | "time" 10 | ) 11 | 12 | // options is the main options store. 13 | // To see what's going on, watch for With* descriptions below. 14 | type options struct { 15 | runOffset int64 16 | runWhence int 17 | 18 | reopenOffset int64 19 | reopenWhence int 20 | 21 | bufioReaderPoolSize int 22 | pollerTimeout time.Duration 23 | updateLagInterval time.Duration 24 | 25 | rateLimiter RateLimiter 26 | leakyBucket bool 27 | } 28 | 29 | // withDefaultOptions sets the initial options. 30 | func withDefaultOptions() []Option { 31 | return []Option{ 32 | WithPollerTimeout(10 * time.Millisecond), 33 | WithReaderInitialPoolSize(4096), 34 | WithSeekOnStartup(0, io.SeekEnd), 35 | WithSeekOnReopen(0, io.SeekStart), 36 | WithUpdateLagInterval(5 * time.Second), 37 | } 38 | } 39 | 40 | type Option func(options *options) 41 | 42 | // WithPollerTimeout is used to timeout when file is fully read, to check changes. 43 | func WithPollerTimeout(duration time.Duration) Option { 44 | return func(options *options) { 45 | options.pollerTimeout = duration 46 | } 47 | } 48 | 49 | // WithReaderInitialPoolSize is used to set the internal initial size of bufio.Reader buffer. 50 | func WithReaderInitialPoolSize(size int) Option { 51 | return func(options *options) { 52 | options.bufioReaderPoolSize = size 53 | } 54 | } 55 | 56 | // WithSeekOnStartup is used to set file.Seek() options, when the file is opened on startup. 57 | // Use io.Seek* constants to set whence. 58 | func WithSeekOnStartup(offset int64, whence int) Option { 59 | return func(options *options) { 60 | options.runOffset = offset 61 | options.runWhence = whence 62 | } 63 | } 64 | 65 | // WithSeekOnRun is used to set file.Seek() options, when the file is opened on startup. 66 | // Use io.Seek* constants to set whence. 67 | func WithSeekOnReopen(offset int64, whence int) Option { 68 | return func(options *options) { 69 | options.reopenOffset = offset 70 | options.reopenWhence = whence 71 | } 72 | } 73 | 74 | // WithUpdateLagInterval is used to know how often update the file lag. 75 | // Frequent update time increasing Seek syscall calls. 76 | func WithUpdateLagInterval(duration time.Duration) Option { 77 | return func(options *options) { 78 | options.updateLagInterval = duration 79 | } 80 | } 81 | 82 | // WithRateLimiter is used to rate limit output lines. Watch RateLimiter interface. 83 | func WithRateLimiter(rl RateLimiter) Option { 84 | return func(options *options) { 85 | options.rateLimiter = rl 86 | } 87 | } 88 | 89 | // WithLeakyBucket is used to skip a read lines, when a listener is full. 90 | func WithLeakyBucket() Option { 91 | return func(options *options) { 92 | options.leakyBucket = true 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /rate_limiter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Yegor Myskin. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tailor 6 | 7 | // RateLimiter provides methods to create a custom rate limiter. 8 | type RateLimiter interface { 9 | // Allow says that a line should be sent to a receiver of a lines. 10 | Allow() bool 11 | 12 | // Close finalizes rate limiter. 13 | Close() 14 | } 15 | -------------------------------------------------------------------------------- /rate_limiter_channel.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Yegor Myskin. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tailor 6 | 7 | import ( 8 | "time" 9 | ) 10 | 11 | type ChannelBasedRateLimiter struct { 12 | t *time.Ticker 13 | } 14 | 15 | // NewChannelBasedRateLimiter creates an instance of rate limiter, which ticker ticks every period to limit the lps. 16 | func NewChannelBasedRateLimiter(lps int) *ChannelBasedRateLimiter { 17 | return &ChannelBasedRateLimiter{ 18 | t: time.NewTicker(time.Second / time.Duration(lps)), 19 | } 20 | } 21 | 22 | // Allow will block until the ticker ticks. 23 | func (rl *ChannelBasedRateLimiter) Allow() bool { 24 | _, ok := <-rl.t.C 25 | return ok 26 | } 27 | 28 | func (rl *ChannelBasedRateLimiter) Close() { 29 | rl.t.Stop() 30 | } 31 | -------------------------------------------------------------------------------- /rate_limiter_channel_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Yegor Myskin. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tailor 6 | 7 | import ( 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func TestChannelBasedRateLimiterDisallow(t *testing.T) { 13 | l := NewChannelBasedRateLimiter(30) 14 | defer l.Close() 15 | 16 | start := time.Now() 17 | for i := 0; i < 100; i++ { 18 | if !l.Allow() { 19 | t.FailNow() 20 | } 21 | } 22 | dur := time.Since(start) 23 | 24 | if dur < 3333*time.Millisecond || dur > 3500*time.Millisecond { 25 | t.Errorf("expected duration: ~3.33s, actual: %s", dur) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /seeker_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Yegor Myskin. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tailor 6 | 7 | import ( 8 | "bufio" 9 | "fmt" 10 | "io" 11 | "io/ioutil" 12 | "os" 13 | "strings" 14 | "testing" 15 | ) 16 | 17 | func TestNewLineFinder(t *testing.T) { 18 | var tests = []struct { 19 | content string 20 | offsetFromStart int64 21 | res string 22 | }{ 23 | {"", 0, ""}, 24 | {"\n", 0, "\n"}, 25 | {"\n\n", 1, "\n"}, 26 | {"\n\na\n", 3, "a\n"}, 27 | {"a", 0, "a"}, 28 | {"a\n", 0, "a\n"}, 29 | {"abc", 2, "abc"}, 30 | {"abc\n", 2, "abc\n"}, 31 | {"a\nb", 2, "b"}, 32 | {"a\nb\n", 2, "b\n"}, 33 | {"aaaaa\nbbbbbbbb\n", 4, "aaaaa\n"}, 34 | {"aaaaa\nbbbbbbbb\n", 10, "bbbbbbbb\n"}, 35 | {strings.Repeat("a", 300), 280, strings.Repeat("a", 300)}, 36 | {strings.Repeat("a", 300) + "\n", 280, strings.Repeat("a", 300) + "\n"}, 37 | {strings.Repeat("a", 100) + "\n" + strings.Repeat("a", 200), 280, strings.Repeat("a", 200)}, 38 | } 39 | 40 | const file = "./tst" 41 | defer os.Remove(file) 42 | 43 | for i, data := range tests { 44 | t.Run(fmt.Sprint(i), func(t *testing.T) { 45 | err := ioutil.WriteFile(file, []byte(data.content), os.ModePerm) 46 | if err != nil { 47 | t.Error(err) 48 | return 49 | } 50 | 51 | f := New(file) 52 | err = f.openFile(data.offsetFromStart, io.SeekStart) 53 | if err != nil { 54 | t.Errorf("[%d] error executing: %s, data: %+v", i, err, data) 55 | return 56 | } 57 | 58 | r := bufio.NewReader(f.file) 59 | line, err := r.ReadString('\n') 60 | if err != nil && err != io.EOF { 61 | t.Errorf("[%d] error reading line: %s, data: %+v", i, err, data) 62 | return 63 | } 64 | 65 | if line != data.res { 66 | t.Errorf("[%d] actual: '%s', want: '%s', data: %+v", i, line, data.res, data) 67 | return 68 | } 69 | }) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /tailor.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Yegor Myskin. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tailor 6 | 7 | import ( 8 | "bufio" 9 | "context" 10 | "io" 11 | "os" 12 | "sync/atomic" 13 | "time" 14 | 15 | "github.com/pkg/errors" 16 | ) 17 | 18 | type Tailor struct { 19 | fileName string 20 | file *os.File 21 | 22 | opts options 23 | 24 | // stats 25 | lastPos int64 26 | lastSize int64 27 | lag int64 28 | 29 | lines chan Line 30 | errs chan error 31 | 32 | working int32 33 | } 34 | 35 | // New prepares the instance of Tailor. It mergers default options with the given options. 36 | func New(filename string, opts ...Option) *Tailor { 37 | t := &Tailor{ 38 | fileName: filename, 39 | } 40 | 41 | for _, p := range [][]Option{withDefaultOptions(), opts} { 42 | for _, o := range p { 43 | o(&t.opts) 44 | } 45 | } 46 | 47 | return t 48 | } 49 | 50 | // Run starts the tailing procedure. 51 | // 1. Opens a file in RO mode and seeks to the newest position by default(can be changed in options). 52 | // 2. Then reads the file line by line and produces lines and errors through the channels. 53 | // If the file has been logrotated, Tailor will follow the first file to the end and after reopen it. 54 | // If error happens file will be closed. 55 | // Tailor makes an exponential sleep to reduce stat syscalls. 56 | func (t *Tailor) Run(ctx context.Context) error { 57 | if !atomic.CompareAndSwapInt32(&t.working, 0, 1) { 58 | return errors.New("already working") 59 | } 60 | 61 | failFinalizer := func() { 62 | if t.file != nil { 63 | _ = t.file.Close() 64 | } 65 | atomic.StoreInt32(&t.working, 0) 66 | } 67 | 68 | err := t.openFile(t.opts.runOffset, t.opts.runWhence) 69 | if err != nil { 70 | failFinalizer() 71 | return errors.Wrap(err, "can't open file for tailing") 72 | } 73 | 74 | t.readLoop(ctx) 75 | 76 | return nil 77 | } 78 | 79 | // readLoop starts goroutine, which reads the given file and send to the line chan tailed strings. 80 | func (t *Tailor) readLoop(ctx context.Context) { 81 | t.lines = make(chan Line) 82 | t.errs = make(chan error) 83 | 84 | go func() { 85 | defer func() { 86 | if t.file != nil { 87 | err := t.file.Close() 88 | if err != nil { 89 | t.errs <- errors.Wrap(err, "error closing file") 90 | } 91 | } 92 | close(t.lines) 93 | close(t.errs) 94 | 95 | if t.opts.rateLimiter != nil { 96 | t.opts.rateLimiter.Close() 97 | } 98 | 99 | atomic.StoreInt32(&t.working, 0) 100 | }() 101 | 102 | r := bufio.NewReaderSize(t.file, t.opts.bufioReaderPoolSize) 103 | lagReporter := time.NewTicker(t.opts.updateLagInterval) 104 | defer lagReporter.Stop() 105 | 106 | pollerTimeout := t.opts.pollerTimeout 107 | for { 108 | select { 109 | case <-ctx.Done(): 110 | return 111 | case <-lagReporter.C: 112 | err := t.updateFileStatus() 113 | if err != nil { 114 | t.errs <- errors.Wrap(err, "error getting file status") 115 | break 116 | } 117 | default: 118 | } 119 | 120 | var ( 121 | part []byte 122 | line []byte 123 | err error 124 | ) 125 | 126 | // a Line can be read partially 127 | // so wait until the new one comes 128 | // if the line wasn't read after 5 tries return it 129 | for i := 0; i < 5; i++ { 130 | part = part[:0] 131 | part, err = r.ReadBytes('\n') 132 | if err == nil || err == io.EOF { 133 | // if Line is new and finished, don't allocate buffer, just copy ref 134 | if len(line) == 0 && len(part) > 0 && part[len(part)-1] == '\n' { 135 | line = part 136 | break 137 | } 138 | } else { 139 | t.errs <- errors.Wrap(err, "error reading line") 140 | return 141 | } 142 | 143 | line = append(line, part...) 144 | 145 | if len(line) == 0 && err == io.EOF { 146 | break 147 | } 148 | 149 | if line[len(line)-1] == '\n' { 150 | break 151 | } 152 | 153 | pollerTimeout = t.exponentialSleep(pollerTimeout, time.Second) 154 | } 155 | 156 | // check that logrotate swapped the file 157 | if len(line) == 0 && err == io.EOF { 158 | isSameFile, err := t.isFileStillTheSame() 159 | if err != nil { 160 | t.errs <- errors.Wrap(err, "error checking that file is the same") 161 | return 162 | } 163 | 164 | if !isSameFile { 165 | err := t.file.Close() 166 | if err != nil { 167 | t.errs <- errors.Wrap(err, "error closing current file") 168 | } 169 | 170 | err = t.openFile(t.opts.reopenOffset, t.opts.reopenWhence) 171 | if err != nil { 172 | t.errs <- errors.Wrap(err, "error reopening file") 173 | return 174 | } 175 | 176 | r = bufio.NewReaderSize(t.file, t.opts.bufioReaderPoolSize) 177 | pollerTimeout = t.opts.pollerTimeout 178 | 179 | continue 180 | } 181 | 182 | pollerTimeout = t.exponentialSleep(pollerTimeout, 5*time.Second) 183 | continue 184 | } 185 | 186 | pollerTimeout = t.opts.pollerTimeout 187 | 188 | if t.opts.rateLimiter == nil || t.opts.rateLimiter.Allow() { 189 | line := Line{ 190 | line: line, 191 | fileName: t.fileName, 192 | } 193 | 194 | if t.opts.leakyBucket { 195 | select { 196 | case t.lines <- line: 197 | default: 198 | } 199 | continue 200 | } 201 | 202 | t.lines <- line 203 | } 204 | } 205 | }() 206 | } 207 | 208 | // Lines returns chanel of read lines. 209 | func (t *Tailor) Lines() chan Line { 210 | return t.lines 211 | } 212 | 213 | // Errors returns chanel of errors, associated with reading files. 214 | func (t *Tailor) Errors() chan error { 215 | return t.errs 216 | } 217 | 218 | // exponentialSleep sleeps for pollerTimeout and returns new exponential grown timeout <= maxWait. 219 | func (t *Tailor) exponentialSleep(pollerTimeout time.Duration, maxWait time.Duration) time.Duration { 220 | time.Sleep(pollerTimeout) 221 | // use exponential poller duration to reduce the load 222 | if pollerTimeout < maxWait { 223 | return pollerTimeout * 2 224 | } 225 | 226 | return maxWait 227 | } 228 | 229 | // isFileStillTheSame checks that opened file wasn't swapped. 230 | func (t *Tailor) isFileStillTheSame() (isSameFile bool, err error) { 231 | var maybeNewFileInfo os.FileInfo 232 | 233 | // maybe the current file is being rotated with a small lag, check it for some tries 234 | for i := 0; i < 2; i++ { 235 | maybeNewFileInfo, err = os.Stat(t.fileName) 236 | if os.IsNotExist(err) { 237 | time.Sleep(time.Second) 238 | continue 239 | } 240 | 241 | if err != nil { 242 | return false, errors.Wrap(err, "error stating maybe new file by name") 243 | } 244 | 245 | break 246 | } 247 | 248 | currentFileInfo, err := t.file.Stat() 249 | if err != nil { 250 | return false, errors.Wrap(err, "error stating current file") 251 | } 252 | 253 | return os.SameFile(currentFileInfo, maybeNewFileInfo), nil 254 | } 255 | 256 | // FileName returns the name of the tailed file. 257 | func (t Tailor) FileName() string { 258 | return t.fileName 259 | } 260 | 261 | // Lag returns approximate lag, updater per interval. 262 | func (t Tailor) Lag() int64 { 263 | return atomic.LoadInt64(&t.lag) 264 | } 265 | 266 | // openFile opens the file for reading, seeks to the beginning of the line at opts.*offset and updates the lag. 267 | func (t *Tailor) openFile(offset int64, whence int) (err error) { 268 | t.file, err = os.Open(t.fileName) 269 | if err != nil { 270 | return errors.Wrap(err, "error opening file") 271 | } 272 | 273 | err = t.seekToLineStart(offset, whence) 274 | if err != nil { 275 | return errors.Wrap(err, "error seeking to line start") 276 | } 277 | 278 | err = t.updateFileStatus() 279 | if err != nil { 280 | return errors.Wrap(err, "error updating file status") 281 | } 282 | 283 | return nil 284 | } 285 | 286 | // seekToLineStart seeks the cursor at the beginning of a line at offset. Internally this function uses a buffer 287 | // to find the beginning of a line. If the byte at offset equals \n, so the next line will be selected. 288 | func (t *Tailor) seekToLineStart(offset int64, whence int) error { 289 | const ( 290 | bufSize int64 = 256 291 | ) 292 | 293 | initialOffset, err := t.file.Seek(offset, whence) 294 | if initialOffset == 0 { 295 | return nil 296 | } 297 | if err == io.EOF { 298 | err = nil 299 | } 300 | if err != nil { 301 | return err 302 | } 303 | 304 | min := func(a, b int64) int64 { 305 | if a < b { 306 | return a 307 | } 308 | return b 309 | } 310 | 311 | var current int64 = 0 312 | Loop: 313 | for { 314 | current += min(bufSize, initialOffset-current) 315 | buf := make([]byte, min(current, bufSize)) 316 | 317 | n, err := t.file.ReadAt(buf, initialOffset-current) 318 | if err != nil && err != io.EOF { 319 | return err 320 | } 321 | buf = buf[:n] 322 | 323 | current -= int64(n) 324 | for i := int64(len(buf)) - 1; i >= 0; i-- { 325 | if buf[i] == '\n' { 326 | break Loop 327 | } 328 | current++ 329 | } 330 | if initialOffset-current == 0 { 331 | break 332 | } 333 | } 334 | 335 | _, err = t.file.Seek(-current, io.SeekCurrent) 336 | if err == io.EOF { 337 | err = nil 338 | } 339 | 340 | return err 341 | } 342 | 343 | // updateFileStatus update a current seek from the file an an actual file size. 344 | func (t *Tailor) updateFileStatus() (err error) { 345 | fi, err := t.file.Stat() 346 | if err != nil { 347 | return err 348 | } 349 | 350 | pos, err := t.file.Seek(0, io.SeekCurrent) 351 | if err != nil { 352 | return err 353 | } 354 | 355 | size := fi.Size() 356 | atomic.StoreInt64(&t.lastPos, pos) 357 | atomic.StoreInt64(&t.lastSize, size) 358 | atomic.StoreInt64(&t.lag, size-pos) 359 | 360 | return nil 361 | } 362 | -------------------------------------------------------------------------------- /tailor_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Yegor Myskin. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tailor_test 6 | 7 | import ( 8 | "context" 9 | "io" 10 | "io/ioutil" 11 | "os" 12 | "strconv" 13 | "sync" 14 | "testing" 15 | "time" 16 | 17 | "github.com/un000/tailor" 18 | ) 19 | 20 | func TestTailFileFromStart(t *testing.T) { 21 | const fileName = "./file_from_start" 22 | fileData := []byte("1\n2\n3\n") 23 | 24 | err := ioutil.WriteFile(fileName, fileData, os.ModePerm) 25 | if err != nil { 26 | t.Error(err) 27 | return 28 | } 29 | defer os.Remove(fileName) 30 | 31 | f := tailor.New(fileName, tailor.WithSeekOnStartup(0, io.SeekStart)) 32 | if fileName != f.FileName() { 33 | t.Error("file name mismatch") 34 | return 35 | } 36 | 37 | ctx, _ := context.WithTimeout(context.Background(), 2*time.Second) 38 | err = f.Run(ctx) 39 | if err != nil { 40 | t.Error(err) 41 | return 42 | } 43 | 44 | var i = 1 45 | defer func() { 46 | if i != 4 { 47 | t.Error("not read to the end, last line:", i) 48 | } 49 | }() 50 | 51 | for ; i <= 3; i++ { 52 | select { 53 | case line, ok := <-f.Lines(): 54 | if !ok { 55 | return 56 | } 57 | 58 | if line.StringTrimmed() != strconv.Itoa(i) { 59 | t.Errorf("want: '%d' actual '%s'", i, line.StringTrimmed()) 60 | return 61 | } 62 | t.Log(line.StringTrimmed()) 63 | case err, ok := <-f.Errors(): 64 | if !ok { 65 | return 66 | } 67 | t.Error(err) 68 | return 69 | } 70 | } 71 | } 72 | 73 | func TestLogrotate(t *testing.T) { 74 | const fileName = "./file_logrotate" 75 | const fileNameRotated = "./file_logrotate_rotated" 76 | fileData := []byte("1\n2\n3\n") 77 | 78 | err := ioutil.WriteFile(fileName, fileData, os.ModePerm) 79 | if err != nil { 80 | t.Error(err) 81 | return 82 | } 83 | defer os.Remove(fileName) 84 | defer os.Remove(fileNameRotated) 85 | 86 | f := tailor.New(fileName, tailor.WithSeekOnStartup(0, io.SeekStart)) 87 | if fileName != f.FileName() { 88 | t.Error("file name mismatch") 89 | return 90 | } 91 | 92 | ctx, _ := context.WithTimeout(context.Background(), 2*time.Second) 93 | err = f.Run(ctx) 94 | if err != nil { 95 | t.Error(err) 96 | return 97 | } 98 | 99 | wg := sync.WaitGroup{} 100 | wg.Add(1) 101 | go func() { 102 | err := os.Rename(fileName, fileNameRotated) 103 | if err != nil { 104 | t.Error("error renaming file") 105 | return 106 | } 107 | 108 | fileData := []byte("4\n5\n6\n") 109 | err = ioutil.WriteFile(fileName, fileData, os.ModePerm) 110 | if err != nil { 111 | t.Error(err) 112 | return 113 | } 114 | 115 | wg.Done() 116 | }() 117 | 118 | var i = 1 119 | defer func() { 120 | if i != 7 { 121 | t.Error("not read to the end, last line:", i) 122 | } 123 | }() 124 | 125 | for ; i <= 6; i++ { 126 | select { 127 | case line, ok := <-f.Lines(): 128 | if !ok { 129 | return 130 | } 131 | 132 | if line.StringTrimmed() != strconv.Itoa(i) { 133 | t.Errorf("want: '%d' actual '%s'", i, line.StringTrimmed()) 134 | return 135 | } 136 | t.Log(line.StringTrimmed()) 137 | case err, ok := <-f.Errors(): 138 | if !ok { 139 | return 140 | } 141 | t.Error(err) 142 | return 143 | } 144 | } 145 | 146 | wg.Wait() 147 | } 148 | --------------------------------------------------------------------------------