├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE.yml ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── test.yml ├── .gitignore ├── History.md ├── LICENSE ├── Readme.md ├── buffer.go ├── buffer_test.go ├── go.mod └── go.sum /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: tj -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.yml: -------------------------------------------------------------------------------- 1 | ## Prerequisites 2 | 3 | * [ ] I searched to see if the issue already exists. 4 | 5 | ## Description 6 | 7 | Describe the bug or feature. 8 | 9 | ## Steps to Reproduce 10 | 11 | Describe the steps required to reproduce the issue if applicable. 12 | 13 | ## Slack 14 | 15 | Join us on Slack https://chat.apex.sh/ 16 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Please open an issue and discuss changes before spending time on them, unless the change is trivial or an issue already exists. 2 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: Tests 3 | jobs: 4 | test: 5 | strategy: 6 | matrix: 7 | go-version: [1.13.x] 8 | platform: [ubuntu-latest, macos-latest, windows-latest] 9 | runs-on: ${{ matrix.platform }} 10 | steps: 11 | - name: Install Go 12 | uses: actions/setup-go@v1 13 | with: 14 | go-version: ${{ matrix.go-version }} 15 | - name: Checkout code 16 | uses: actions/checkout@v1 17 | - name: Test 18 | run: go test ./... -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .envrc 2 | -------------------------------------------------------------------------------- /History.md: -------------------------------------------------------------------------------- 1 | 2 | v1.2.0 / 2020-08-06 3 | =================== 4 | 5 | * change DefaultFlushInterval to 30s 6 | 7 | v1.1.0 / 2020-08-03 8 | =================== 9 | 10 | * add Buffer.FlushSync() for synchronous flushing 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2020 TJ Holowaychuk tj@tjholowaychuk.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | 'Software'), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Buffer 2 | 3 | Generic buffer for batching entries, such as log entries. 4 | 5 | --- 6 | 7 | [![GoDoc](https://godoc.org/github.com/tj/go-buffer?status.svg)](https://godoc.org/github.com/tj/go-buffer) 8 | ![](https://img.shields.io/badge/license-MIT-blue.svg) 9 | ![](https://img.shields.io/badge/status-stable-green.svg) 10 | ![](https://github.com/tj/go-buffer/workflows/Tests/badge.svg) 11 | 12 | ## Sponsors 13 | 14 | This project is sponsored by [CTO.ai](https://cto.ai/), making it easy for development teams to create and share workflow automations without leaving the command line. 15 | 16 | [![](https://apex-software.imgix.net/github/sponsors/cto.png)](https://cto.ai/) 17 | 18 | And my [GitHub sponsors](https://github.com/sponsors/tj): 19 | 20 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/0) 21 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/1) 22 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/2) 23 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/3) 24 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/4) 25 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/5) 26 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/6) 27 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/7) 28 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/8) 29 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/9) 30 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/10) 31 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/11) 32 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/12) 33 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/13) 34 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/14) 35 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/15) 36 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/16) 37 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/17) 38 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/18) 39 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/19) 40 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/20) 41 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/21) 42 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/22) 43 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/23) 44 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/24) 45 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/25) 46 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/26) 47 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/27) 48 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/28) 49 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/29) 50 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/30) 51 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/31) 52 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/32) 53 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/33) 54 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/34) 55 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/35) 56 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/36) 57 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/37) 58 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/38) 59 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/39) 60 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/40) 61 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/41) 62 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/42) 63 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/43) 64 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/44) 65 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/45) 66 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/46) 67 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/47) 68 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/48) 69 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/49) 70 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/50) 71 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/51) 72 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/52) 73 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/53) 74 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/54) 75 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/55) 76 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/56) 77 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/57) 78 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/58) 79 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/59) 80 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/60) 81 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/61) 82 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/62) 83 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/63) 84 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/64) 85 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/65) 86 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/66) 87 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/67) 88 | [](https://sponsors-api-u2fftug6kq-uc.a.run.app/sponsor/profile/68) 89 | -------------------------------------------------------------------------------- /buffer.go: -------------------------------------------------------------------------------- 1 | // Package buffer provides a generic buffer or batching mechanism for flushing 2 | // entries at a given size or interval, useful for cases such as batching log 3 | // events. 4 | package buffer 5 | 6 | import ( 7 | "context" 8 | "log" 9 | "os" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | // TODO: make logging optional, accept a Printfer 15 | 16 | // logs instance. 17 | var logs = log.New(os.Stderr, "buffer ", log.LstdFlags) 18 | 19 | // temporary is the interfaced used for temporary errors. 20 | type temporary interface { 21 | Temporary() bool 22 | } 23 | 24 | // DefaultMaxEntries is the default max entries limit. 25 | var DefaultMaxEntries = 250 26 | 27 | // DefaultMaxRetries is the default max retries limit. 28 | var DefaultMaxRetries = 3 29 | 30 | // DefaultFlushInterval is the default flush interval. 31 | var DefaultFlushInterval = time.Second * 30 32 | 33 | // DefaultFlushTimeout is the default flush timeout. 34 | var DefaultFlushTimeout = time.Second * 15 35 | 36 | // FlushFunc is the flush callback function used to flush entries. 37 | type FlushFunc func(context.Context, []interface{}) error 38 | 39 | // ErrorFunc is the error callback function used to report flushing errors, 40 | // called when flushing fails and MaxRetries has exceeded on temporary errors. 41 | type ErrorFunc func(error) 42 | 43 | // Option function. 44 | type Option func(*Buffer) 45 | 46 | // New buffer with the given options. 47 | func New(options ...Option) *Buffer { 48 | var v Buffer 49 | v.handleFlush = noopFlush 50 | v.handleError = noopError 51 | v.maxEntries = DefaultMaxEntries 52 | v.maxRetries = DefaultMaxRetries 53 | v.flushInterval = DefaultFlushInterval 54 | v.flushTimeout = DefaultFlushTimeout 55 | 56 | for _, o := range options { 57 | o(&v) 58 | } 59 | 60 | v.done = make(chan struct{}) 61 | if v.flushInterval > 0 { 62 | go v.intervalFlush() 63 | } 64 | 65 | return &v 66 | } 67 | 68 | // WithFlushHandler sets the function handling flushes. 69 | func WithFlushHandler(fn FlushFunc) Option { 70 | return func(v *Buffer) { 71 | v.handleFlush = fn 72 | } 73 | } 74 | 75 | // WithErrorHandler sets the function handling errors. 76 | func WithErrorHandler(fn ErrorFunc) Option { 77 | return func(v *Buffer) { 78 | v.handleError = fn 79 | } 80 | } 81 | 82 | // WithMaxEntries sets the maximum number of entries before flushing. 83 | func WithMaxEntries(n int) Option { 84 | return func(v *Buffer) { 85 | v.maxEntries = n 86 | } 87 | } 88 | 89 | // WithMaxRetries sets the maximum number of retries for temporary flush errors. 90 | func WithMaxRetries(n int) Option { 91 | return func(v *Buffer) { 92 | v.maxRetries = n 93 | } 94 | } 95 | 96 | // WithFlushInterval sets the interval at which events are periodically flushed. 97 | func WithFlushInterval(d time.Duration) Option { 98 | return func(v *Buffer) { 99 | v.flushInterval = d 100 | } 101 | } 102 | 103 | // WithFlushTimeout sets the flush timeout. 104 | func WithFlushTimeout(d time.Duration) Option { 105 | return func(v *Buffer) { 106 | v.flushTimeout = d 107 | } 108 | } 109 | 110 | // Buffer is used to batch entries. 111 | type Buffer struct { 112 | pendingFlushes sync.WaitGroup 113 | done chan struct{} 114 | 115 | // callbacks 116 | handleFlush FlushFunc 117 | handleError ErrorFunc 118 | 119 | // limits 120 | maxEntries int 121 | maxRetries int 122 | flushInterval time.Duration 123 | flushTimeout time.Duration 124 | 125 | // buffer 126 | mu sync.Mutex 127 | values []interface{} 128 | } 129 | 130 | // Push adds a value to the buffer. 131 | func (b *Buffer) Push(value interface{}) { 132 | b.mu.Lock() 133 | defer b.mu.Unlock() 134 | 135 | b.values = append(b.values, value) 136 | 137 | if len(b.values) >= b.maxEntries { 138 | b.flush() 139 | } 140 | } 141 | 142 | // Flush flushes any pending entries asynchronously. 143 | func (b *Buffer) Flush() { 144 | b.mu.Lock() 145 | defer b.mu.Unlock() 146 | 147 | b.flush() 148 | } 149 | 150 | // FlushSync flushes any pending entries synchronously. 151 | func (b *Buffer) FlushSync() { 152 | b.mu.Lock() 153 | defer b.mu.Unlock() 154 | 155 | b.doFlush(b.values) 156 | b.values = nil 157 | } 158 | 159 | // Close flushes any pending entries, and waits for flushing to complete. This 160 | // method should be called before exiting your program to ensure entries have 161 | // flushed properly. 162 | func (b *Buffer) Close() { 163 | b.Flush() 164 | close(b.done) 165 | b.pendingFlushes.Wait() 166 | } 167 | 168 | // intervalFlush starts a loop flushing at the FlushInterval. 169 | func (b *Buffer) intervalFlush() { 170 | tick := time.NewTicker(b.flushInterval) 171 | for { 172 | select { 173 | case <-tick.C: 174 | b.Flush() 175 | case <-b.done: 176 | tick.Stop() 177 | return 178 | } 179 | } 180 | } 181 | 182 | // flush performs a flush asynchronuously. 183 | func (b *Buffer) flush() { 184 | values := b.values 185 | b.values = nil 186 | 187 | if len(values) == 0 { 188 | return 189 | } 190 | 191 | b.pendingFlushes.Add(1) 192 | 193 | go func() { 194 | b.doFlush(values) 195 | b.pendingFlushes.Done() 196 | }() 197 | } 198 | 199 | // doFlush handles flushing of the given values. 200 | func (b *Buffer) doFlush(values []interface{}) { 201 | var retries int 202 | 203 | retry: 204 | ctx, cancel := context.WithTimeout(context.Background(), b.flushTimeout) 205 | 206 | err := b.handleFlush(ctx, values) 207 | cancel() 208 | 209 | // temporary error, retry if we haven't exceeded MaxEntries 210 | if e, ok := err.(temporary); ok && e.Temporary() { 211 | logs.Printf("temporary error flushing %d entries: %v", len(values), e) 212 | time.Sleep(time.Second) // TODO: backoff 213 | retries++ 214 | if retries < b.maxRetries { 215 | goto retry 216 | } 217 | logs.Printf("max retries of %d exceeded", b.maxRetries) 218 | } 219 | 220 | if err != nil { 221 | b.handleError(err) 222 | } 223 | } 224 | 225 | // noopFlush function. 226 | func noopFlush(context.Context, []interface{}) error { 227 | return nil 228 | } 229 | 230 | // noopError function. 231 | func noopError(err error) { 232 | logs.Printf("unhandled error: %v", "") 233 | } 234 | -------------------------------------------------------------------------------- /buffer_test.go: -------------------------------------------------------------------------------- 1 | package buffer_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "testing" 8 | "time" 9 | 10 | "github.com/tj/assert" 11 | 12 | "github.com/tj/go-buffer" 13 | ) 14 | 15 | // Test forced flush. 16 | func TestBuffer_Flush(t *testing.T) { 17 | var mu sync.Mutex 18 | var flushes int 19 | var flushed []interface{} 20 | 21 | flush := func(ctx context.Context, values []interface{}) error { 22 | mu.Lock() 23 | defer mu.Unlock() 24 | flushes++ 25 | flushed = append(flushed, values...) 26 | return nil 27 | } 28 | 29 | errors := func(err error) { 30 | assert.NoError(t, err) 31 | } 32 | 33 | b := buffer.New( 34 | buffer.WithFlushHandler(flush), 35 | buffer.WithErrorHandler(errors), 36 | ) 37 | 38 | b.Push("hello") 39 | b.Push(" ") 40 | 41 | b.Flush() 42 | 43 | b.Push("world") 44 | b.Push("!") 45 | 46 | b.Close() 47 | 48 | assert.Equal(t, 2, flushes, "flush count") 49 | assert.Len(t, flushed, 4) 50 | } 51 | 52 | // Test sync forced flush. 53 | func TestBuffer_FlushSync(t *testing.T) { 54 | var mu sync.Mutex 55 | var flushes int 56 | var flushed []interface{} 57 | 58 | flush := func(ctx context.Context, values []interface{}) error { 59 | mu.Lock() 60 | defer mu.Unlock() 61 | flushes++ 62 | flushed = append(flushed, values...) 63 | return nil 64 | } 65 | 66 | errors := func(err error) { 67 | assert.NoError(t, err) 68 | } 69 | 70 | b := buffer.New( 71 | buffer.WithFlushHandler(flush), 72 | buffer.WithErrorHandler(errors), 73 | ) 74 | 75 | b.Push("hello") 76 | b.Push(" ") 77 | 78 | b.FlushSync() 79 | 80 | b.Push("world") 81 | b.Push("!") 82 | 83 | b.Close() 84 | 85 | assert.Equal(t, 2, flushes, "flush count") 86 | assert.Len(t, flushed, 4) 87 | } 88 | 89 | // Test max entries. 90 | func TestBuffer_maxEntries(t *testing.T) { 91 | var mu sync.Mutex 92 | var flushes int 93 | var flushed []interface{} 94 | 95 | flush := func(ctx context.Context, values []interface{}) error { 96 | mu.Lock() 97 | defer mu.Unlock() 98 | flushes++ 99 | flushed = append(flushed, values...) 100 | return nil 101 | } 102 | 103 | errors := func(err error) { 104 | assert.NoError(t, err) 105 | } 106 | 107 | b := buffer.New( 108 | buffer.WithFlushHandler(flush), 109 | buffer.WithErrorHandler(errors), 110 | buffer.WithMaxEntries(3), 111 | ) 112 | 113 | b.Push("hello") 114 | b.Push(" ") 115 | b.Push("world") 116 | b.Push("!") 117 | 118 | b.Close() 119 | 120 | assert.Len(t, flushed, 4) 121 | assert.Equal(t, 2, flushes, "flush count") 122 | } 123 | 124 | // Test flush interval. 125 | func TestBuffer_flushInterval(t *testing.T) { 126 | var mu sync.Mutex 127 | var flushes int 128 | var flushed []interface{} 129 | 130 | flush := func(ctx context.Context, values []interface{}) error { 131 | mu.Lock() 132 | defer mu.Unlock() 133 | flushes++ 134 | flushed = append(flushed, values...) 135 | return nil 136 | } 137 | 138 | errors := func(err error) { 139 | assert.NoError(t, err) 140 | } 141 | 142 | b := buffer.New( 143 | buffer.WithFlushHandler(flush), 144 | buffer.WithErrorHandler(errors), 145 | buffer.WithMaxEntries(100), 146 | buffer.WithFlushInterval(time.Millisecond*50), 147 | ) 148 | 149 | b.Push("hello") 150 | time.Sleep(time.Millisecond * 150) 151 | 152 | b.Push(" ") 153 | time.Sleep(time.Millisecond * 150) 154 | 155 | b.Push("world") 156 | b.Push("!") 157 | 158 | b.Close() 159 | 160 | assert.Len(t, flushed, 4) 161 | assert.Equal(t, 3, flushes, "flush count") 162 | } 163 | 164 | // temporaryError . 165 | type temporaryError struct{} 166 | 167 | // Error implementation. 168 | func (e temporaryError) Error() string { 169 | return "rate limited" 170 | } 171 | 172 | // Temporary implementation. 173 | func (e temporaryError) Temporary() bool { 174 | return true 175 | } 176 | 177 | // Test flush retries on temporary errors. 178 | func TestBuffer_flushRetriesOk(t *testing.T) { 179 | var mu sync.Mutex 180 | var flushes int 181 | 182 | flush := func(ctx context.Context, values []interface{}) error { 183 | mu.Lock() 184 | defer mu.Unlock() 185 | flushes++ 186 | if flushes < 3 { 187 | return temporaryError{} 188 | } 189 | return nil 190 | } 191 | 192 | errors := func(err error) { 193 | assert.NoError(t, err) 194 | } 195 | 196 | b := buffer.New( 197 | buffer.WithFlushHandler(flush), 198 | buffer.WithErrorHandler(errors), 199 | ) 200 | 201 | b.Push("hello") 202 | b.Push("world") 203 | b.Push("!") 204 | 205 | b.Close() 206 | 207 | assert.Equal(t, 3, flushes, "flush count") 208 | } 209 | 210 | // Test exceeding retries. 211 | func TestBuffer_flushRetriesExceeded(t *testing.T) { 212 | var mu sync.Mutex 213 | var flushes int 214 | var err error 215 | 216 | flush := func(ctx context.Context, values []interface{}) error { 217 | mu.Lock() 218 | defer mu.Unlock() 219 | flushes++ 220 | return temporaryError{} 221 | } 222 | 223 | errors := func(e error) { 224 | err = e 225 | } 226 | 227 | b := buffer.New( 228 | buffer.WithFlushHandler(flush), 229 | buffer.WithErrorHandler(errors), 230 | ) 231 | 232 | b.Push("hello") 233 | b.Push("world") 234 | b.Push("!") 235 | 236 | b.Close() 237 | 238 | assert.Equal(t, 3, flushes, "flush count") 239 | assert.EqualError(t, err, `rate limited`) 240 | } 241 | 242 | // Test regular flush errors. 243 | func TestBuffer_flushErrors(t *testing.T) { 244 | var mu sync.Mutex 245 | var flushes int 246 | var err error 247 | 248 | flush := func(ctx context.Context, values []interface{}) error { 249 | mu.Lock() 250 | defer mu.Unlock() 251 | flushes++ 252 | return fmt.Errorf("boom") 253 | } 254 | 255 | errors := func(e error) { 256 | err = e 257 | } 258 | 259 | b := buffer.New( 260 | buffer.WithFlushHandler(flush), 261 | buffer.WithErrorHandler(errors), 262 | ) 263 | 264 | b.Push("hello") 265 | b.Push("world") 266 | b.Push("!") 267 | 268 | b.Close() 269 | 270 | assert.Equal(t, 1, flushes, "flush count") 271 | assert.EqualError(t, err, `boom`) 272 | } 273 | 274 | // Benchmark pushing. 275 | func BenchmarkPush(b *testing.B) { 276 | buf := buffer.New( 277 | buffer.WithMaxEntries(250), 278 | ) 279 | 280 | b.ResetTimer() 281 | b.ReportAllocs() 282 | 283 | for i := 0; i < b.N; i++ { 284 | buf.Push("hello") 285 | } 286 | 287 | buf.Close() 288 | } 289 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tj/go-buffer 2 | 3 | go 1.14 4 | 5 | require github.com/tj/assert v0.0.3 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 5 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 6 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 7 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 8 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 9 | github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= 10 | github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= 11 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 12 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 13 | gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo= 14 | gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 15 | --------------------------------------------------------------------------------