├── .github └── workflows │ └── go.yml ├── .gitignore ├── LICENSE ├── README.md ├── buffer.go ├── buffer_test.go ├── example_test.go ├── go.mod ├── mock_flusher.go └── mock_flusher_test.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.18 20 | 21 | - name: Build 22 | run: go build -v ./... 23 | 24 | - name: Test 25 | run: go test -v -race -coverprofile=coverage.txt -covermode=atomic ./... 26 | 27 | - name: Upload coverage 28 | uses: codecov/codecov-action@v2 29 | with: 30 | token: ${{ secrets.CODECOV_TOKEN }} 31 | file: coverage.txt 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 wurui 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # async-buffer 2 | 3 | [![Go](https://github.com/woorui/async-buffer/actions/workflows/go.yml/badge.svg)](https://github.com/woorui/async-buffer/actions/workflows/go.yml) 4 | [![codecov](https://codecov.io/gh/woorui/async-buffer/branch/main/graph/badge.svg?token=G7OK0KG9YT)](https://codecov.io/gh/woorui/async-buffer) 5 | [![Go Reference](https://pkg.go.dev/badge/github.com/woorui/async-buffer.svg)](https://pkg.go.dev/github.com/woorui/async-buffer) 6 | 7 | The async-buffer buffer data that can be flushed when reach threshold or duration limit. It is multi-goroutinue safe. 8 | 9 | **It only support go1.18 or later** 10 | 11 | ## Why you need it? 12 | 13 | ### An Usecase: 14 | 15 | You have a message queue subscriber server. 16 | 17 | The Server receives messages one by one and inserts them into your database, 18 | 19 | But there is a big performance gap between one by one insertion and batch insertion to your database. 20 | 21 | So that to use async-buffer to buffer data then find timing to batch insert them. 22 | 23 | ## Installation 24 | 25 | ``` 26 | go get -u github.com/woorui/async-buffer 27 | ``` 28 | 29 | ## Quick start 30 | 31 | The `Write`, `Flush`, `Close` api are goroutinue-safed. 32 | 33 | ```go 34 | package main 35 | 36 | import ( 37 | "context" 38 | "fmt" 39 | "time" 40 | 41 | buffer "github.com/woorui/async-buffer" 42 | ) 43 | 44 | // pp implements Flusher interface 45 | type pp struct{} 46 | 47 | func (p *pp) Flush(strs []string) error { 48 | return print(strs) 49 | } 50 | 51 | func print(strs []string) error { 52 | fmt.Printf("print: %v \n", strs) 53 | return nil 54 | } 55 | 56 | func main() { 57 | // can also call buffer.FlushFunc` to adapt a function to Flusher 58 | buf := buffer.New[string](&pp{}, buffer.Option[string]{ 59 | Threshold: 5, 60 | FlushInterval: 3 * time.Second, 61 | WriteTimeout: time.Second, 62 | FlushTimeout: time.Second, 63 | ErrHandler: func(err error, t []string) { fmt.Printf("err: %v, ele: %v", err, t) }, 64 | }) 65 | 66 | // data maybe loss if Close() is not be called 67 | defer buf.Close() 68 | 69 | // 1. flush at threshold 70 | buf.Write("a", "b", "c", "d", "e", "f") 71 | 72 | // 2. time to flush automatically 73 | buf.Write("aaaaa") 74 | buf.Write("bbbbb") 75 | buf.Write("ccccc", "ddddd") 76 | time.Sleep(5 * time.Second) 77 | 78 | // 3. flush manually and write call `WriteWithContext` 79 | buf.WriteWithContext(context.Background(), "eeeee", "fffff") 80 | buf.Flush() 81 | } 82 | 83 | ``` 84 | 85 | ## License 86 | 87 | [MIT License](https://github.com/woorui/async-buffer/blob/main/LICENSE) 88 | -------------------------------------------------------------------------------- /buffer.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "time" 9 | ) 10 | 11 | // DefaultDataBackupSize is the default size of buffer data backup 12 | const DefaultDataBackupSize = 128 13 | 14 | var ( 15 | // ErrClosed represents a closed buffer 16 | ErrClosed = errors.New("async-buffer: buffer is closed") 17 | // ErrWriteTimeout returned if write timeout 18 | ErrWriteTimeout = errors.New("async-buffer: write timeout") 19 | // ErrFlushTimeout returned if flush timeout 20 | ErrFlushTimeout = errors.New("async-buffer: flush timeout") 21 | ) 22 | 23 | // Flusher holds FlushFunc, Flusher tell Buffer how to flush data. 24 | type Flusher[T any] interface { 25 | Flush(elements []T) error 26 | } 27 | 28 | // The FlushFunc is an adapter to allow the use of ordinary functions 29 | // as a Flusher. FlushFunc(f) is a Flusher that calls f. 30 | type FlushFunc[T any] func(elements []T) error 31 | 32 | // Flush calls FlushFunc itself. 33 | func (f FlushFunc[T]) Flush(elements []T) error { 34 | return f(elements) 35 | } 36 | 37 | // DefaultErrHandler prints error and the size of elements to stderr. 38 | func DefaultErrHandler[T any](err error, elements []T) { 39 | fmt.Fprintf( 40 | os.Stderr, 41 | "async-buffer: error while flushing error = %v, backup size = %d\n", err, len(elements)) 42 | } 43 | 44 | // Option for New the buffer. 45 | // 46 | // If both Threshold and FlushInterval are set to zero, Writing is Flushing. 47 | type Option[T any] struct { 48 | // Threshold indicates that the buffer is large enough to trigger flushing, 49 | // if Threshold is zero, do not judge threshold. 50 | Threshold uint32 51 | // WriteTimeout set write timeout, set to zero if a negative, zero means no timeout. 52 | WriteTimeout time.Duration 53 | // FlushTimeout flush timeout, set to zero if a negative, zero means no timeout. 54 | FlushTimeout time.Duration 55 | // FlushInterval indicates the interval between automatic flushes, set to zero if a negative. 56 | // There is automatic flushing if zero FlushInterval. 57 | FlushInterval time.Duration 58 | // ErrHandler handles errors, print error and the size of elements to stderr in default. 59 | ErrHandler func(err error, elements []T) 60 | } 61 | 62 | // Buffer represents an async buffer. 63 | // 64 | // The Buffer automatically flush data within a cycle 65 | // flushing is also triggered when the data reaches the specified threshold. 66 | // 67 | // If both Threshold and FlushInterval are setted to zero, Writing is Flushing. 68 | // 69 | // You can also flush data manually by calling `Flush`. 70 | type Buffer[T any] struct { 71 | ctx context.Context // ctx controls the lifecycle of Buffer 72 | cancel context.CancelFunc // cancel is used to stop Buffer flushing 73 | datas chan T // accept data 74 | doFlush chan struct{} // flush signal 75 | tickerC <-chan time.Time // tickerC flushs datas, when tickerC is nil, Buffer do not timed flushing 76 | tickerStop func() // tickerStop stop the ticker 77 | option Option[T] // options 78 | flusher Flusher[T] // Flusher is the Flusher that flushes outputs the buffer to a permanent destination 79 | done chan struct{} // done ensures internal `run` function exit 80 | } 81 | 82 | // New returns the async buffer based on option 83 | func New[T any](flusher Flusher[T], option Option[T]) *Buffer[T] { 84 | ctx, cancel := context.WithCancel(context.Background()) 85 | 86 | tickerC, tickerStop := wrapNewTicker(option.FlushInterval) 87 | 88 | backupSize := DefaultDataBackupSize 89 | if threshold := option.Threshold; threshold != 0 { 90 | backupSize = int(threshold) * 2 91 | } 92 | 93 | if option.ErrHandler == nil { 94 | option.ErrHandler = DefaultErrHandler[T] 95 | } 96 | 97 | b := &Buffer[T]{ 98 | ctx: ctx, 99 | cancel: cancel, 100 | datas: make(chan T, backupSize), 101 | doFlush: make(chan struct{}, 1), 102 | tickerC: tickerC, 103 | tickerStop: tickerStop, 104 | option: option, 105 | flusher: flusher, 106 | done: make(chan struct{}, 1), 107 | } 108 | 109 | go b.run() 110 | 111 | return b 112 | } 113 | 114 | // WriteWithContext writes elements to buffer, 115 | // It returns the count the written element and a closed error if buffer was closed. 116 | func (b *Buffer[T]) WriteWithContext(ctx context.Context, elements ...T) (int, error) { 117 | select { 118 | case <-ctx.Done(): 119 | return 0, ctx.Err() 120 | default: 121 | } 122 | return b.Write(elements...) 123 | } 124 | 125 | // Write writes elements to buffer, 126 | // It returns the count the written element and a closed error if buffer was closed. 127 | func (b *Buffer[T]) Write(elements ...T) (int, error) { 128 | if b.option.Threshold == 0 && b.option.FlushInterval == 0 { 129 | return b.writeDirect(elements) 130 | } 131 | 132 | select { 133 | case <-b.ctx.Done(): 134 | return 0, ErrClosed 135 | default: 136 | } 137 | c, stop := wrapNewTimer(b.option.WriteTimeout) 138 | defer stop() 139 | 140 | n := 0 141 | for _, ele := range elements { 142 | select { 143 | case <-c: 144 | return n, ErrWriteTimeout 145 | case b.datas <- ele: 146 | n++ 147 | } 148 | } 149 | 150 | return n, nil 151 | } 152 | 153 | func (b *Buffer[T]) writeDirect(elements []T) (int, error) { 154 | var ( 155 | n = len(elements) 156 | errch = make(chan error, 1) 157 | ) 158 | 159 | go func() { 160 | errch <- b.flusher.Flush(elements) 161 | }() 162 | 163 | c, stop := wrapNewTimer(b.option.WriteTimeout) 164 | defer stop() 165 | 166 | var err error 167 | select { 168 | case err = <-errch: 169 | case <-c: 170 | return 0, ErrWriteTimeout 171 | } 172 | if err != nil { 173 | return 0, err 174 | } 175 | return n, nil 176 | } 177 | 178 | // run do flushing in the background. 179 | func (b *Buffer[T]) run() { 180 | flat := make([]T, 0, b.option.Threshold) 181 | 182 | for { 183 | select { 184 | case <-b.ctx.Done(): 185 | close(b.datas) 186 | b.internalFlush(flat) 187 | close(b.done) 188 | return 189 | case d := <-b.datas: 190 | flat = append(flat, d) 191 | if b.option.Threshold == 0 { 192 | continue 193 | } 194 | if len(flat) == cap(flat) { 195 | b.internalFlush(flat) 196 | flat = flat[:0] 197 | } 198 | case <-b.doFlush: 199 | b.internalFlush(flat) 200 | flat = flat[:0] 201 | case <-b.tickerC: 202 | b.internalFlush(flat) 203 | flat = flat[:0] 204 | } 205 | } 206 | } 207 | 208 | func (b *Buffer[T]) internalFlush(elements []T) { 209 | if len(elements) == 0 { 210 | return 211 | } 212 | 213 | flat := elements[:len(elements):len(elements)] 214 | 215 | done := make(chan struct{}, 1) 216 | go func() { 217 | if err := b.flusher.Flush(flat); err != nil { 218 | b.option.ErrHandler(err, flat) 219 | } 220 | done <- struct{}{} 221 | }() 222 | 223 | c, stop := wrapNewTimer(b.option.FlushTimeout) 224 | defer stop() 225 | 226 | select { 227 | case <-c: 228 | b.option.ErrHandler(ErrFlushTimeout, flat) 229 | case <-done: 230 | } 231 | } 232 | 233 | // Flush flushs elements once. 234 | func (b *Buffer[T]) Flush() { b.doFlush <- struct{}{} } 235 | 236 | // Close stop flushing and handles rest elements. 237 | func (b *Buffer[T]) Close() error { 238 | select { 239 | case <-b.ctx.Done(): 240 | return ErrClosed 241 | default: 242 | } 243 | b.cancel() 244 | b.tickerStop() 245 | 246 | <-b.done 247 | 248 | flat := make([]T, 0) 249 | 250 | for v := range b.datas { 251 | flat = append(flat, v) 252 | } 253 | 254 | if len(flat) == 0 { 255 | return nil 256 | } 257 | 258 | if err := b.flusher.Flush(flat); err != nil { 259 | return err 260 | } 261 | 262 | return nil 263 | } 264 | 265 | func wrapNewTicker(d time.Duration) (<-chan time.Time, func()) { 266 | var ( 267 | c = (<-chan time.Time)(nil) 268 | stop = func() {} 269 | ) 270 | if d != 0 { 271 | t := time.NewTicker(d) 272 | c = t.C 273 | stop = t.Stop 274 | } 275 | 276 | return c, stop 277 | } 278 | 279 | func wrapNewTimer(d time.Duration) (<-chan time.Time, func() bool) { 280 | var ( 281 | c = (<-chan time.Time)(nil) 282 | stop = func() bool { return false } 283 | ) 284 | 285 | if d != 0 { 286 | t := time.NewTimer(d) 287 | c = t.C 288 | stop = t.Stop 289 | } 290 | 291 | return c, stop 292 | } 293 | -------------------------------------------------------------------------------- /buffer_test.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "reflect" 8 | "sync" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func TestAsyncBuffer(t *testing.T) { 14 | flusher := newStringCounter("", time.Microsecond) 15 | 16 | buf := New[string](flusher, Option[string]{Threshold: 10, FlushInterval: time.Millisecond}) 17 | 18 | m := map[string]int{ 19 | "AA": 100, 20 | "BB": 123, 21 | "CC": 42, 22 | } 23 | 24 | var wg sync.WaitGroup 25 | for k, v := range m { 26 | for i := 0; i < v; i++ { 27 | wg.Add(1) 28 | go func(k string) { 29 | defer wg.Done() 30 | _, err := buf.Write(k) 31 | if err != nil { 32 | t.Errorf("TestAsyncBuffer unexcept error: %v\n", err) 33 | } 34 | }(k) 35 | } 36 | } 37 | 38 | wg.Add(1) 39 | go func() { 40 | defer wg.Done() 41 | buf.Flush() 42 | }() 43 | 44 | wg.Wait() 45 | 46 | buf.Close() 47 | 48 | actual := flusher.result() 49 | 50 | if !reflect.DeepEqual(m, actual) { 51 | t.Errorf("TestAsyncBuffer want: %v, actual: %v", m, actual) 52 | } 53 | } 54 | 55 | func TestCallFlush(t *testing.T) { 56 | flusher := newStringCounter("", time.Microsecond) 57 | 58 | buf := New[string](flusher, Option[string]{Threshold: 1000, FlushInterval: time.Hour}) 59 | 60 | m := map[string]int{"AA": 100} 61 | 62 | var wg sync.WaitGroup 63 | for k, v := range m { 64 | for i := 0; i < v; i++ { 65 | wg.Add(1) 66 | go func(k string) { 67 | defer wg.Done() 68 | _, err := buf.Write(k) 69 | if err != nil { 70 | fmt.Println(err) 71 | } 72 | }(k) 73 | } 74 | } 75 | 76 | wg.Add(1) 77 | go func() { 78 | defer wg.Done() 79 | buf.Flush() 80 | }() 81 | 82 | wg.Wait() 83 | 84 | buf.Close() 85 | 86 | actual := flusher.result() 87 | 88 | if !reflect.DeepEqual(m, actual) { 89 | t.Errorf("TestCallFlush want: %v, actual: %v", m, actual) 90 | } 91 | } 92 | 93 | func TestWriteAfterClose(t *testing.T) { 94 | co := newStringCounter("", 200*time.Millisecond) 95 | 96 | threshold := uint32(1) 97 | 98 | buf := New[string](co, Option[string]{ 99 | Threshold: threshold, 100 | FlushInterval: time.Hour, 101 | WriteTimeout: 200 * time.Millisecond, 102 | }) 103 | 104 | buf.Close() 105 | 106 | n, err := buf.Write("CC") 107 | 108 | if n != 0 || err != ErrClosed { 109 | t.Errorf( 110 | "TestWriteTimeout want: %d, %v, actual: %d, %v", 111 | 0, ErrClosed, 112 | n, err, 113 | ) 114 | } 115 | } 116 | 117 | func TestWriteTimeout(t *testing.T) { 118 | co := newStringCounter("", time.Hour) 119 | 120 | threshold := uint32(1) 121 | 122 | buf := New[string](co, Option[string]{ 123 | Threshold: threshold, 124 | FlushInterval: time.Hour, // block the flushing. 125 | WriteTimeout: 200 * time.Millisecond, 126 | }) 127 | 128 | // make buffer.datas full. 129 | // buf.datas cap is 2 (Threshold*2), It will consume one immediately, 130 | // then set to 3 for making block when flush the third. 131 | for i := 0; i < int(threshold)*2+1; i++ { 132 | buf.datas <- "KK" 133 | } 134 | 135 | n, err := buf.Write("CC") 136 | 137 | if n != 0 || err != ErrWriteTimeout { 138 | t.Errorf( 139 | "TestWriteTimeout want: %d, %v, actual: %d, %v", 140 | n, err, 141 | 0, ErrWriteTimeout, 142 | ) 143 | } 144 | } 145 | 146 | func TestWriteDirect(t *testing.T) { 147 | co := newStringCounter("", 100*time.Millisecond) 148 | 149 | buf := New[string](co, Option[string]{ 150 | Threshold: 0, // make write direct. 151 | FlushInterval: 0, // make write direct. 152 | WriteTimeout: 0, 153 | }) 154 | 155 | // make buffer.datas full. 156 | // buf.datas cap is 2 (DefaultDataBackupSize*2) when Threshold is zero, 157 | // It will consume one immediately, 158 | // then set to 3 for making block when flush the third. 159 | for i := 0; i < DefaultDataBackupSize+1; i++ { 160 | buf.datas <- "KK" 161 | } 162 | 163 | n, err := buf.Write("CC", "DD", "EE", "FF") 164 | 165 | if n != 4 || err != nil { 166 | t.Errorf( 167 | "TestWriteDirect want: %d, %v, actual: %d, %v", 168 | 0, ErrWriteTimeout, 169 | n, err, 170 | ) 171 | } 172 | } 173 | 174 | func TestWriteWithContext(t *testing.T) { 175 | co := newStringCounter("", 100*time.Millisecond) 176 | 177 | buf := New[string](co, Option[string]{}) 178 | 179 | ctx, cancel := context.WithCancel(context.Background()) 180 | 181 | n, err := buf.WriteWithContext(ctx, "CC", "DD", "EE", "FF") 182 | 183 | if n != 4 || err != nil { 184 | t.Errorf( 185 | "TestWriteWithContext before cancel ctx want: %d, %v, actual: %d, %v", 186 | 0, nil, 187 | n, err, 188 | ) 189 | } 190 | 191 | cancel() 192 | n, err = buf.WriteWithContext(ctx, "CC", "DD", "EE", "FF") 193 | 194 | if n != 0 || err != ctx.Err() { 195 | t.Errorf( 196 | "TestWriteWithContext after cancel ctx want: %d, %v, actual: %d, %v", 197 | 0, ctx.Err(), 198 | n, err, 199 | ) 200 | } 201 | } 202 | 203 | func TestWriteDirectTimeout(t *testing.T) { 204 | co := newStringCounter("", time.Hour) 205 | 206 | buf := New[string](co, Option[string]{ 207 | Threshold: 0, // make write direct. 208 | FlushInterval: 0, // make write direct. 209 | WriteTimeout: 200 * time.Millisecond, 210 | }) 211 | 212 | // make buffer.datas full. 213 | // buf.datas cap is 2 (DefaultDataBackupSize*2) when Threshold is zero, 214 | // It will consume one immediately, 215 | // then set to 3 for making block when flush the third. 216 | for i := 0; i < DefaultDataBackupSize+1; i++ { 217 | buf.datas <- "KK" 218 | } 219 | 220 | n, err := buf.Write("CC") 221 | 222 | if n != 0 || err != ErrWriteTimeout { 223 | t.Errorf( 224 | "TestWriteDirectTimeout want: %d, %v, actual: %d, %v", 225 | 0, ErrWriteTimeout, 226 | n, err, 227 | ) 228 | } 229 | } 230 | 231 | func TestWriteDirectError(t *testing.T) { 232 | co := newStringCounter("ERROR", time.Millisecond) 233 | 234 | buf := New[string](co, Option[string]{ 235 | Threshold: 0, // make write direct. 236 | FlushInterval: 0, // make write direct. 237 | WriteTimeout: 200 * time.Millisecond, 238 | }) 239 | 240 | n, err := buf.Write("ERROR") 241 | 242 | if n != 0 || err != errErrInput { 243 | t.Errorf( 244 | "TestWriteDirectError want: %d, %v, actual: %d, %v", 245 | 0, errErrInput, 246 | n, err, 247 | ) 248 | } 249 | } 250 | 251 | func TestCloseError(t *testing.T) { 252 | co := newStringCounter("ERROR", 100*time.Millisecond) 253 | 254 | buf := New[string](co, Option[string]{ 255 | Threshold: 100000, 256 | FlushInterval: time.Hour, 257 | WriteTimeout: 200 * time.Millisecond, 258 | }) 259 | 260 | // make buf.datas is not empty when close. 261 | for i := 0; i < 100; i++ { 262 | buf.Write("ERROR", "ERROR", "ERROR", "ERROR") 263 | } 264 | 265 | err := buf.Close() 266 | 267 | if len(buf.datas) != 0 { 268 | if err != errErrInput { 269 | t.Errorf( 270 | "TestCloseError want: %v, actual: %v", 271 | errErrInput, 272 | err, 273 | ) 274 | } 275 | } 276 | } 277 | 278 | func TestClose(t *testing.T) { 279 | co := newStringCounter("", 100*time.Millisecond) 280 | 281 | buf := New[string](co, Option[string]{ 282 | Threshold: 100000, 283 | FlushInterval: time.Hour, 284 | WriteTimeout: 200 * time.Millisecond, 285 | }) 286 | 287 | // make buf.datas is not empty when close. 288 | for i := 0; i < 100; i++ { 289 | buf.Write("AAA", "BBB", "CCC", "DDD") 290 | } 291 | 292 | err := buf.Close() 293 | 294 | if len(buf.datas) != 0 { 295 | if err != nil { 296 | t.Errorf( 297 | "TestClose want: %v, actual: %v", 298 | nil, 299 | err, 300 | ) 301 | } 302 | } 303 | } 304 | 305 | func TestCloseTwice(t *testing.T) { 306 | co := newStringCounter("", 100*time.Millisecond) 307 | 308 | buf := New[string](co, Option[string]{ 309 | Threshold: 100000, 310 | FlushInterval: time.Hour, 311 | WriteTimeout: 200 * time.Millisecond, 312 | }) 313 | 314 | buf.Close() 315 | err := buf.Close() 316 | 317 | if len(buf.datas) != 0 { 318 | if err != nil { 319 | t.Errorf( 320 | "TestCloseTwice want: %v, actual: %v", 321 | ErrClosed, 322 | err, 323 | ) 324 | } 325 | } 326 | } 327 | 328 | func TestInternalFlushFlushError(t *testing.T) { 329 | co := newStringCounter("ERROR", time.Millisecond) 330 | 331 | var ev errRecoder 332 | 333 | buf := New[string](co, Option[string]{ 334 | Threshold: 10, 335 | FlushInterval: time.Hour, 336 | WriteTimeout: 200 * time.Millisecond, 337 | ErrHandler: ev.log, 338 | }) 339 | 340 | errElements := []string{"ERROR"} 341 | 342 | buf.internalFlush(errElements) 343 | 344 | if ev.err != errErrInput || !reflect.DeepEqual(ev.elements, errElements) { 345 | t.Errorf( 346 | "TestInternalFlushFlushError want: %v, %v, actual: %v, %v", 347 | errErrInput, errElements, 348 | ev.err, ev.elements, 349 | ) 350 | } 351 | } 352 | 353 | func TestInternalFlushTimeout(t *testing.T) { 354 | co := newStringCounter("", time.Hour) 355 | 356 | var ev errRecoder 357 | 358 | buf := New[string](co, Option[string]{ 359 | Threshold: 10, 360 | FlushInterval: time.Hour, 361 | FlushTimeout: 200 * time.Millisecond, 362 | ErrHandler: ev.log, 363 | }) 364 | 365 | elements := []string{"ASD"} 366 | 367 | buf.internalFlush(elements) 368 | 369 | if ev.err != ErrFlushTimeout || !reflect.DeepEqual(ev.elements, elements) { 370 | t.Errorf( 371 | "TestInternalFlushTimeout want: %v, %v, actual: %v, %v", 372 | errErrInput, elements, 373 | ev.err, ev.elements, 374 | ) 375 | } 376 | } 377 | 378 | func TestFlushFunc(t *testing.T) { 379 | co := newStringCounter("", time.Microsecond) 380 | 381 | flushfunc := co.Flush 382 | 383 | buf := New[string](FlushFunc[string](flushfunc), Option[string]{Threshold: 1}) 384 | defer buf.Close() 385 | 386 | buf.Write("asd") 387 | } 388 | 389 | func TestDefaultErrHandler(t *testing.T) { 390 | DefaultErrHandler(errors.New("mock_error"), []string{"A", "B", "C"}) 391 | } 392 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package buffer_test 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | buffer "github.com/woorui/async-buffer" 8 | ) 9 | 10 | // pp implements Flusher interface 11 | type pp struct{} 12 | 13 | func (p *pp) Flush(strs []string) error { 14 | return print(strs) 15 | } 16 | 17 | func print(strs []string) error { 18 | for _, s := range strs { 19 | fmt.Printf("%s ", s) 20 | } 21 | return nil 22 | } 23 | 24 | func Example() { 25 | // can also call buffer.FlushFunc` to adapt a function to Flusher 26 | buf := buffer.New[string](&pp{}, buffer.Option[string]{ 27 | Threshold: 1, 28 | FlushInterval: 3 * time.Second, 29 | WriteTimeout: time.Second, 30 | FlushTimeout: time.Second, 31 | ErrHandler: func(err error, t []string) { fmt.Printf("err: %v, ele: %v", err, t) }, 32 | }) 33 | // data maybe loss if Close() is not be called 34 | defer buf.Close() 35 | 36 | buf.Write("a", "b", "c", "d", "e", "f") 37 | 38 | // Output: 39 | // a b c d e f 40 | } 41 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/woorui/async-buffer 2 | 3 | go 1.18 4 | -------------------------------------------------------------------------------- /mock_flusher.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | var errErrInput = errors.New("error input") 10 | 11 | // stringCounter counts how many times does string appears. 12 | type stringCounter struct { 13 | mu *sync.Mutex 14 | m map[string]int 15 | errInput string 16 | mockFlushCost time.Duration 17 | } 18 | 19 | func newStringCounter(errInput string, flushCost time.Duration) *stringCounter { 20 | return &stringCounter{ 21 | mu: &sync.Mutex{}, 22 | m: make(map[string]int), 23 | errInput: errInput, 24 | mockFlushCost: flushCost, 25 | } 26 | } 27 | 28 | // Flush implements Flusher interface. 29 | func (c *stringCounter) Flush(str []string) error { 30 | time.Sleep(c.mockFlushCost) 31 | c.mu.Lock() 32 | defer c.mu.Unlock() 33 | for _, v := range str { 34 | if v == c.errInput && c.errInput != "" { 35 | return errErrInput 36 | } 37 | vv := c.m[v] 38 | vv++ 39 | c.m[v] = vv 40 | } 41 | return nil 42 | } 43 | 44 | func (c *stringCounter) result() map[string]int { 45 | c.mu.Lock() 46 | defer c.mu.Unlock() 47 | result := make(map[string]int, len(c.m)) 48 | for k, v := range c.m { 49 | result[k] = v 50 | } 51 | return result 52 | } 53 | 54 | // errRecoder records err and err elements 55 | type errRecoder struct { 56 | err error 57 | elements []string 58 | } 59 | 60 | func (e *errRecoder) log(err error, elements []string) { 61 | e.err = err 62 | e.elements = elements 63 | } 64 | -------------------------------------------------------------------------------- /mock_flusher_test.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func Test_newStringCounter(t *testing.T) { 10 | type args struct { 11 | errInput string 12 | flushCost time.Duration 13 | elements []string 14 | } 15 | tests := []struct { 16 | name string 17 | args args 18 | want map[string]int 19 | wantErr error 20 | }{ 21 | { 22 | name: "normal", 23 | args: args{ 24 | errInput: "", 25 | flushCost: time.Millisecond, 26 | elements: []string{"A", "B", "C", "D", "D"}, 27 | }, 28 | want: map[string]int{"A": 1, "B": 1, "C": 1, "D": 2}, 29 | wantErr: nil, 30 | }, 31 | { 32 | name: "error", 33 | args: args{ 34 | errInput: "A", 35 | flushCost: time.Millisecond, 36 | elements: []string{"A", "B", "C", "D", "D"}, 37 | }, 38 | want: map[string]int{}, 39 | wantErr: errErrInput, 40 | }, 41 | } 42 | for _, tt := range tests { 43 | t.Run(tt.name, func(t *testing.T) { 44 | co := newStringCounter(tt.args.errInput, tt.args.flushCost) 45 | 46 | gotErr := co.Flush(tt.args.elements) 47 | 48 | if gotErr != tt.wantErr { 49 | t.Errorf("stringCounter got error = %v, want error %v", gotErr, tt.wantErr) 50 | } 51 | 52 | got := co.result() 53 | 54 | if !reflect.DeepEqual(got, tt.want) { 55 | t.Errorf("stringCounter got = %v, want %v", got, tt.want) 56 | } 57 | }) 58 | } 59 | } 60 | --------------------------------------------------------------------------------