├── gopher.png
├── gopher.xcf
├── Makefile
├── suite_test.go
├── .gitignore
├── .github
└── workflows
│ └── go.yml
├── go.mod
├── bench_test.go
├── LICENSE
├── options_test.go
├── options.go
├── buffer.go
├── go.sum
├── README.md
└── buffer_test.go
/gopher.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/globocom/go-buffer/HEAD/gopher.png
--------------------------------------------------------------------------------
/gopher.xcf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/globocom/go-buffer/HEAD/gopher.xcf
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | test:
2 | go run github.com/onsi/ginkgo/ginkgo -keepGoing -progress -timeout 1m -race --randomizeAllSpecs --randomizeSuites
3 |
4 | bench:
5 | go test -bench=. -run=Benchmark
6 |
--------------------------------------------------------------------------------
/suite_test.go:
--------------------------------------------------------------------------------
1 | package buffer_test
2 |
3 | import (
4 | "testing"
5 |
6 | . "github.com/onsi/ginkgo"
7 | . "github.com/onsi/gomega"
8 | )
9 |
10 | func TestGoBuffer(t *testing.T) {
11 | RegisterFailHandler(Fail)
12 | RunSpecs(t, "go-buffer suite")
13 | }
14 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Binaries for programs and plugins
2 | *.exe
3 | *.exe~
4 | *.dll
5 | *.so
6 | *.dylib
7 |
8 | # Test binary, built with `go test -c`
9 | *.test
10 |
11 | # Output of the go coverage tool, specifically when used with LiteIDE
12 | *.out
13 | .idea
14 | # Dependency directories (remove the comment below to include it)
15 | # vendor/
16 |
--------------------------------------------------------------------------------
/.github/workflows/go.yml:
--------------------------------------------------------------------------------
1 | name: Go
2 |
3 | on:
4 | push:
5 | branches: [ master ]
6 | pull_request:
7 | branches: [ master ]
8 |
9 | jobs:
10 |
11 | build:
12 | name: Build
13 | runs-on: ubuntu-latest
14 | steps:
15 |
16 | - name: Set up Go 1.x
17 | uses: actions/setup-go@v2
18 | with:
19 | go-version: 1.24
20 | id: go
21 |
22 | - name: Check out code into the Go module directory
23 | uses: actions/checkout@v2
24 |
25 | - name: Get dependencies
26 | run: |
27 | go get -v -t -d ./...
28 | - name: Build
29 | run: go build -v .
30 |
31 | - name: Test
32 | run: make test
33 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/globocom/go-buffer/v3
2 |
3 | retract (
4 | v3.0.0 // Published prematurely
5 | v3.0.1 // Contains retractions only
6 | )
7 |
8 | go 1.24
9 |
10 | require (
11 | github.com/onsi/ginkgo v1.13.0
12 | github.com/onsi/gomega v1.10.1
13 | )
14 |
15 | require (
16 | github.com/fsnotify/fsnotify v1.4.9 // indirect
17 | github.com/nxadm/tail v1.4.4 // indirect
18 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 // indirect
19 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 // indirect
20 | golang.org/x/text v0.3.2 // indirect
21 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
22 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
23 | gopkg.in/yaml.v2 v2.3.0 // indirect
24 | )
25 |
--------------------------------------------------------------------------------
/bench_test.go:
--------------------------------------------------------------------------------
1 | package buffer_test
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/globocom/go-buffer/v3"
7 | )
8 |
9 | func BenchmarkBuffer(b *testing.B) {
10 | noop := func([]int) {}
11 |
12 | b.Run("push only", func(b *testing.B) {
13 | sut := buffer.New(
14 | noop,
15 | buffer.WithSize(uint(b.N)+1),
16 | )
17 | defer sut.Close()
18 |
19 | for b.Loop() {
20 | err := sut.Push(1)
21 | if err != nil {
22 | b.Fail()
23 | }
24 | }
25 | })
26 |
27 | b.Run("push and flush", func(b *testing.B) {
28 | sut := buffer.New(
29 | noop,
30 | buffer.WithSize(1),
31 | )
32 | defer sut.Close()
33 |
34 | for b.Loop() {
35 | err := sut.Push(1)
36 | if err != nil {
37 | b.Fail()
38 | }
39 | }
40 | })
41 | }
42 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 Globo.com
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/options_test.go:
--------------------------------------------------------------------------------
1 | package buffer_test
2 |
3 | import (
4 | "time"
5 |
6 | . "github.com/onsi/ginkgo"
7 | . "github.com/onsi/gomega"
8 |
9 | "github.com/globocom/go-buffer/v3"
10 | )
11 |
12 | var _ = Describe("Options", func() {
13 | It("sets up size", func() {
14 | // arrange
15 | opts := &buffer.Options{}
16 |
17 | // act
18 | buffer.WithSize(10)(opts)
19 |
20 | // assert
21 | Expect(opts.Size).To(BeIdenticalTo(uint(10)))
22 | })
23 |
24 | It("sets up flush interval", func() {
25 | // arrange
26 | opts := &buffer.Options{}
27 |
28 | // act
29 | buffer.WithFlushInterval(5 * time.Second)(opts)
30 |
31 | // assert
32 | Expect(opts.FlushInterval).To(Equal(5 * time.Second))
33 | })
34 |
35 | It("sets up push timeout", func() {
36 | // arrange
37 | opts := &buffer.Options{}
38 |
39 | // act
40 | buffer.WithPushTimeout(10 * time.Second)(opts)
41 |
42 | // assert
43 | Expect(opts.PushTimeout).To(Equal(10 * time.Second))
44 | })
45 |
46 | It("sets up flush timeout", func() {
47 | // arrange
48 | opts := &buffer.Options{}
49 |
50 | // act
51 | buffer.WithFlushTimeout(15 * time.Second)(opts)
52 |
53 | // assert
54 | Expect(opts.FlushTimeout).To(Equal(15 * time.Second))
55 | })
56 |
57 | It("sets up close timeout", func() {
58 | // arrange
59 | opts := &buffer.Options{}
60 |
61 | // act
62 | buffer.WithCloseTimeout(3 * time.Second)(opts)
63 |
64 | // assert
65 | Expect(opts.CloseTimeout).To(Equal(3 * time.Second))
66 | })
67 | })
68 |
--------------------------------------------------------------------------------
/options.go:
--------------------------------------------------------------------------------
1 | package buffer
2 |
3 | import (
4 | "errors"
5 | "fmt"
6 | "time"
7 | )
8 |
9 | const (
10 | invalidSize = "size cannot be zero"
11 | invalidInterval = "interval must be greater than zero (%s)"
12 | invalidTimeout = "timeout cannot be negative (%s)"
13 | )
14 |
15 | type (
16 | // Configuration options.
17 | Options struct {
18 | Size uint
19 | FlushInterval time.Duration
20 | PushTimeout time.Duration
21 | FlushTimeout time.Duration
22 | CloseTimeout time.Duration
23 | }
24 |
25 | // Option setter.
26 | Option func(*Options)
27 | )
28 |
29 | // WithSize sets the size of the buffer.
30 | func WithSize(size uint) Option {
31 | return func(options *Options) {
32 | options.Size = size
33 | }
34 | }
35 |
36 | // WithFlushInterval sets the interval between automatic flushes.
37 | func WithFlushInterval(interval time.Duration) Option {
38 | return func(options *Options) {
39 | options.FlushInterval = interval
40 | }
41 | }
42 |
43 | // WithPushTimeout sets how long a push should wait before giving up.
44 | func WithPushTimeout(timeout time.Duration) Option {
45 | return func(options *Options) {
46 | options.PushTimeout = timeout
47 | }
48 | }
49 |
50 | // WithFlushTimeout sets how long a manual flush should wait before giving up.
51 | func WithFlushTimeout(timeout time.Duration) Option {
52 | return func(options *Options) {
53 | options.FlushTimeout = timeout
54 | }
55 | }
56 |
57 | // WithCloseTimeout sets how long
58 | func WithCloseTimeout(timeout time.Duration) Option {
59 | return func(options *Options) {
60 | options.CloseTimeout = timeout
61 | }
62 | }
63 |
64 | func validateOptions(options *Options) error {
65 | if options.Size == 0 {
66 | return errors.New(invalidSize)
67 | }
68 | if options.FlushInterval < 0 {
69 | return fmt.Errorf(invalidInterval, "FlushInterval")
70 | }
71 | if options.PushTimeout < 0 {
72 | return fmt.Errorf(invalidTimeout, "PushTimeout")
73 | }
74 | if options.FlushTimeout < 0 {
75 | return fmt.Errorf(invalidTimeout, "FlushTimeout")
76 | }
77 | if options.CloseTimeout < 0 {
78 | return fmt.Errorf(invalidTimeout, "CloseTimeout")
79 | }
80 |
81 | return nil
82 | }
83 |
84 | func resolveOptions(opts ...Option) *Options {
85 | options := &Options{
86 | Size: 0,
87 | FlushInterval: 0,
88 | PushTimeout: time.Second,
89 | FlushTimeout: time.Second,
90 | CloseTimeout: time.Second,
91 | }
92 |
93 | for _, opt := range opts {
94 | opt(options)
95 | }
96 |
97 | if err := validateOptions(options); err != nil {
98 | panic(err)
99 | }
100 |
101 | return options
102 | }
103 |
--------------------------------------------------------------------------------
/buffer.go:
--------------------------------------------------------------------------------
1 | package buffer
2 |
3 | import (
4 | "errors"
5 | "io"
6 | "time"
7 | )
8 |
9 | var (
10 | // ErrTimeout indicates an operation has timed out.
11 | ErrTimeout = errors.New("operation timed-out")
12 | // ErrClosed indicates the buffer is closed and can no longer be used.
13 | ErrClosed = errors.New("buffer is closed")
14 | )
15 |
16 | type (
17 | // Buffer represents a data buffer that is asynchronously flushed, either manually or automatically.
18 | Buffer[T any] struct {
19 | io.Closer
20 | flushFunc func([]T)
21 | dataCh chan T
22 | flushCh chan struct{}
23 | closeCh chan struct{}
24 | doneCh chan struct{}
25 | options *Options
26 | }
27 | )
28 |
29 | // New creates a new buffer instance with the provided flush function and options.
30 | // It panics if provided with a nil flush function.
31 | func New[T any](flushFunc func([]T), opts ...Option) *Buffer[T] {
32 | if flushFunc == nil {
33 | panic("flush function cannot be nil")
34 | }
35 |
36 | buffer := &Buffer[T]{
37 | flushFunc: flushFunc,
38 | dataCh: make(chan T),
39 | flushCh: make(chan struct{}),
40 | closeCh: make(chan struct{}),
41 | doneCh: make(chan struct{}),
42 | options: resolveOptions(opts...),
43 | }
44 | go buffer.consume()
45 |
46 | return buffer
47 | }
48 |
49 | // Push appends an item to the end of the buffer.
50 | //
51 | // It returns an ErrTimeout if it cannot be performed in a timely fashion, and
52 | // an ErrClosed if the buffer has been closed.
53 | func (buffer *Buffer[T]) Push(item T) error {
54 | if buffer.closed() {
55 | return ErrClosed
56 | }
57 |
58 | select {
59 | case buffer.dataCh <- item:
60 | return nil
61 | case <-time.After(buffer.options.PushTimeout):
62 | return ErrTimeout
63 | }
64 | }
65 |
66 | // Flush outputs the buffer to a permanent destination.
67 | //
68 | // It returns an ErrTimeout if if cannot be performed in a timely fashion, and
69 | // an ErrClosed if the buffer has been closed.
70 | func (buffer *Buffer[T]) Flush() error {
71 | if buffer.closed() {
72 | return ErrClosed
73 | }
74 |
75 | select {
76 | case buffer.flushCh <- struct{}{}:
77 | return nil
78 | case <-time.After(buffer.options.FlushTimeout):
79 | return ErrTimeout
80 | }
81 | }
82 |
83 | // Close flushes the buffer and prevents it from being further used.
84 | //
85 | // It returns an ErrTimeout if if cannot be performed in a timely fashion, and
86 | // an ErrClosed if the buffer has already been closed.
87 | //
88 | // An ErrTimeout can either mean that a flush could not be triggered, or it can
89 | // mean that a flush was triggered but it has not finished yet. In any case it is
90 | // safe to call Close again.
91 | func (buffer *Buffer[T]) Close() error {
92 | if buffer.closed() {
93 | return ErrClosed
94 | }
95 |
96 | select {
97 | case buffer.closeCh <- struct{}{}:
98 | // noop
99 | case <-time.After(buffer.options.CloseTimeout):
100 | return ErrTimeout
101 | }
102 |
103 | select {
104 | case <-buffer.doneCh:
105 | close(buffer.dataCh)
106 | close(buffer.flushCh)
107 | close(buffer.closeCh)
108 | return nil
109 | case <-time.After(buffer.options.CloseTimeout):
110 | return ErrTimeout
111 | }
112 | }
113 |
114 | func (buffer *Buffer[T]) closed() bool {
115 | select {
116 | case <-buffer.doneCh:
117 | return true
118 | default:
119 | return false
120 | }
121 | }
122 |
123 | func (buffer *Buffer[T]) consume() {
124 | count := 0
125 | items := make([]T, buffer.options.Size)
126 | mustFlush := false
127 | ticker, stopTicker := newTicker(buffer.options.FlushInterval)
128 |
129 | isOpen := true
130 | for isOpen {
131 | select {
132 | case item := <-buffer.dataCh:
133 | items[count] = item
134 | count++
135 | mustFlush = count >= len(items)
136 | case <-ticker:
137 | mustFlush = count > 0
138 | case <-buffer.flushCh:
139 | mustFlush = count > 0
140 | case <-buffer.closeCh:
141 | isOpen = false
142 | mustFlush = count > 0
143 | }
144 |
145 | if mustFlush {
146 | stopTicker()
147 | buffer.flushFunc(items[:count])
148 |
149 | count = 0
150 | items = make([]T, buffer.options.Size)
151 | mustFlush = false
152 | ticker, stopTicker = newTicker(buffer.options.FlushInterval)
153 | }
154 | }
155 |
156 | stopTicker()
157 | close(buffer.doneCh)
158 | }
159 |
160 | func newTicker(interval time.Duration) (<-chan time.Time, func()) {
161 | if interval == 0 {
162 | return nil, func() {}
163 | }
164 |
165 | ticker := time.NewTicker(interval)
166 | return ticker.C, ticker.Stop
167 | }
168 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
2 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
3 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
4 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
5 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
6 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
7 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
8 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
9 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
10 | github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
11 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
12 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
13 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
14 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
15 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
16 | github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
17 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
18 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
19 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
20 | github.com/onsi/ginkgo v1.13.0 h1:M76yO2HkZASFjXL0HSoZJ1AYEmQxNJmY41Jx1zNUq1Y=
21 | github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
22 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
23 | github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
24 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
25 | github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
26 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
27 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
28 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
29 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
30 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
31 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
32 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
33 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
34 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
35 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
36 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
37 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
38 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
39 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
40 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
41 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
42 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
43 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
44 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
45 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
46 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
47 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
48 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
49 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
50 | google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
51 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
52 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
53 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
54 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
55 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
56 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
57 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
58 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
59 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
60 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 | 
3 |
4 |
5 | 
6 |
7 | 
8 |
9 | 
10 |
11 | 
12 |
13 |
14 |
15 | # go-buffer
16 |
17 | `go-buffer` represents a buffer that asynchronously flushes its contents. It is useful for applications that need to
18 | aggregate data before writing it to an external storage. A buffer is flushed manually, or automatically when it becomes
19 | full or after an interval has elapsed, whichever comes first.
20 |
21 | ## Installation
22 |
23 | go get github.com/globocom/go-buffer
24 |
25 | Go < 1.18:
26 |
27 | go get github.com/globocom/go-buffer@v2
28 |
29 | ## Examples
30 |
31 | > [!NOTE]
32 | > For v2, see [Examples v2](#examples-v2).
33 |
34 | ### Size-triggered flush
35 |
36 | ```go
37 | package main
38 |
39 | import (
40 | "time"
41 |
42 | "github.com/globocom/go-buffer/v3"
43 | )
44 |
45 | func main() {
46 | buff := buffer.New(
47 | // call this function when the buffer needs flushing
48 | func(items []string) {
49 | for _, item := range items {
50 | println(string)
51 | }
52 | },
53 | // buffer can hold up to 5 items
54 | buffer.WithSize(5),
55 | )
56 | // ensure the buffer
57 | defer buff.Close()
58 |
59 | buff.Push("item 1")
60 | buff.Push("item 2")
61 | buff.Push("item 3")
62 | buff.Push("item 4")
63 | buff.Push("item 5")
64 |
65 | // block the current goroutine
66 | time.Sleep(3 * time.Second)
67 |
68 | println("done")
69 | }
70 | ```
71 |
72 | ### Interval-triggered flush
73 |
74 | ```go
75 | package main
76 |
77 | import (
78 | "time"
79 |
80 | "github.com/globocom/go-buffer/v3"
81 | )
82 |
83 | func main() {
84 | buff := buffer.New(
85 | // call this function when the buffer needs flushing
86 | func(items []string) {
87 | for _, item := range items {
88 | println(item)
89 | }
90 | },
91 | // buffer can hold up to 5 items
92 | buffer.WithSize(5),
93 | // buffer will be flushed every second, regardless of
94 | // how many items were pushed
95 | buffer.WithFlushInterval(time.Second),
96 | )
97 | defer buff.Close()
98 |
99 | buff.Push("item 1")
100 | buff.Push("item 2")
101 | buff.Push("item 3")
102 |
103 | // block the current goroutine
104 | time.Sleep(3 * time.Second)
105 |
106 | println("done")
107 | }
108 | ```
109 |
110 | ### Manual flush
111 |
112 | ```go
113 | package main
114 |
115 | import (
116 | "time"
117 |
118 | "github.com/globocom/go-buffer/v3"
119 | )
120 |
121 | func main() {
122 | buff := buffer.New(
123 | // call this function when the buffer needs flushing
124 | func(items []string) {
125 | for _, item := range items {
126 | println(item)
127 | }
128 | },
129 | // buffer can hold up to 5 items
130 | buffer.WithSize(5),
131 | )
132 | defer buff.Close()
133 |
134 | buff.Push("item 1")
135 | buff.Push("item 2")
136 | buff.Push("item 3")
137 |
138 | // block the current goroutine
139 | time.Sleep(3*time.Second)
140 |
141 | buff.Flush()
142 | println("done")
143 | }
144 | ```
145 |
146 | ## Examples v2
147 |
148 | ### Size-triggered flush
149 |
150 | ```go
151 | package main
152 |
153 | import (
154 | "time"
155 |
156 | "github.com/globocom/go-buffer/v2"
157 | )
158 |
159 | func main() {
160 | buff := buffer.New(
161 | // buffer can hold up to 5 items
162 | buffer.WithSize(5),
163 | // call this function when the buffer needs flushing
164 | buffer.WithFlusher(buffer.FlusherFunc(func(items []interface{}) {
165 | for _, item := range items {
166 | println(item.(string))
167 | }
168 | })),
169 | )
170 | // ensure the buffer
171 | defer buff.Close()
172 |
173 | buff.Push("item 1")
174 | buff.Push("item 2")
175 | buff.Push("item 3")
176 | buff.Push("item 4")
177 | buff.Push("item 5")
178 |
179 | // block the current goroutine
180 | time.Sleep(3 * time.Second)
181 |
182 | println("done")
183 | }
184 | ```
185 |
186 | ### Interval-triggered flush
187 |
188 | ```go
189 | package main
190 |
191 | import (
192 | "time"
193 |
194 | "github.com/globocom/go-buffer/v2"
195 | )
196 |
197 | func main() {
198 | buff := buffer.New(
199 | // buffer can hold up to 5 items
200 | buffer.WithSize(5),
201 | // buffer will be flushed every second, regardless of
202 | // how many items were pushed
203 | buffer.WithFlushInterval(time.Second),
204 | // call this function when the buffer needs flushing
205 | buffer.WithFlusher(buffer.FlusherFunc(func(items []interface{}) {
206 | for _, item := range items {
207 | println(item.(string))
208 | }
209 | })),
210 | )
211 | defer buff.Close()
212 |
213 | buff.Push("item 1")
214 | buff.Push("item 2")
215 | buff.Push("item 3")
216 |
217 | // block the current goroutine
218 | time.Sleep(3 * time.Second)
219 |
220 | println("done")
221 | }
222 | ```
223 |
224 | ### Manual flush
225 |
226 | ```go
227 | package main
228 |
229 | import (
230 | "time"
231 |
232 | "github.com/globocom/go-buffer/v2"
233 | )
234 |
235 | func main() {
236 | buff := buffer.New(
237 | // buffer can hold up to 5 items
238 | buffer.WithSize(5),
239 | // call this function when the buffer needs flushing
240 | buffer.WithFlusher(buffer.FlusherFunc(func(items []interface{}) {
241 | for _, item := range items {
242 | println(item.(string))
243 | }
244 | })),
245 | )
246 | defer buff.Close()
247 |
248 | buff.Push("item 1")
249 | buff.Push("item 2")
250 | buff.Push("item 3")
251 |
252 | // block the current goroutine
253 | time.Sleep(3*time.Second)
254 |
255 | buff.Flush()
256 | println("done")
257 | }
258 | ```
259 |
260 | ## Documentation
261 |
262 | Visit [Pkg.go.dev](https://pkg.go.dev/github.com/globocom/go-buffer) for full documentation.
263 |
264 | ## License
265 |
266 | [MIT License](https://github.com/globocom/go-buffer/blob/master/LICENSE)
267 |
--------------------------------------------------------------------------------
/buffer_test.go:
--------------------------------------------------------------------------------
1 | package buffer_test
2 |
3 | import (
4 | "time"
5 |
6 | . "github.com/onsi/ginkgo"
7 | . "github.com/onsi/gomega"
8 |
9 | "github.com/globocom/go-buffer/v3"
10 | )
11 |
12 | var _ = Describe("Buffer", func() {
13 | var flusher *MockFlusher
14 |
15 | BeforeEach(func() {
16 | flusher = NewMockFlusher()
17 | })
18 |
19 | Context("Constructor", func() {
20 | It("creates a new Buffer instance", func() {
21 | // act
22 | sut := buffer.New(
23 | flusher.Flush,
24 | buffer.WithSize(10),
25 | )
26 |
27 | // assert
28 | Expect(sut).NotTo(BeNil())
29 | })
30 |
31 | It("panics when provided an invalid flusher", func() {
32 | Expect(func() {
33 | buffer.New[string](
34 | nil,
35 | buffer.WithSize(1),
36 | )
37 | }).To(Panic())
38 | })
39 |
40 | Context("invalid options", func() {
41 | It("panics when provided an invalid size", func() {
42 | Expect(func() {
43 | buffer.New(
44 | flusher.Flush,
45 | buffer.WithSize(0),
46 | )
47 | }).To(Panic())
48 | })
49 |
50 | It("panics when provided an invalid flush interval", func() {
51 | Expect(func() {
52 | buffer.New(
53 | flusher.Flush,
54 | buffer.WithSize(1),
55 | buffer.WithFlushInterval(-1),
56 | )
57 | }).To(Panic())
58 | })
59 |
60 | It("panics when provided an invalid push timeout", func() {
61 | Expect(func() {
62 | buffer.New(
63 | flusher.Flush,
64 | buffer.WithSize(1),
65 | buffer.WithPushTimeout(-1),
66 | )
67 | }).To(Panic())
68 | })
69 |
70 | It("panics when provided an invalid flush timeout", func() {
71 | Expect(func() {
72 | buffer.New(
73 | flusher.Flush,
74 | buffer.WithSize(1),
75 | buffer.WithFlushTimeout(-1),
76 | )
77 | }).To(Panic())
78 | })
79 |
80 | It("panics when provided an invalid close timeout", func() {
81 | Expect(func() {
82 | buffer.New(
83 | flusher.Flush,
84 | buffer.WithSize(1),
85 | buffer.WithCloseTimeout(-1),
86 | )
87 | }).To(Panic())
88 | })
89 | })
90 | })
91 |
92 | Context("Pushing", func() {
93 | It("pushes items into the buffer when Push is called", func() {
94 | // arrange
95 | sut := buffer.New(
96 | flusher.Flush,
97 | buffer.WithSize(3),
98 | )
99 |
100 | // act
101 | err1 := sut.Push("a")
102 | err2 := sut.Push("b")
103 | err3 := sut.Push("c")
104 |
105 | // assert
106 | Expect(err1).To(Succeed())
107 | Expect(err2).To(Succeed())
108 | Expect(err3).To(Succeed())
109 | })
110 |
111 | It("fails when Push cannot execute in a timely fashion", func() {
112 | // arrange
113 | flusher.Func = func() { select {} }
114 | sut := buffer.New(
115 | flusher.Flush,
116 | buffer.WithSize(2),
117 | buffer.WithPushTimeout(time.Second),
118 | )
119 |
120 | // act
121 | err1 := sut.Push("a")
122 | err2 := sut.Push("b")
123 | err3 := sut.Push("c")
124 |
125 | // assert
126 | Expect(err1).To(Succeed())
127 | Expect(err2).To(Succeed())
128 | Expect(err3).To(MatchError(buffer.ErrTimeout))
129 | })
130 |
131 | It("fails when the buffer is closed", func() {
132 | // arrange
133 | sut := buffer.New(
134 | flusher.Flush,
135 | buffer.WithSize(2),
136 | )
137 | _ = sut.Close()
138 |
139 | // act
140 | err := sut.Push("a")
141 |
142 | // assert
143 | Expect(err).To(MatchError(buffer.ErrClosed))
144 | })
145 | })
146 |
147 | Context("Flushing", func() {
148 | It("flushes the buffer when it fills up", func(done Done) {
149 | // arrange
150 | sut := buffer.New(
151 | flusher.Flush,
152 | buffer.WithSize(5),
153 | )
154 |
155 | // act
156 | _ = sut.Push("a")
157 | _ = sut.Push("b")
158 | _ = sut.Push("c")
159 | _ = sut.Push("d")
160 | _ = sut.Push("e")
161 |
162 | // assert
163 | result := <-flusher.Done
164 | Expect(result.Items).To(ConsistOf("a", "b", "c", "d", "e"))
165 | close(done)
166 | })
167 |
168 | It("flushes the buffer when the provided interval has elapsed", func(done Done) {
169 | // arrange
170 | interval := 3 * time.Second
171 | start := time.Now()
172 | sut := buffer.New(
173 | flusher.Flush,
174 | buffer.WithSize(5),
175 | buffer.WithFlushInterval(interval),
176 | )
177 |
178 | // act
179 | _ = sut.Push("a")
180 |
181 | // assert
182 | result := <-flusher.Done
183 | Expect(result.Items).To(ConsistOf("a"))
184 | Expect(result.Time).To(BeTemporally("~", start, interval+time.Second))
185 | close(done)
186 | }, 5)
187 |
188 | It("flushes the buffer when Flush is called", func(done Done) {
189 | // arrange
190 | sut := buffer.New(
191 | flusher.Flush,
192 | buffer.WithSize(3),
193 | )
194 | _ = sut.Push("a")
195 | _ = sut.Push("b")
196 |
197 | // act
198 | err := sut.Flush()
199 |
200 | // assert
201 | result := <-flusher.Done
202 | Expect(err).To(Succeed())
203 | Expect(result.Items).To(ConsistOf("a", "b"))
204 | close(done)
205 | })
206 |
207 | It("fails when Flush cannot execute in a timely fashion", func() {
208 | // arrange
209 | flusher.Func = func() { time.Sleep(3 * time.Second) }
210 | sut := buffer.New(
211 | flusher.Flush,
212 | buffer.WithSize(1),
213 | buffer.WithFlushTimeout(time.Second),
214 | )
215 | _ = sut.Push("a")
216 |
217 | // act
218 | err := sut.Flush()
219 |
220 | // assert
221 | Expect(err).To(MatchError(buffer.ErrTimeout))
222 | })
223 |
224 | It("fails when the buffer is closed", func() {
225 | // arrange
226 | sut := buffer.New[string](
227 | flusher.Flush,
228 | buffer.WithSize(2),
229 | )
230 | _ = sut.Close()
231 |
232 | // act
233 | err := sut.Flush()
234 |
235 | // assert
236 | Expect(err).To(MatchError(buffer.ErrClosed))
237 | })
238 | })
239 |
240 | Context("Closing", func() {
241 | It("flushes the buffer and closes it when Close is called", func(done Done) {
242 | // arrange
243 | sut := buffer.New(
244 | flusher.Flush,
245 | buffer.WithSize(3),
246 | )
247 | _ = sut.Push("a")
248 | _ = sut.Push("b")
249 |
250 | // act
251 | err := sut.Close()
252 |
253 | // assert
254 | result := <-flusher.Done
255 | Expect(err).To(Succeed())
256 | Expect(result.Items).To(ConsistOf("a", "b"))
257 | close(done)
258 | })
259 |
260 | It("fails when Close cannot execute in a timely fashion", func() {
261 | // arrange
262 | flusher.Func = func() { time.Sleep(2 * time.Second) }
263 |
264 | sut := buffer.New(
265 | flusher.Flush,
266 | buffer.WithSize(1),
267 | buffer.WithCloseTimeout(time.Second),
268 | )
269 | _ = sut.Push("a")
270 |
271 | // act
272 | err := sut.Close()
273 |
274 | // assert
275 | Expect(err).To(MatchError(buffer.ErrTimeout))
276 | })
277 |
278 | It("fails when the buffer is closed", func() {
279 | // arrange
280 | flusher.Func = func() { time.Sleep(2 * time.Second) }
281 |
282 | sut := buffer.New(
283 | flusher.Flush,
284 | buffer.WithSize(1),
285 | buffer.WithCloseTimeout(time.Second),
286 | )
287 | _ = sut.Close()
288 |
289 | // act
290 | err := sut.Close()
291 |
292 | // assert
293 | Expect(err).To(MatchError(buffer.ErrClosed))
294 | })
295 |
296 | It("allows Close to be called again if it fails", func() {
297 | // arrange
298 | flusher.Func = func() { time.Sleep(2 * time.Second) }
299 |
300 | sut := buffer.New(
301 | flusher.Flush,
302 | buffer.WithSize(1),
303 | buffer.WithCloseTimeout(time.Second),
304 | )
305 | _ = sut.Push("a")
306 |
307 | // act
308 | err1 := sut.Close()
309 | time.Sleep(time.Second)
310 | err2 := sut.Close()
311 |
312 | // assert
313 | Expect(err1).To(MatchError(buffer.ErrTimeout))
314 | Expect(err2).To(Succeed())
315 | })
316 | })
317 | })
318 |
319 | type (
320 | MockFlusher struct {
321 | Done chan *WriteCall
322 | Func func()
323 | }
324 |
325 | WriteCall struct {
326 | Time time.Time
327 | Items []string
328 | }
329 | )
330 |
331 | func (flusher *MockFlusher) Flush(items []string) {
332 | call := &WriteCall{
333 | Time: time.Now(),
334 | Items: items,
335 | }
336 |
337 | if flusher.Func != nil {
338 | flusher.Func()
339 | }
340 |
341 | flusher.Done <- call
342 | }
343 |
344 | func NewMockFlusher() *MockFlusher {
345 | return &MockFlusher{
346 | Done: make(chan *WriteCall, 1),
347 | }
348 | }
349 |
--------------------------------------------------------------------------------