├── doc.go ├── go.mod ├── go.sum ├── .github └── workflows │ └── main.yml ├── bufpipe_example_test.go ├── LICENSE ├── README.md ├── CREDITS ├── bufpipe_test.go └── bufpipe.go /doc.go: -------------------------------------------------------------------------------- 1 | // Package bufpipe provides a IO pipe, has variable-sized buffer. 2 | package bufpipe 3 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/acomagu/bufpipe 2 | 3 | go 1.12 4 | 5 | require github.com/matryer/is v1.2.0 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= 2 | github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= 3 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: main 2 | on: [push] 3 | jobs: 4 | test: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v3 8 | - uses: actions/setup-go@v3 9 | - run: go test -v ./... 10 | -------------------------------------------------------------------------------- /bufpipe_example_test.go: -------------------------------------------------------------------------------- 1 | package bufpipe_test 2 | 3 | import ( 4 | "io" 5 | "os" 6 | 7 | "github.com/acomagu/bufpipe" 8 | ) 9 | 10 | func Example() { 11 | r, w := bufpipe.New(nil) 12 | 13 | done := make(chan struct{}) 14 | go func() { 15 | io.Copy(os.Stdout, r) 16 | done <- struct{}{} 17 | }() 18 | 19 | io.WriteString(w, "abc") 20 | io.WriteString(w, "def") 21 | w.Close() 22 | <-done 23 | // Output: abcdef 24 | } 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 acomagu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bufpipe: Buffered Pipe 2 | 3 | [![CircleCI](https://img.shields.io/github/actions/workflow/status/acomagu/bufpipe/main.yml.svg?style=flat-square)](https://github.com/acomagu/bufpipe/actions/workflows/main.yml) [![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg?style=flat-square)](https://godoc.org/github.com/acomagu/bufpipe) 4 | 5 | The buffered version of io.Pipe. It's safe for concurrent use. 6 | 7 | ## How does it differ from io.Pipe? 8 | 9 | Writes never block because the pipe has variable-sized buffer. 10 | 11 | ```Go 12 | r, w := bufpipe.New(nil) 13 | io.WriteString(w, "abc") // No blocking. 14 | io.WriteString(w, "def") // No blocking, too. 15 | w.Close() 16 | io.Copy(os.Stdout, r) 17 | // Output: abcdef 18 | ``` 19 | 20 | [Playground](https://play.golang.org/p/PdyBAS3pVob) 21 | 22 | ## How does it differ from bytes.Buffer? 23 | 24 | Reads block if the internal buffer is empty until the writer is closed. 25 | 26 | ```Go 27 | r, w := bufpipe.New(nil) 28 | 29 | done := make(chan struct{}) 30 | go func() { 31 | io.Copy(os.Stdout, r) // The reads block until the writer is closed. 32 | done <- struct{}{} 33 | }() 34 | 35 | io.WriteString(w, "abc") 36 | io.WriteString(w, "def") 37 | w.Close() 38 | <-done 39 | // Output: abcdef 40 | ``` 41 | 42 | [Playground](https://play.golang.org/p/UppmyLeRgX6) 43 | 44 | ## Contribution 45 | 46 | ### Generate CREDITS 47 | 48 | The [CREDITS](./CREDITS) file are generated by [gocredits](https://github.com/Songmu/gocredits). Update it when the dependencies are changed. 49 | 50 | ``` 51 | $ gocredits > CREDITS 52 | ``` 53 | -------------------------------------------------------------------------------- /CREDITS: -------------------------------------------------------------------------------- 1 | Go (the standard library) 2 | https://golang.org/ 3 | ---------------------------------------------------------------- 4 | Copyright (c) 2009 The Go Authors. All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are 8 | met: 9 | 10 | * Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following disclaimer 14 | in the documentation and/or other materials provided with the 15 | distribution. 16 | * Neither the name of Google Inc. nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | ================================================================ 33 | 34 | github.com/matryer/is 35 | https://github.com/matryer/is 36 | ---------------------------------------------------------------- 37 | MIT License 38 | 39 | Copyright (c) 2017-2018 Mat Ryer 40 | 41 | Permission is hereby granted, free of charge, to any person obtaining a copy 42 | of this software and associated documentation files (the "Software"), to deal 43 | in the Software without restriction, including without limitation the rights 44 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 45 | copies of the Software, and to permit persons to whom the Software is 46 | furnished to do so, subject to the following conditions: 47 | 48 | The above copyright notice and this permission notice shall be included in all 49 | copies or substantial portions of the Software. 50 | 51 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 52 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 53 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 54 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 55 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 56 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 57 | SOFTWARE. 58 | 59 | ================================================================ 60 | 61 | -------------------------------------------------------------------------------- /bufpipe_test.go: -------------------------------------------------------------------------------- 1 | package bufpipe_test 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "sort" 8 | "testing" 9 | "time" 10 | 11 | "github.com/acomagu/bufpipe" 12 | "github.com/matryer/is" 13 | ) 14 | 15 | func TestPipeWriter_NoBlocking(t *testing.T) { 16 | is := is.New(t) 17 | 18 | r, w := bufpipe.New(nil) 19 | io.WriteString(w, "abc") 20 | io.WriteString(w, "def") 21 | w.Close() 22 | 23 | b, err := ioutil.ReadAll(r) 24 | is.NoErr(err) 25 | is.Equal(b, []byte("abcdef")) 26 | } 27 | 28 | func TestMultiBlocking(t *testing.T) { 29 | is := is.New(t) 30 | 31 | results := make(chan []byte) 32 | block := func(r io.Reader) { 33 | b := make([]byte, 3) 34 | n, err := r.Read(b) 35 | is.NoErr(err) 36 | results <- b[:n] 37 | } 38 | 39 | r, w := bufpipe.New(nil) 40 | go block(r) 41 | go block(r) 42 | go block(r) 43 | 44 | time.Sleep(time.Millisecond) // Ensure blocking. 45 | 46 | data := []string{"abc", "def", "ghi"} 47 | for _, s := range data { 48 | n, err := w.Write([]byte(s)) 49 | is.NoErr(err) 50 | is.Equal(n, 3) 51 | } 52 | 53 | var ss []string 54 | for i := 0; i < 3; i++ { 55 | ss = append(ss, string(<-results)) 56 | } 57 | sort.Strings(ss) 58 | is.Equal(ss, data) 59 | } 60 | 61 | func TestPipeWriter_Close(t *testing.T) { 62 | is := is.New(t) 63 | 64 | r, w := bufpipe.New([]byte("abc")) 65 | n, err := w.Write([]byte("def")) 66 | is.NoErr(err) 67 | is.Equal(n, 3) 68 | 69 | is.NoErr(w.Close()) 70 | 71 | buf := make([]byte, 3) 72 | n, err = r.Read(buf) 73 | is.NoErr(err) 74 | is.Equal(buf[:n], []byte("abc")) 75 | 76 | n, err = r.Read(buf) 77 | is.NoErr(err) 78 | is.Equal(buf[:n], []byte("def")) 79 | 80 | _, err = r.Read(buf) 81 | is.Equal(err, io.EOF) 82 | } 83 | 84 | func TestPipeWriter_CloseWithError(t *testing.T) { 85 | is := is.New(t) 86 | 87 | r, w := bufpipe.New([]byte("abc")) 88 | n, err := w.Write([]byte("def")) 89 | is.NoErr(err) 90 | is.Equal(n, 3) 91 | 92 | expect := fmt.Errorf("original error") 93 | is.NoErr(w.CloseWithError(expect)) 94 | 95 | buf := make([]byte, 3) 96 | n, err = r.Read(buf) 97 | is.NoErr(err) 98 | is.Equal(buf[:n], []byte("abc")) 99 | 100 | n, err = r.Read(buf) 101 | is.NoErr(err) 102 | is.Equal(buf[:n], []byte("def")) 103 | 104 | _, err = r.Read(buf) 105 | is.Equal(err, expect) 106 | } 107 | 108 | func TestPipeReader_Close(t *testing.T) { 109 | is := is.New(t) 110 | 111 | r, w := bufpipe.New([]byte("abc")) 112 | is.NoErr(r.Close()) 113 | 114 | n, err := w.Write([]byte("abc")) 115 | is.Equal(err, bufpipe.ErrClosedPipe) 116 | is.Equal(n, 0) 117 | } 118 | 119 | func TestPipeReader_CloseWithError(t *testing.T) { 120 | is := is.New(t) 121 | 122 | expect := fmt.Errorf("original error") 123 | 124 | r, w := bufpipe.New([]byte("abc")) 125 | is.NoErr(r.CloseWithError(expect)) 126 | 127 | n, err := w.Write([]byte("abc")) 128 | is.Equal(err, expect) 129 | is.Equal(n, 0) 130 | } 131 | 132 | func TestPipeReader_WriterCloseNoDeadlock(t *testing.T) { 133 | r, w := bufpipe.New(nil) 134 | 135 | done := make(chan struct{}) 136 | go func(t *testing.T) { 137 | buf := make([]byte, 800) 138 | r.Read(buf) 139 | done <- struct{}{} 140 | }(t) 141 | 142 | time.Sleep(300 * time.Millisecond) 143 | w.Close() 144 | 145 | <-done 146 | } 147 | 148 | -------------------------------------------------------------------------------- /bufpipe.go: -------------------------------------------------------------------------------- 1 | package bufpipe 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io" 7 | "sync" 8 | ) 9 | 10 | // ErrClosedPipe is the error used for read or write operations on a closed pipe. 11 | var ErrClosedPipe = errors.New("bufpipe: read/write on closed pipe") 12 | 13 | type pipe struct { 14 | cond *sync.Cond 15 | buf *bytes.Buffer 16 | rerr, werr error 17 | } 18 | 19 | // A PipeReader is the read half of a pipe. 20 | type PipeReader struct { 21 | *pipe 22 | } 23 | 24 | // A PipeWriter is the write half of a pipe. 25 | type PipeWriter struct { 26 | *pipe 27 | } 28 | 29 | // New creates a synchronous pipe using buf as its initial contents. It can be 30 | // used to connect code expecting an io.Reader with code expecting an io.Writer. 31 | // 32 | // Unlike io.Pipe, writes never block because the internal buffer has variable 33 | // size. Reads block only when the buffer is empty. 34 | // 35 | // It is safe to call Read and Write in parallel with each other or with Close. 36 | // Parallel calls to Read and parallel calls to Write are also safe: the 37 | // individual calls will be gated sequentially. 38 | // 39 | // The new pipe takes ownership of buf, and the caller should not use buf after 40 | // this call. New is intended to prepare a PipeReader to read existing data. It 41 | // can also be used to set the initial size of the internal buffer for writing. 42 | // To do that, buf should have the desired capacity but a length of zero. 43 | func New(buf []byte) (*PipeReader, *PipeWriter) { 44 | p := &pipe{ 45 | buf: bytes.NewBuffer(buf), 46 | cond: sync.NewCond(new(sync.Mutex)), 47 | } 48 | return &PipeReader{ 49 | pipe: p, 50 | }, &PipeWriter{ 51 | pipe: p, 52 | } 53 | } 54 | 55 | // Read implements the standard Read interface: it reads data from the pipe, 56 | // reading from the internal buffer, otherwise blocking until a writer arrives 57 | // or the write end is closed. If the write end is closed with an error, that 58 | // error is returned as err; otherwise err is io.EOF. 59 | func (r *PipeReader) Read(data []byte) (int, error) { 60 | r.cond.L.Lock() 61 | defer r.cond.L.Unlock() 62 | 63 | RETRY: 64 | n, err := r.buf.Read(data) 65 | // If not closed and no read, wait for writing. 66 | if err == io.EOF && r.rerr == nil && n == 0 { 67 | r.cond.Wait() 68 | goto RETRY 69 | } 70 | if err == io.EOF { 71 | return n, r.rerr 72 | } 73 | return n, err 74 | } 75 | 76 | // Close closes the reader; subsequent writes from the write half of the pipe 77 | // will return error ErrClosedPipe. 78 | func (r *PipeReader) Close() error { 79 | return r.CloseWithError(nil) 80 | } 81 | 82 | // CloseWithError closes the reader; subsequent writes to the write half of the 83 | // pipe will return the error err. 84 | func (r *PipeReader) CloseWithError(err error) error { 85 | r.cond.L.Lock() 86 | defer r.cond.L.Unlock() 87 | 88 | if err == nil { 89 | err = ErrClosedPipe 90 | } 91 | r.werr = err 92 | return nil 93 | } 94 | 95 | // Write implements the standard Write interface: it writes data to the internal 96 | // buffer. If the read end is closed with an error, that err is returned as err; 97 | // otherwise err is ErrClosedPipe. 98 | func (w *PipeWriter) Write(data []byte) (int, error) { 99 | w.cond.L.Lock() 100 | defer w.cond.L.Unlock() 101 | 102 | if w.werr != nil { 103 | return 0, w.werr 104 | } 105 | 106 | n, err := w.buf.Write(data) 107 | w.cond.Signal() 108 | return n, err 109 | } 110 | 111 | // Close closes the writer; subsequent reads from the read half of the pipe will 112 | // return io.EOF once the internal buffer get empty. 113 | func (w *PipeWriter) Close() error { 114 | return w.CloseWithError(nil) 115 | } 116 | 117 | // Close closes the writer; subsequent reads from the read half of the pipe will 118 | // return err once the internal buffer get empty. 119 | func (w *PipeWriter) CloseWithError(err error) error { 120 | w.cond.L.Lock() 121 | defer w.cond.L.Unlock() 122 | 123 | if err == nil { 124 | err = io.EOF 125 | } 126 | w.rerr = err 127 | w.cond.Broadcast() 128 | return nil 129 | } 130 | --------------------------------------------------------------------------------