├── unlinkfile_unix.go ├── License ├── go.mod ├── README.md ├── .github └── workflows │ └── ci.yml ├── go.sum ├── unlinkfile_windows.go ├── pipeat.go └── pipeat_test.go /unlinkfile_unix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package pipeat 4 | 5 | import ( 6 | "os" 7 | ) 8 | 9 | func unlinkFile(file *os.File) (*os.File, error) { 10 | err := os.Remove(file.Name()) 11 | return file, err 12 | } 13 | -------------------------------------------------------------------------------- /License: -------------------------------------------------------------------------------- 1 | CC0 ; http://creativecommons.org/publicdomain/zero/1.0/ 2 | 3 | To the extent possible under law, John Eikenberry has waived all copyright and 4 | related or neighboring rights to aws-tools. This work is published from: United 5 | States. 6 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/eikenb/pipeat 2 | 3 | go 1.24.0 4 | 5 | require ( 6 | github.com/stretchr/testify v1.11.1 7 | golang.org/x/sys v0.37.0 8 | ) 9 | 10 | require ( 11 | github.com/davecgh/go-spew v1.1.1 // indirect 12 | github.com/pmezard/go-difflib v1.0.0 // indirect 13 | gopkg.in/yaml.v3 v3.0.1 // indirect 14 | ) 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | PipeAt 2 | ====== 3 | 4 | Works like io.Pipe() but allows use of ReadAt/WriteAt and asynchronous 5 | operations. Useful for connecting IO pipelines where one or both ends require 6 | offset based file access. 7 | 8 | [![GoDoc](http://godoc.org/github.com/eikenb/pipeat?status.svg)](http://godoc.org/github.com/eikenb/pipeat) 9 | [![Build Status](https://github.com/eikenb/pipeat/actions/workflows/ci.yml/badge.svg)](https://github.com/eikenb/pipeat/actions) 10 | 11 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | run-tests: 9 | name: Run test cases 10 | runs-on: ${{ matrix.os }} 11 | strategy: 12 | matrix: 13 | os: [ubuntu-latest, macos-latest, windows-latest] 14 | go: [^1] 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v2 21 | with: 22 | go-version: ${{ matrix.go }} 23 | 24 | - name: Run tests 25 | run: | 26 | go test -v . 27 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 5 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 6 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 7 | golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= 8 | golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 9 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 10 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 11 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 12 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 13 | -------------------------------------------------------------------------------- /unlinkfile_windows.go: -------------------------------------------------------------------------------- 1 | package pipeat 2 | 3 | import ( 4 | "os" 5 | 6 | "golang.org/x/sys/windows" 7 | ) 8 | 9 | func unlinkFile(file *os.File) (*os.File, error) { 10 | name := file.Name() 11 | err := file.Close() 12 | if err != nil { 13 | return file, err 14 | } 15 | // https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea 16 | handle, err := windows.CreateFile( 17 | windows.StringToUTF16Ptr(name), // File Name 18 | windows.GENERIC_READ|windows.GENERIC_WRITE|windows.DELETE, // Desired Access 19 | windows.FILE_SHARE_DELETE, // Share Mode 20 | nil, // Security Attributes 21 | windows.TRUNCATE_EXISTING, // Create Disposition 22 | windows.FILE_ATTRIBUTE_TEMPORARY|windows.FILE_FLAG_DELETE_ON_CLOSE, // Flags & Attributes 23 | 0, // Template File 24 | ) 25 | if err != nil { 26 | return file, err 27 | } 28 | file = os.NewFile(uintptr(handle), name) 29 | err = os.Remove(name) 30 | return file, err 31 | } 32 | -------------------------------------------------------------------------------- /pipeat.go: -------------------------------------------------------------------------------- 1 | package pipeat 2 | 3 | // Attempts to provide similar funcationality to io.Pipe except supporting 4 | // ReaderAt and WriterAt. It uses a temp file as a buffer in order to implement 5 | // the interfaces as well as allow for asyncronous writes. 6 | 7 | // Author: John Eikenberry 8 | // License: CC0 9 | 10 | import ( 11 | "fmt" 12 | "io" 13 | "io/ioutil" 14 | "os" 15 | "sort" 16 | "sync" 17 | ) 18 | 19 | // Used to track write ahead areas of a file. That is, areas where there is a 20 | // gap in the file data earlier in the file. Possible with concurrent writes. 21 | type span struct { 22 | start, end int64 23 | } 24 | 25 | type spans []span 26 | 27 | func (c spans) Len() int { return len(c) } 28 | func (c spans) Swap(i, j int) { c[i], c[j] = c[j], c[i] } 29 | func (c spans) Less(i, j int) bool { return c[i].start < c[j].start } 30 | 31 | // The wrapper around the underlying temp file. 32 | type pipeFile struct { 33 | *os.File 34 | fileLock sync.RWMutex 35 | rCond sync.Cond // used to signal readers from writers 36 | wCond sync.Cond // used to signal writers from readers in sync mode 37 | dataLock sync.RWMutex // serialize access to meta data (below) 38 | syncMode bool // if true the writer is only allowed to write until the reader requested 39 | endln int64 40 | ahead spans 41 | readeroff int64 // offset where Read() last read 42 | writeroff int64 // file offset allowed for the writer in sync mode 43 | readed int64 // size readed as bytes, useful for stats 44 | written int64 // size written as bytes, useful for stats 45 | rerr error 46 | werr error 47 | eow chan struct{} // end of writing 48 | eor chan struct{} // end of reading 49 | } 50 | 51 | func newPipeFile(dirPath string) (*pipeFile, error) { 52 | file, err := ioutil.TempFile(dirPath, "pipefile") 53 | if err != nil { 54 | return nil, err 55 | } 56 | file, err = unlinkFile(file) 57 | if err != nil { 58 | return nil, err 59 | } 60 | f := &pipeFile{File: file, 61 | eow: make(chan struct{}), 62 | eor: make(chan struct{})} 63 | f.rCond.L = f.dataLock.RLocker() // Readers cond locker 64 | f.wCond.L = f.dataLock.RLocker() // Writers cond locker 65 | return f, nil 66 | } 67 | 68 | // waitForReadable waits until the requested data is available to read. 69 | // We can stop waiting if the writer ended or if the reader is closed. 70 | // The reader can be closed using Close() or CloseWithError we don't 71 | // want to wait on a closed reader 72 | func (f *pipeFile) waitForReadable(off int64) { 73 | f.dataLock.RLock() 74 | defer f.dataLock.RUnlock() 75 | 76 | for off >= f.endln { 77 | select { 78 | case <-f.eow: 79 | trace("eow") 80 | return 81 | case <-f.eor: 82 | trace("eor") 83 | return 84 | default: 85 | f.rCond.Wait() 86 | } 87 | } 88 | } 89 | 90 | func (f *pipeFile) updateReadedBytes(n int) { 91 | f.dataLock.Lock() 92 | defer f.dataLock.Unlock() 93 | f.readed += int64(n) 94 | } 95 | 96 | func (f *pipeFile) readerror() error { 97 | f.dataLock.RLock() 98 | defer f.dataLock.RUnlock() 99 | return f.rerr 100 | } 101 | 102 | func (f *pipeFile) setReaderror(err error) { 103 | f.dataLock.Lock() 104 | defer f.dataLock.Unlock() 105 | 106 | if f.rerr == nil { 107 | f.rerr = err 108 | close(f.eor) 109 | } 110 | f.rCond.Broadcast() 111 | f.wCond.Broadcast() 112 | } 113 | 114 | // waitForWritable don't allow to write more than the reader requested. 115 | // If the pipe is not in sync mode it does nothing. 116 | // We can stop waiting it the reader need more data or if the reader or 117 | // the writer are closed 118 | func (f *pipeFile) waitForWritable() { 119 | if !f.syncMode { 120 | return 121 | } 122 | f.dataLock.RLock() 123 | defer f.dataLock.RUnlock() 124 | 125 | for f.endln > f.writeroff { 126 | select { 127 | case <-f.eow: 128 | trace("eow") 129 | return 130 | case <-f.eor: 131 | trace("eor") 132 | return 133 | default: 134 | f.wCond.Wait() 135 | } 136 | } 137 | } 138 | 139 | func (f *pipeFile) updateWrittenBytes(n int) { 140 | f.dataLock.Lock() 141 | defer f.dataLock.Unlock() 142 | f.written += int64(n) 143 | } 144 | 145 | func (f *pipeFile) writeerror() error { 146 | f.dataLock.RLock() 147 | defer f.dataLock.RUnlock() 148 | return f.werr 149 | } 150 | 151 | func (f *pipeFile) setWriteerror(err error) { 152 | f.dataLock.Lock() 153 | defer f.dataLock.Unlock() 154 | 155 | if f.werr == nil { 156 | f.werr = err 157 | close(f.eow) 158 | } 159 | f.rCond.Broadcast() 160 | f.wCond.Broadcast() 161 | } 162 | 163 | // set the new allowed write offset and signal the writers. 164 | // Do nothing if not in sync mode 165 | func (f *pipeFile) setWriteoff(off int64) { 166 | if !f.syncMode { 167 | return 168 | } 169 | f.dataLock.Lock() 170 | defer f.dataLock.Unlock() 171 | if off > f.writeroff { 172 | f.writeroff = off 173 | f.wCond.Broadcast() 174 | } 175 | } 176 | 177 | // PipeWriterAt is the io.WriterAt side of pipe. 178 | type PipeWriterAt struct { 179 | f *pipeFile 180 | asyncWriter bool 181 | } 182 | 183 | // PipeReaderAt is the io.ReaderAt side of pipe. 184 | type PipeReaderAt struct { 185 | f *pipeFile 186 | } 187 | 188 | // Pipe creates an asynchronous file based pipe. It can be used to connect code 189 | // expecting an io.ReaderAt with code expecting an io.WriterAt. Writes all go 190 | // to an unlinked temporary file, reads start up as the file gets written up to 191 | // their area. It is safe to call multiple ReadAt and WriteAt in parallel with 192 | // each other. 193 | func Pipe() (*PipeReaderAt, *PipeWriterAt, error) { 194 | return PipeInDir("") 195 | } 196 | 197 | // PipeInDir just like Pipe but the temporary file is created inside the specified 198 | // directory 199 | func PipeInDir(dirPath string) (*PipeReaderAt, *PipeWriterAt, error) { 200 | return newPipe(dirPath, false) 201 | } 202 | 203 | // AsyncWriterPipe is just like Pipe but the writer is allowed to close before 204 | // the reader is finished. Whereas in Pipe the writer blocks until the reader 205 | // is done. 206 | func AsyncWriterPipe() (*PipeReaderAt, *PipeWriterAt, error) { 207 | return AsyncWriterPipeInDir("") 208 | } 209 | 210 | // AsyncWriterPipeInDir is just like AsyncWriterPipe but the temporary file is created 211 | // inside the specified directory 212 | func AsyncWriterPipeInDir(dirPath string) (*PipeReaderAt, *PipeWriterAt, error) { 213 | return newPipe(dirPath, true) 214 | } 215 | 216 | func newPipe(dirPath string, asyncWriter bool) (*PipeReaderAt, *PipeWriterAt, error) { 217 | fp, err := newPipeFile(dirPath) 218 | if err != nil { 219 | return nil, nil, err 220 | } 221 | fp.syncMode = !asyncWriter 222 | return &PipeReaderAt{fp}, &PipeWriterAt{fp, asyncWriter}, nil 223 | } 224 | 225 | // ReadAt implements the standard ReaderAt interface. It blocks if it gets 226 | // ahead of the writer. You can call it from multiple threads. 227 | func (r *PipeReaderAt) ReadAt(p []byte, off int64) (int, error) { 228 | trace("readat", off) 229 | 230 | r.f.setWriteoff(off + int64(len(p))) 231 | r.f.waitForReadable(off + int64(len(p))) 232 | 233 | r.f.fileLock.RLock() 234 | defer r.f.fileLock.RUnlock() 235 | 236 | if err := r.f.readerror(); err != nil { 237 | trace("end readat(1):", off, 0, err) 238 | return 0, err 239 | } 240 | 241 | n, err := r.f.File.ReadAt(p, off) 242 | r.f.updateReadedBytes(n) 243 | if err != nil { 244 | if werr := r.f.writeerror(); werr != nil { 245 | err = werr 246 | } 247 | } 248 | trace("end readat(2):", off, n, err) 249 | return n, err 250 | } 251 | 252 | // It can also function as a io.Reader 253 | func (r *PipeReaderAt) Read(p []byte) (int, error) { 254 | trace("read", len(p)) 255 | n, err := r.ReadAt(p, r.f.readeroff) 256 | if n > 0 { 257 | r.f.dataLock.Lock() 258 | defer r.f.dataLock.Unlock() 259 | r.f.readeroff = r.f.readeroff + int64(n) 260 | } 261 | trace("end read", n, err) 262 | return n, err 263 | } 264 | 265 | // GetReadedBytes returns the bytes readed 266 | func (r *PipeReaderAt) GetReadedBytes() int64 { 267 | r.f.dataLock.RLock() 268 | defer r.f.dataLock.RUnlock() 269 | return r.f.readed 270 | } 271 | 272 | // Close will Close the temp file and subsequent writes or reads will return an 273 | // ErrClosePipe error. 274 | func (r *PipeReaderAt) Close() error { 275 | return r.CloseWithError(nil) 276 | } 277 | 278 | // CloseWithError sets error and otherwise behaves like Close. 279 | func (r *PipeReaderAt) CloseWithError(err error) error { 280 | if err == nil { 281 | err = io.EOF 282 | } 283 | r.f.fileLock.Lock() 284 | defer r.f.fileLock.Unlock() 285 | r.f.setReaderror(err) 286 | return r.f.File.Close() 287 | } 288 | 289 | // WriteAt implements the standard WriterAt interface. It will write to the 290 | // temp file without blocking. You can call it from multiple threads. 291 | func (w *PipeWriterAt) WriteAt(p []byte, off int64) (int, error) { 292 | //trace("writeat: ", string(p), off) 293 | //defer trace("wrote: ", string(p), off) 294 | 295 | w.f.waitForWritable() 296 | 297 | w.f.fileLock.RLock() 298 | defer w.f.fileLock.RUnlock() 299 | 300 | if err := w.f.writeerror(); err != nil { 301 | return 0, err 302 | } 303 | n, err := w.f.File.WriteAt(p, off) 304 | w.f.updateWrittenBytes(n) 305 | if err != nil { 306 | if err = w.f.readerror(); err != nil { 307 | return 0, err 308 | } 309 | return 0, io.EOF 310 | } 311 | 312 | w.f.dataLock.Lock() 313 | defer w.f.dataLock.Unlock() 314 | 315 | if off == w.f.endln { 316 | w.f.endln = w.f.endln + int64(n) 317 | newtip := 0 318 | for i, s := range w.f.ahead { 319 | if s.start == w.f.endln { 320 | w.f.endln = s.end 321 | newtip = i + 1 322 | } 323 | } 324 | if newtip > 0 { // clean up ahead queue 325 | w.f.ahead = append(w.f.ahead[:0], w.f.ahead[newtip:]...) 326 | } 327 | w.f.rCond.Broadcast() 328 | } else { 329 | w.f.ahead = append(w.f.ahead, span{off, off + int64(n)}) 330 | sort.Sort(w.f.ahead) // should already be sorted.. 331 | } 332 | // trace(w.f.ahead) 333 | return n, err 334 | } 335 | 336 | // Write provides a standard io.Writer interface. 337 | func (w *PipeWriterAt) Write(p []byte) (int, error) { 338 | w.f.waitForWritable() 339 | w.f.fileLock.RLock() 340 | defer w.f.fileLock.RUnlock() 341 | n, err := w.f.Write(p) 342 | w.f.updateWrittenBytes(n) 343 | if n > 0 { 344 | w.f.dataLock.Lock() 345 | defer w.f.dataLock.Unlock() 346 | w.f.endln += int64(n) 347 | w.f.rCond.Broadcast() 348 | } 349 | return n, err 350 | } 351 | 352 | // GetWrittenBytes returns the bytes written 353 | func (w *PipeWriterAt) GetWrittenBytes() int64 { 354 | w.f.dataLock.RLock() 355 | defer w.f.dataLock.RUnlock() 356 | return w.f.written 357 | } 358 | 359 | // Close on the writer will let the reader know that writing is complete. Once 360 | // the reader catches up it will continue to return 0 bytes and an EOF error. 361 | func (w *PipeWriterAt) Close() error { 362 | return w.CloseWithError(nil) 363 | } 364 | 365 | // CloseWithError sets the error and otherwise behaves like Close. 366 | func (w *PipeWriterAt) CloseWithError(err error) error { 367 | if err == nil { 368 | err = io.EOF 369 | } 370 | w.f.setWriteerror(err) 371 | // write is closed at this point, should I expose a way to check this? 372 | if !w.asyncWriter { 373 | w.WaitForReader() 374 | } 375 | return nil 376 | } 377 | 378 | // WaitForReader will block until the reader is closed. Returns the error set 379 | // when the reader closed. 380 | func (w *PipeWriterAt) WaitForReader() error { 381 | <-w.f.eor 382 | return w.f.readerror() 383 | } 384 | 385 | // debugging stuff 386 | const watch = false 387 | 388 | func trace(p ...interface{}) { 389 | if watch { 390 | fmt.Println(p...) 391 | } 392 | } 393 | -------------------------------------------------------------------------------- /pipeat_test.go: -------------------------------------------------------------------------------- 1 | package pipeat 2 | 3 | // Author: John Eikenberry 4 | // License: CC0 5 | 6 | import ( 7 | "bytes" 8 | "errors" 9 | "io" 10 | "io/fs" 11 | "math/rand" 12 | "strings" 13 | "sync" 14 | "testing" 15 | "time" 16 | 17 | "github.com/stretchr/testify/assert" 18 | ) 19 | 20 | var paragraph = "In programming, concurrency is the composition of independently executing processes, while parallelism is the simultaneous execution of (possibly related) computations. Concurrency is about dealing with lots of things at once. Parallelism is about doing lots of things at once." 21 | 22 | type chunk struct { 23 | offset int64 24 | word []byte 25 | } 26 | 27 | func chunkify(s string) []chunk { 28 | ss := strings.Split(s, " ") 29 | chunks := make([]chunk, len(ss)) 30 | offset := 0 31 | for i, s := range ss { 32 | if i+1 != len(ss) { 33 | s = s + " " 34 | } 35 | chunks[i] = chunk{int64(offset), []byte(s)} 36 | offset += len(s) 37 | } 38 | return chunks 39 | } 40 | 41 | func segmentify(chunks []chunk, num int) [][]chunk { 42 | segments := make([][]chunk, num) 43 | segSize := (len(chunks) / num) + 1 44 | var start, end int 45 | for i := 0; i < num; i++ { 46 | start, end = end, end+segSize 47 | if end <= len(chunks) { 48 | segments[i] = chunks[start:end] 49 | } else { 50 | segments[i] = chunks[start:] 51 | } 52 | } 53 | return segments 54 | } 55 | 56 | func reverse(t []chunk) []chunk { 57 | for l, r := 0, len(t)-1; l < r; l, r = l+1, r-1 { 58 | t[l], t[r] = t[r], t[l] 59 | } 60 | return t 61 | } 62 | 63 | type readers interface { 64 | io.Reader 65 | io.ReaderAt 66 | } 67 | 68 | type reader struct { 69 | r readers 70 | buf []byte 71 | *sync.Mutex // to protect buf 72 | } 73 | 74 | func newReader(r readers) *reader { 75 | return &reader{ 76 | r: r, 77 | Mutex: &sync.Mutex{}, 78 | } 79 | } 80 | 81 | func (r *reader) writeat(b []byte, off int) { 82 | r.Lock() 83 | defer r.Unlock() 84 | // fmt.Println("reader writeat", len(b), off, string(b)) 85 | if off+len(b) < len(r.buf) { 86 | copy(r.buf[off:], b) 87 | } else if off == len(r.buf) { 88 | r.buf = append(r.buf, b...) 89 | } else { 90 | ndata := make([]byte, off+len(b)) 91 | copy(ndata, r.buf) 92 | copy(ndata[off:], b) 93 | r.buf = ndata 94 | } 95 | } 96 | 97 | func (r *reader) String() string { 98 | r.Lock() 99 | defer r.Unlock() 100 | return string(r.buf) 101 | } 102 | 103 | func sectionReader(r *reader, start, end int64) *reader { 104 | section := io.NewSectionReader(r.r, start, end) 105 | return newReader(section) 106 | } 107 | 108 | type readat func([]byte, int64) (int, error) 109 | type write func([]byte, int) 110 | 111 | func basicReader(r readat, w write) { 112 | offset := 0 113 | size := 10 114 | for { 115 | b := make([]byte, size) 116 | n, err := r(b, int64(offset)) 117 | w(b[:n], offset) 118 | if err == io.EOF { 119 | return 120 | } 121 | if err != nil { 122 | panic(err) 123 | } 124 | offset += n 125 | } 126 | } 127 | 128 | func simpleReader(t *testing.T, r *reader) { 129 | defer r.r.(*PipeReaderAt).Close() 130 | basicReader(r.r.ReadAt, r.writeat) 131 | } 132 | 133 | func ioReader(t *testing.T, r *reader) { 134 | defer r.r.(*PipeReaderAt).Close() 135 | read := func(b []byte, _ int64) (int, error) { 136 | return r.r.Read(b) 137 | } 138 | basicReader(read, r.writeat) 139 | } 140 | 141 | func ccReader(t *testing.T, r *reader, workers int) { 142 | defer r.r.(*PipeReaderAt).Close() 143 | segSize := int64((len(paragraph) / workers) + 1) 144 | var sections = make([]*reader, workers) 145 | var start, end int64 146 | for i := 0; i < workers; i++ { 147 | start, end = end, end+segSize 148 | sections[i] = sectionReader(r, start, segSize) 149 | } 150 | wg := &sync.WaitGroup{} 151 | wg.Add(workers) 152 | for i := 0; i < workers; i++ { 153 | go func(i int) { 154 | s := sections[i] 155 | basicReader(s.r.ReadAt, s.writeat) 156 | wg.Done() 157 | }(i) 158 | } 159 | wg.Wait() 160 | buf := bytes.NewBuffer(nil) 161 | for i := 0; i < workers; i++ { 162 | buf.Write(sections[i].buf) 163 | } 164 | r.writeat(buf.Bytes(), 0) 165 | } 166 | 167 | func simpleWriter(t *testing.T, w *PipeWriterAt) { 168 | for _, t := range chunkify(paragraph) { 169 | w.WriteAt(t.word, t.offset) 170 | } 171 | w.Close() 172 | } 173 | 174 | func ioWriter(t *testing.T, w *PipeWriterAt) { 175 | for _, t := range chunkify(paragraph) { 176 | w.Write(t.word) 177 | } 178 | w.Close() 179 | } 180 | 181 | func reverseWriter(t *testing.T, w *PipeWriterAt) { 182 | chunks := reverse(chunkify(paragraph)) 183 | for _, t := range chunks { 184 | w.WriteAt(t.word, t.offset) 185 | } 186 | w.Close() 187 | } 188 | 189 | func randomWriter(t *testing.T, w *PipeWriterAt) { 190 | chunks := chunkify(paragraph) 191 | picks := rand.Perm(len(chunks)) 192 | for _, i := range picks { 193 | t := chunks[i] 194 | w.WriteAt(t.word, t.offset) 195 | } 196 | w.Close() 197 | } 198 | 199 | func ccWriter(t *testing.T, w *PipeWriterAt, workers int) { 200 | chunks := chunkify(paragraph) 201 | segments := segmentify(chunks, workers) 202 | resultsWg := &sync.WaitGroup{} 203 | resultsWg.Add(workers) 204 | for i := 0; i < len(segments); i++ { 205 | go func(j int, segment []chunk) { 206 | for _, t := range segment { 207 | w.WriteAt(t.word, t.offset) 208 | } 209 | resultsWg.Done() 210 | }(i, segments[i]) 211 | } 212 | resultsWg.Wait() 213 | w.Close() 214 | } 215 | 216 | func TestCcWrite(t *testing.T) { 217 | r, w, err := Pipe() 218 | if err != nil { 219 | panic(err) 220 | } 221 | workers := 4 222 | 223 | reader := newReader(r) 224 | go simpleReader(t, reader) 225 | go ccWriter(t, w, workers) 226 | w.WaitForReader() 227 | assert.Equal(t, reader.String(), paragraph) 228 | assert.Equal(t, r.GetReadedBytes(), int64(len(paragraph))) 229 | assert.Equal(t, w.GetWrittenBytes(), int64(len(paragraph))) 230 | } 231 | 232 | func TestCcRead(t *testing.T) { 233 | r, w, err := Pipe() 234 | if err != nil { 235 | panic(err) 236 | } 237 | workers := 4 238 | 239 | reader := newReader(r) 240 | go ccReader(t, reader, workers) 241 | go simpleWriter(t, w) 242 | w.WaitForReader() 243 | assert.Equal(t, reader.String(), paragraph) 244 | assert.Equal(t, r.GetReadedBytes(), int64(len(paragraph))) 245 | assert.Equal(t, w.GetWrittenBytes(), int64(len(paragraph))) 246 | } 247 | 248 | func TestCcRWrite(t *testing.T) { 249 | r, w, err := Pipe() 250 | if err != nil { 251 | panic(err) 252 | } 253 | workers := 4 254 | 255 | reader := newReader(r) 256 | go ccReader(t, reader, workers) 257 | go ccWriter(t, w, workers) 258 | w.WaitForReader() 259 | assert.Equal(t, reader.String(), paragraph) 260 | assert.Equal(t, r.GetReadedBytes(), int64(len(paragraph))) 261 | assert.Equal(t, w.GetWrittenBytes(), int64(len(paragraph))) 262 | } 263 | 264 | func TestSimpleWrite(t *testing.T) { 265 | r, w, err := Pipe() 266 | if err != nil { 267 | panic(err) 268 | } 269 | reader := newReader(r) 270 | go simpleReader(t, reader) 271 | go simpleWriter(t, w) 272 | w.WaitForReader() 273 | trace(reader.String()) 274 | assert.Equal(t, reader.String(), paragraph) 275 | assert.Equal(t, r.GetReadedBytes(), int64(len(paragraph))) 276 | assert.Equal(t, w.GetWrittenBytes(), int64(len(paragraph))) 277 | } 278 | 279 | func TestWaitOnRead(t *testing.T) { 280 | r, w, err := Pipe() 281 | if err != nil { 282 | panic(err) 283 | } 284 | reader := newReader(r) 285 | go func() { 286 | time.Sleep(time.Millisecond) 287 | simpleReader(t, reader) 288 | }() 289 | simpleWriter(t, w) 290 | w.WaitForReader() 291 | assert.Equal(t, reader.String(), paragraph) 292 | assert.Equal(t, r.GetReadedBytes(), int64(len(paragraph))) 293 | assert.Equal(t, w.GetWrittenBytes(), int64(len(paragraph))) 294 | } 295 | 296 | func TestReverseWrite(t *testing.T) { 297 | r, w, err := Pipe() 298 | if err != nil { 299 | panic(err) 300 | } 301 | reader := newReader(r) 302 | go simpleReader(t, reader) 303 | go reverseWriter(t, w) 304 | w.WaitForReader() 305 | trace(reader.String()) 306 | assert.Equal(t, reader.String(), paragraph) 307 | assert.Equal(t, r.GetReadedBytes(), int64(len(paragraph))) 308 | assert.Equal(t, w.GetWrittenBytes(), int64(len(paragraph))) 309 | } 310 | 311 | func TestRandomWrite(t *testing.T) { 312 | rand.Seed(17) 313 | r, w, err := Pipe() 314 | if err != nil { 315 | panic(err) 316 | } 317 | reader := newReader(r) 318 | go simpleReader(t, reader) 319 | go randomWriter(t, w) 320 | w.WaitForReader() 321 | trace(reader.String(), len(reader.String())) 322 | assert.Equal(t, reader.String(), paragraph) 323 | assert.Equal(t, r.GetReadedBytes(), int64(len(paragraph))) 324 | assert.Equal(t, w.GetWrittenBytes(), int64(len(paragraph))) 325 | } 326 | 327 | // io.Read paired with concurrent writer 328 | func TestIoRead(t *testing.T) { 329 | r, w, err := Pipe() 330 | if err != nil { 331 | panic(err) 332 | } 333 | workers := 4 334 | 335 | reader := newReader(r) 336 | go ioReader(t, reader) 337 | go ccWriter(t, w, workers) 338 | w.WaitForReader() 339 | assert.Equal(t, reader.String(), paragraph) 340 | assert.Equal(t, int64(len(paragraph)), r.GetReadedBytes()) 341 | assert.Equal(t, int64(len(paragraph)), w.GetWrittenBytes()) 342 | assert.Equal(t, int64(len(paragraph)), r.f.readeroff) 343 | assert.Equal(t, int64(len(paragraph)), r.f.endln) 344 | } 345 | 346 | // io.Writer paired wit concurrent reader 347 | func TestIoWrite(t *testing.T) { 348 | r, w, err := Pipe() 349 | if err != nil { 350 | panic(err) 351 | } 352 | workers := 4 353 | 354 | reader := newReader(r) 355 | go ccReader(t, reader, workers) 356 | go ioWriter(t, w) 357 | w.WaitForReader() 358 | assert.Equal(t, reader.String(), paragraph) 359 | assert.Equal(t, r.GetReadedBytes(), int64(len(paragraph))) 360 | assert.Equal(t, w.GetWrittenBytes(), int64(len(paragraph))) 361 | } 362 | 363 | // test close 364 | func TestReadClose(t *testing.T) { 365 | r, w, err := Pipe() 366 | if err != nil { 367 | panic(err) 368 | } 369 | r.Close() 370 | b := make([]byte, 1) 371 | n, err := r.ReadAt(b, 10) 372 | assert.Equal(t, 0, n) 373 | assert.Equal(t, io.EOF, err) 374 | b = []byte("hi") 375 | n, err = w.WriteAt(b, 10) 376 | assert.Equal(t, 0, n) 377 | assert.Equal(t, io.EOF, err) 378 | } 379 | 380 | func TestReadCloseWithError(t *testing.T) { 381 | r, w, err := Pipe() 382 | if err != nil { 383 | panic(err) 384 | } 385 | errTest := errors.New("test error") 386 | r.CloseWithError(errTest) 387 | b := make([]byte, 1) 388 | n, err := r.ReadAt(b, 10) 389 | assert.Equal(t, 0, n) 390 | assert.Equal(t, errTest, err) 391 | b = []byte("hi") 392 | n, err = w.WriteAt(b, 10) 393 | assert.Equal(t, 0, n) 394 | assert.Equal(t, errTest, err) 395 | } 396 | 397 | func TestWriteClose(t *testing.T) { 398 | r, w, err := Pipe() 399 | if err != nil { 400 | panic(err) 401 | } 402 | go func() { 403 | w.Close() 404 | }() 405 | <-w.f.eow // used to detect that close is done except block on reader 406 | b := []byte("hi") 407 | n, err := w.WriteAt(b, 10) 408 | assert.Equal(t, 0, n, "shouldn't have been able to write") 409 | assert.Equal(t, io.EOF, err) 410 | b = make([]byte, 1) 411 | n, err = r.ReadAt(b, 10) 412 | assert.Equal(t, 0, n, "shouldn't have been able to read") 413 | assert.Equal(t, io.EOF, err) 414 | } 415 | 416 | func TestWriteCloseWithError(t *testing.T) { 417 | r, w, err := Pipe() 418 | if err != nil { 419 | panic(err) 420 | } 421 | errTest := errors.New("test error") 422 | go func() { 423 | w.CloseWithError(errTest) 424 | }() 425 | <-w.f.eow // used to detect that close is done except block on reader 426 | b := []byte("hi") 427 | n, err := w.WriteAt(b, 10) 428 | assert.Equal(t, 0, n, "shouldn't have been able to write") 429 | assert.Equal(t, errTest, err) 430 | b = make([]byte, 1) 431 | n, err = r.ReadAt(b, 10) 432 | assert.Equal(t, 0, n, "shouldn't have been able to read") 433 | assert.Equal(t, errTest, err) 434 | r.Close() 435 | } 436 | 437 | // same as above, but tests AsyncWriterPipe() 438 | func TestAsyncWriteCloseWithError(t *testing.T) { 439 | r, w, err := AsyncWriterPipe() 440 | if err != nil { 441 | panic(err) 442 | } 443 | errTest := errors.New("test error") 444 | w.CloseWithError(errTest) 445 | b := []byte("hi") 446 | n, err := w.WriteAt(b, 10) 447 | assert.Equal(t, 0, n) 448 | assert.Equal(t, errTest, err) 449 | b = make([]byte, 1) 450 | n, err = r.ReadAt(b, 10) 451 | assert.Equal(t, 0, n) 452 | assert.Equal(t, errTest, err) 453 | } 454 | 455 | func TestPipeInDir(t *testing.T) { 456 | r, w, err := PipeInDir("") 457 | if err != nil { 458 | panic(err) 459 | } 460 | err = r.Close() 461 | if err != nil { 462 | panic(err) 463 | } 464 | err = w.Close() 465 | if err != nil { 466 | panic(err) 467 | } 468 | _, _, err = PipeInDir("invalid dir") 469 | if err == nil { 470 | panic("pipe in dir must file, the requested directory does not exists") 471 | } 472 | } 473 | 474 | func TestAsyncPipeInDir(t *testing.T) { 475 | r, w, err := AsyncWriterPipeInDir("") 476 | if err != nil { 477 | panic(err) 478 | } 479 | err = w.Close() 480 | if err != nil { 481 | panic(err) 482 | } 483 | err = r.Close() 484 | if err != nil { 485 | panic(err) 486 | } 487 | _, _, err = AsyncWriterPipeInDir("invalid dir") 488 | if err == nil { 489 | panic("async writer pipe in dir must file, the requested directory does not exists") 490 | } 491 | } 492 | 493 | func TestUnlockWriteOnReaderClose(t *testing.T) { 494 | r, w, err := Pipe() 495 | if err != nil { 496 | panic(err) 497 | } 498 | // the first write will succeed, the second one will remain blocked 499 | // waiting for the reader, when the reader is closed waitForWritable 500 | // returns and WriteAt can finish 501 | errTest := errors.New("test error") 502 | b := []byte("hi") 503 | n, err := w.WriteAt(b, 0) 504 | assert.Equal(t, len(b), n) 505 | assert.Equal(t, nil, err) 506 | go func() { 507 | r.CloseWithError(errTest) 508 | }() 509 | // this write will be unlocked when close ends 510 | n, err = w.WriteAt(b, 5) 511 | assert.Equal(t, 0, n) 512 | assert.Equal(t, errTest, err) 513 | w.Close() 514 | } 515 | 516 | func TestUnlockWriteOnWriterClose(t *testing.T) { 517 | r, w, err := Pipe() 518 | if err != nil { 519 | panic(err) 520 | } 521 | // the first write will succeed, the second one will remain blocked 522 | // waiting for the reader, when the writer is closed waitForWritable 523 | // returns and WriteAt can finish 524 | errTest := errors.New("test error") 525 | b := []byte("hi") 526 | n, err := w.WriteAt(b, 0) 527 | assert.Equal(t, len(b), n) 528 | assert.Equal(t, nil, err) 529 | go func() { 530 | w.CloseWithError(errTest) 531 | }() 532 | // this write will be unlocked when close ends 533 | n, err = w.WriteAt(b, 5) 534 | assert.Equal(t, 0, n) 535 | assert.Equal(t, errTest, err) 536 | r.Close() 537 | } 538 | 539 | func TestDoubleClose(t *testing.T) { 540 | r, w, err := Pipe() 541 | if err != nil { 542 | panic(err) 543 | } 544 | err = r.Close() 545 | assert.NoError(t, err) 546 | err = w.Close() 547 | assert.NoError(t, err) 548 | 549 | err = r.Close() 550 | assert.ErrorIs(t, err, fs.ErrClosed) 551 | err = w.Close() 552 | assert.NoError(t, err) 553 | } 554 | --------------------------------------------------------------------------------