├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── batching_channel.go ├── batching_channel_test.go ├── black_hole.go ├── black_hole_test.go ├── channels.go ├── channels_test.go ├── infinite_channel.go ├── infinite_channel_test.go ├── native_channel.go ├── native_channel_test.go ├── overflowing_channel.go ├── overflowing_channel_test.go ├── resizable_channel.go ├── resizable_channel_test.go ├── ring_channel.go ├── ring_channel_test.go ├── shared_buffer.go └── shared_buffer_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: false 3 | 4 | script: go test -v -race -timeout 10s ./... 5 | 6 | go: 7 | - 1.1 8 | - 1.2 9 | - 1.3 10 | - 1.4 11 | - 1.5 12 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | #### Version 1.1.0 (2015-11-22) 4 | 5 | Bug Fixes: 6 | - The `Len()` and `Cap()` methods on several implementations were racy 7 | ([#18](https://github.com/eapache/channels/issues/18)). 8 | 9 | Note: Fixing the above issue led to a fairly substantial performance hit 10 | (anywhere from 10-25% in benchmarks depending on use case) and involved fairly 11 | major refactoring, which is why this is being released as v1.1.0 instead 12 | of v1.0.1. 13 | 14 | #### Version 1.0.0 (2015-01-24) 15 | 16 | Version 1.0.0 is the first tagged release. All core functionality was available 17 | at this point. 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Evan Huus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **WARNING**: This package is end-of-life, has several long-standing bugs, and 2 | is not receiving any more fixes. Additionally, the design has several problems 3 | and no longer really makes sense given Go's addition of generic types. 4 | It was still an interesting project, and might provide inspiration for tricks 5 | you can do with channels, but caveat emptor. 6 | 7 | channels 8 | ======== 9 | 10 | [![GoDoc](https://godoc.org/github.com/eapache/channels?status.png)](https://godoc.org/github.com/eapache/channels) 11 | [![Code of Conduct](https://img.shields.io/badge/code%20of%20conduct-active-blue.svg)](https://eapache.github.io/conduct.html) 12 | 13 | A collection of helper functions and special types for working with and 14 | extending [Go](https://golang.org/)'s existing channels. Due to limitations 15 | of Go's type system, importing this library directly is often not practical for 16 | production code. It serves equally well, however, as a reference guide and 17 | template for implementing many common idioms; if you use it in this way I would 18 | appreciate the inclusion of some sort of credit in the resulting code. 19 | 20 | See https://godoc.org/github.com/eapache/channels for full documentation or 21 | https://gopkg.in/eapache/channels.v1 for a versioned import path. 22 | 23 | Requires Go version 1.1 or later, as certain necessary elements of the `reflect` 24 | package were not present in 1.0. 25 | 26 | Most of the buffered channel types in this package are backed by a very fast 27 | queue implementation that used to be built into this package but has now been 28 | extracted into its own package at https://github.com/eapache/queue. 29 | 30 | *Note:* Several types in this package provide so-called "infinite" buffers. Be 31 | very careful using these, as no buffer is truly infinite. If such a buffer 32 | grows too large your program will run out of memory and crash. Caveat emptor. 33 | -------------------------------------------------------------------------------- /batching_channel.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | // BatchingChannel implements the Channel interface, with the change that instead of producing individual elements 4 | // on Out(), it batches together the entire internal buffer each time. Trying to construct an unbuffered batching channel 5 | // will panic, that configuration is not supported (and provides no benefit over an unbuffered NativeChannel). 6 | type BatchingChannel struct { 7 | input, output chan interface{} 8 | length chan int 9 | buffer []interface{} 10 | size BufferCap 11 | } 12 | 13 | func NewBatchingChannel(size BufferCap) *BatchingChannel { 14 | if size == None { 15 | panic("channels: BatchingChannel does not support unbuffered behaviour") 16 | } 17 | if size < 0 && size != Infinity { 18 | panic("channels: invalid negative size in NewBatchingChannel") 19 | } 20 | ch := &BatchingChannel{ 21 | input: make(chan interface{}), 22 | output: make(chan interface{}), 23 | length: make(chan int), 24 | size: size, 25 | } 26 | go ch.batchingBuffer() 27 | return ch 28 | } 29 | 30 | func (ch *BatchingChannel) In() chan<- interface{} { 31 | return ch.input 32 | } 33 | 34 | // Out returns a <-chan interface{} in order that BatchingChannel conforms to the standard Channel interface provided 35 | // by this package, however each output value is guaranteed to be of type []interface{} - a slice collecting the most 36 | // recent batch of values sent on the In channel. The slice is guaranteed to not be empty or nil. In practice the net 37 | // result is that you need an additional type assertion to access the underlying values. 38 | func (ch *BatchingChannel) Out() <-chan interface{} { 39 | return ch.output 40 | } 41 | 42 | func (ch *BatchingChannel) Len() int { 43 | return <-ch.length 44 | } 45 | 46 | func (ch *BatchingChannel) Cap() BufferCap { 47 | return ch.size 48 | } 49 | 50 | func (ch *BatchingChannel) Close() { 51 | close(ch.input) 52 | } 53 | 54 | func (ch *BatchingChannel) batchingBuffer() { 55 | var input, output, nextInput chan interface{} 56 | nextInput = ch.input 57 | input = nextInput 58 | 59 | for input != nil || output != nil { 60 | select { 61 | case elem, open := <-input: 62 | if open { 63 | ch.buffer = append(ch.buffer, elem) 64 | } else { 65 | input = nil 66 | nextInput = nil 67 | } 68 | case output <- ch.buffer: 69 | ch.buffer = nil 70 | case ch.length <- len(ch.buffer): 71 | } 72 | 73 | if len(ch.buffer) == 0 { 74 | input = nextInput 75 | output = nil 76 | } else if ch.size != Infinity && len(ch.buffer) >= int(ch.size) { 77 | input = nil 78 | output = ch.output 79 | } else { 80 | input = nextInput 81 | output = ch.output 82 | } 83 | } 84 | 85 | close(ch.output) 86 | close(ch.length) 87 | } 88 | -------------------------------------------------------------------------------- /batching_channel_test.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "testing" 4 | 5 | func testBatches(t *testing.T, ch Channel) { 6 | go func() { 7 | for i := 0; i < 1000; i++ { 8 | ch.In() <- i 9 | } 10 | ch.Close() 11 | }() 12 | 13 | i := 0 14 | for val := range ch.Out() { 15 | for _, elem := range val.([]interface{}) { 16 | if i != elem.(int) { 17 | t.Fatal("batching channel expected", i, "but got", elem.(int)) 18 | } 19 | i++ 20 | } 21 | } 22 | } 23 | 24 | func TestBatchingChannel(t *testing.T) { 25 | ch := NewBatchingChannel(Infinity) 26 | testBatches(t, ch) 27 | 28 | ch = NewBatchingChannel(2) 29 | testBatches(t, ch) 30 | 31 | ch = NewBatchingChannel(1) 32 | testChannelConcurrentAccessors(t, "batching channel", ch) 33 | } 34 | 35 | func TestBatchingChannelCap(t *testing.T) { 36 | ch := NewBatchingChannel(Infinity) 37 | if ch.Cap() != Infinity { 38 | t.Error("incorrect capacity on infinite channel") 39 | } 40 | 41 | ch = NewBatchingChannel(5) 42 | if ch.Cap() != 5 { 43 | t.Error("incorrect capacity on infinite channel") 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /black_hole.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | // BlackHole implements the InChannel interface and provides an analogue for the "Discard" variable in 4 | // the ioutil package - it never blocks, and simply discards every value it reads. The number of items 5 | // discarded in this way is counted and returned from Len. 6 | type BlackHole struct { 7 | input chan interface{} 8 | length chan int 9 | count int 10 | } 11 | 12 | func NewBlackHole() *BlackHole { 13 | ch := &BlackHole{ 14 | input: make(chan interface{}), 15 | length: make(chan int), 16 | } 17 | go ch.discard() 18 | return ch 19 | } 20 | 21 | func (ch *BlackHole) In() chan<- interface{} { 22 | return ch.input 23 | } 24 | 25 | func (ch *BlackHole) Len() int { 26 | val, open := <-ch.length 27 | if open { 28 | return val 29 | } else { 30 | return ch.count 31 | } 32 | } 33 | 34 | func (ch *BlackHole) Cap() BufferCap { 35 | return Infinity 36 | } 37 | 38 | func (ch *BlackHole) Close() { 39 | close(ch.input) 40 | } 41 | 42 | func (ch *BlackHole) discard() { 43 | for { 44 | select { 45 | case _, open := <-ch.input: 46 | if !open { 47 | close(ch.length) 48 | return 49 | } 50 | ch.count++ 51 | case ch.length <- ch.count: 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /black_hole_test.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "testing" 4 | 5 | func TestBlackHole(t *testing.T) { 6 | discard := NewBlackHole() 7 | 8 | for i := 0; i < 1000; i++ { 9 | discard.In() <- i 10 | } 11 | 12 | discard.Close() 13 | 14 | if discard.Len() != 1000 { 15 | t.Error("blackhole expected 1000 was", discard.Len()) 16 | } 17 | 18 | // no asserts here, this is just for the race detector's benefit 19 | ch := NewBlackHole() 20 | go ch.Len() 21 | go ch.Cap() 22 | 23 | go func() { 24 | ch.In() <- nil 25 | }() 26 | } 27 | -------------------------------------------------------------------------------- /channels.go: -------------------------------------------------------------------------------- 1 | /* 2 | WARNING: This package is end-of-life, has several long-standing bugs, and is not receiving any more 3 | fixes. Additionally, the design has several problems and no longer really makes sense given Go's 4 | addition of generic types. It was still an interesting project, and might provide inspiration for 5 | tricks you can do with channels, but caveat emptor. 6 | 7 | Package channels provides a collection of helper functions, interfaces and implementations for 8 | working with and extending the capabilities of golang's existing channels. The main interface of 9 | interest is Channel, though sub-interfaces are also provided for cases where the full Channel interface 10 | cannot be met (for example, InChannel for write-only channels). 11 | 12 | For integration with native typed golang channels, functions Wrap and Unwrap are provided which do the 13 | appropriate type conversions. The NativeChannel, NativeInChannel and NativeOutChannel type definitions 14 | are also provided for use with native channels which already carry values of type interface{}. 15 | 16 | The heart of the package consists of several distinct implementations of the Channel interface, including 17 | channels backed by special buffers (resizable, infinite, ring buffers, etc) and other useful types. A 18 | "black hole" channel for discarding unwanted values (similar in purpose to ioutil.Discard or /dev/null) 19 | rounds out the set. 20 | 21 | Helper functions for operating on Channels include Pipe and Tee (which behave much like their Unix 22 | namesakes), as well as Multiplex and Distribute. "Weak" versions of these functions also exist, which 23 | do not close their output channel(s) on completion. 24 | 25 | Due to limitations of Go's type system, importing this library directly is often not practical for 26 | production code. It serves equally well, however, as a reference guide and template for implementing 27 | many common idioms; if you use it in this way I would appreciate the inclusion of some sort of credit 28 | in the resulting code. 29 | 30 | Warning: several types in this package provide so-called "infinite" buffers. Be *very* careful using 31 | these, as no buffer is truly infinite - if such a buffer grows too large your program will run out of 32 | memory and crash. Caveat emptor. 33 | 34 | Warning: the channel types from this package are implemented by spawning goroutines. If a channel from 35 | this package passes out of scope without being explicitly emptied and closed, it will leak the 36 | goroutine and remaining values. 37 | */ 38 | package channels 39 | 40 | import "reflect" 41 | 42 | // BufferCap represents the capacity of the buffer backing a channel. Valid values consist of all 43 | // positive integers, as well as the special values below. 44 | type BufferCap int 45 | 46 | const ( 47 | // None is the capacity for channels that have no buffer at all. 48 | None BufferCap = 0 49 | // Infinity is the capacity for channels with no limit on their buffer size. 50 | Infinity BufferCap = -1 51 | ) 52 | 53 | // Buffer is an interface for any channel that provides access to query the state of its buffer. 54 | // Even unbuffered channels can implement this interface by simply returning 0 from Len() and None from Cap(). 55 | type Buffer interface { 56 | Len() int // The number of elements currently buffered. 57 | Cap() BufferCap // The maximum number of elements that can be buffered. 58 | } 59 | 60 | // SimpleInChannel is an interface representing a writeable channel that does not necessarily 61 | // implement the Buffer interface. 62 | type SimpleInChannel interface { 63 | In() chan<- interface{} // The writeable end of the channel. 64 | Close() // Closes the channel. It is an error to write to In() after calling Close(). 65 | } 66 | 67 | // InChannel is an interface representing a writeable channel with a buffer. 68 | type InChannel interface { 69 | SimpleInChannel 70 | Buffer 71 | } 72 | 73 | // SimpleOutChannel is an interface representing a readable channel that does not necessarily 74 | // implement the Buffer interface. 75 | type SimpleOutChannel interface { 76 | Out() <-chan interface{} // The readable end of the channel. 77 | } 78 | 79 | // OutChannel is an interface representing a readable channel implementing the Buffer interface. 80 | type OutChannel interface { 81 | SimpleOutChannel 82 | Buffer 83 | } 84 | 85 | // SimpleChannel is an interface representing a channel that is both readable and writeable, 86 | // but does not necessarily implement the Buffer interface. 87 | type SimpleChannel interface { 88 | SimpleInChannel 89 | SimpleOutChannel 90 | } 91 | 92 | // Channel is an interface representing a channel that is readable, writeable and implements 93 | // the Buffer interface 94 | type Channel interface { 95 | SimpleChannel 96 | Buffer 97 | } 98 | 99 | func pipe(input SimpleOutChannel, output SimpleInChannel, closeWhenDone bool) { 100 | for elem := range input.Out() { 101 | output.In() <- elem 102 | } 103 | if closeWhenDone { 104 | output.Close() 105 | } 106 | } 107 | 108 | func multiplex(output SimpleInChannel, inputs []SimpleOutChannel, closeWhenDone bool) { 109 | inputCount := len(inputs) 110 | cases := make([]reflect.SelectCase, inputCount) 111 | for i := range cases { 112 | cases[i].Dir = reflect.SelectRecv 113 | cases[i].Chan = reflect.ValueOf(inputs[i].Out()) 114 | } 115 | for inputCount > 0 { 116 | chosen, recv, recvOK := reflect.Select(cases) 117 | if recvOK { 118 | output.In() <- recv.Interface() 119 | } else { 120 | cases[chosen].Chan = reflect.ValueOf(nil) 121 | inputCount-- 122 | } 123 | } 124 | if closeWhenDone { 125 | output.Close() 126 | } 127 | } 128 | 129 | func tee(input SimpleOutChannel, outputs []SimpleInChannel, closeWhenDone bool) { 130 | cases := make([]reflect.SelectCase, len(outputs)) 131 | for i := range cases { 132 | cases[i].Dir = reflect.SelectSend 133 | } 134 | for elem := range input.Out() { 135 | for i := range cases { 136 | cases[i].Chan = reflect.ValueOf(outputs[i].In()) 137 | cases[i].Send = reflect.ValueOf(elem) 138 | } 139 | for _ = range cases { 140 | chosen, _, _ := reflect.Select(cases) 141 | cases[chosen].Chan = reflect.ValueOf(nil) 142 | } 143 | } 144 | if closeWhenDone { 145 | for i := range outputs { 146 | outputs[i].Close() 147 | } 148 | } 149 | } 150 | 151 | func distribute(input SimpleOutChannel, outputs []SimpleInChannel, closeWhenDone bool) { 152 | cases := make([]reflect.SelectCase, len(outputs)) 153 | for i := range cases { 154 | cases[i].Dir = reflect.SelectSend 155 | cases[i].Chan = reflect.ValueOf(outputs[i].In()) 156 | } 157 | for elem := range input.Out() { 158 | for i := range cases { 159 | cases[i].Send = reflect.ValueOf(elem) 160 | } 161 | reflect.Select(cases) 162 | } 163 | if closeWhenDone { 164 | for i := range outputs { 165 | outputs[i].Close() 166 | } 167 | } 168 | } 169 | 170 | // Pipe connects the input channel to the output channel so that 171 | // they behave as if a single channel. 172 | func Pipe(input SimpleOutChannel, output SimpleInChannel) { 173 | go pipe(input, output, true) 174 | } 175 | 176 | // Multiplex takes an arbitrary number of input channels and multiplexes their output into a single output 177 | // channel. When all input channels have been closed, the output channel is closed. Multiplex with a single 178 | // input channel is equivalent to Pipe (though slightly less efficient). 179 | func Multiplex(output SimpleInChannel, inputs ...SimpleOutChannel) { 180 | if len(inputs) == 0 { 181 | panic("channels: Multiplex requires at least one input") 182 | } 183 | go multiplex(output, inputs, true) 184 | } 185 | 186 | // Tee (like its Unix namesake) takes a single input channel and an arbitrary number of output channels 187 | // and duplicates each input into every output. When the input channel is closed, all outputs channels are closed. 188 | // Tee with a single output channel is equivalent to Pipe (though slightly less efficient). 189 | func Tee(input SimpleOutChannel, outputs ...SimpleInChannel) { 190 | if len(outputs) == 0 { 191 | panic("channels: Tee requires at least one output") 192 | } 193 | go tee(input, outputs, true) 194 | } 195 | 196 | // Distribute takes a single input channel and an arbitrary number of output channels and duplicates each input 197 | // into *one* available output. If multiple outputs are waiting for a value, one is chosen at random. When the 198 | // input channel is closed, all outputs channels are closed. Distribute with a single output channel is 199 | // equivalent to Pipe (though slightly less efficient). 200 | func Distribute(input SimpleOutChannel, outputs ...SimpleInChannel) { 201 | if len(outputs) == 0 { 202 | panic("channels: Distribute requires at least one output") 203 | } 204 | go distribute(input, outputs, true) 205 | } 206 | 207 | // WeakPipe behaves like Pipe (connecting the two channels) except that it does not close 208 | // the output channel when the input channel is closed. 209 | func WeakPipe(input SimpleOutChannel, output SimpleInChannel) { 210 | go pipe(input, output, false) 211 | } 212 | 213 | // WeakMultiplex behaves like Multiplex (multiplexing multiple inputs into a single output) except that it does not close 214 | // the output channel when the input channels are closed. 215 | func WeakMultiplex(output SimpleInChannel, inputs ...SimpleOutChannel) { 216 | if len(inputs) == 0 { 217 | panic("channels: WeakMultiplex requires at least one input") 218 | } 219 | go multiplex(output, inputs, false) 220 | } 221 | 222 | // WeakTee behaves like Tee (duplicating a single input into multiple outputs) except that it does not close 223 | // the output channels when the input channel is closed. 224 | func WeakTee(input SimpleOutChannel, outputs ...SimpleInChannel) { 225 | if len(outputs) == 0 { 226 | panic("channels: WeakTee requires at least one output") 227 | } 228 | go tee(input, outputs, false) 229 | } 230 | 231 | // WeakDistribute behaves like Distribute (distributing a single input amongst multiple outputs) except that 232 | // it does not close the output channels when the input channel is closed. 233 | func WeakDistribute(input SimpleOutChannel, outputs ...SimpleInChannel) { 234 | if len(outputs) == 0 { 235 | panic("channels: WeakDistribute requires at least one output") 236 | } 237 | go distribute(input, outputs, false) 238 | } 239 | 240 | // Wrap takes any readable channel type (chan or <-chan but not chan<-) and 241 | // exposes it as a SimpleOutChannel for easy integration with existing channel sources. 242 | // Wrap adds an unavoidable buffer around the input channel, which can mess with 243 | // synchronization or the apparent length of the input channel. 244 | // It panics if the input is not a readable channel. 245 | func Wrap(ch interface{}) SimpleOutChannel { 246 | t := reflect.TypeOf(ch) 247 | if t.Kind() != reflect.Chan || t.ChanDir()&reflect.RecvDir == 0 { 248 | panic("channels: input to Wrap must be readable channel") 249 | } 250 | realChan := make(chan interface{}) 251 | 252 | go func() { 253 | v := reflect.ValueOf(ch) 254 | for { 255 | x, ok := v.Recv() 256 | if !ok { 257 | close(realChan) 258 | return 259 | } 260 | realChan <- x.Interface() 261 | } 262 | }() 263 | 264 | return NativeOutChannel(realChan) 265 | } 266 | 267 | // Unwrap takes a SimpleOutChannel and uses reflection to pipe it to a typed native channel for 268 | // easy integration with existing channel sources. Output can be any writable channel type (chan or chan<-). 269 | // Unwrap adds an unavoidable buffer around the input channel, which can mess with 270 | // synchronization or the apparent length of the input channel. 271 | // It panics if the output is not a writable channel, or if a value is received that cannot be sent on the 272 | // output channel. 273 | func Unwrap(input SimpleOutChannel, output interface{}) { 274 | t := reflect.TypeOf(output) 275 | if t.Kind() != reflect.Chan || t.ChanDir()&reflect.SendDir == 0 { 276 | panic("channels: input to Unwrap must be readable channel") 277 | } 278 | 279 | go func() { 280 | v := reflect.ValueOf(output) 281 | for { 282 | x, ok := <-input.Out() 283 | if !ok { 284 | v.Close() 285 | return 286 | } 287 | v.Send(reflect.ValueOf(x)) 288 | } 289 | }() 290 | } 291 | -------------------------------------------------------------------------------- /channels_test.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func testChannel(t *testing.T, name string, ch Channel) { 10 | go func() { 11 | for i := 0; i < 1000; i++ { 12 | ch.In() <- i 13 | } 14 | ch.Close() 15 | }() 16 | for i := 0; i < 1000; i++ { 17 | val := <-ch.Out() 18 | if i != val.(int) { 19 | t.Fatal(name, "expected", i, "but got", val.(int)) 20 | } 21 | } 22 | } 23 | 24 | func testChannelPair(t *testing.T, name string, in InChannel, out OutChannel) { 25 | go func() { 26 | for i := 0; i < 1000; i++ { 27 | in.In() <- i 28 | } 29 | in.Close() 30 | }() 31 | for i := 0; i < 1000; i++ { 32 | val := <-out.Out() 33 | if i != val.(int) { 34 | t.Fatal("pair", name, "expected", i, "but got", val.(int)) 35 | } 36 | } 37 | } 38 | 39 | func testChannelConcurrentAccessors(t *testing.T, name string, ch Channel) { 40 | // no asserts here, this is just for the race detector's benefit 41 | go ch.Len() 42 | go ch.Cap() 43 | 44 | go func() { 45 | ch.In() <- nil 46 | }() 47 | 48 | go func() { 49 | <-ch.Out() 50 | }() 51 | } 52 | 53 | func TestPipe(t *testing.T) { 54 | a := NewNativeChannel(None) 55 | b := NewNativeChannel(None) 56 | 57 | Pipe(a, b) 58 | 59 | testChannelPair(t, "pipe", a, b) 60 | } 61 | 62 | func TestWeakPipe(t *testing.T) { 63 | a := NewNativeChannel(None) 64 | b := NewNativeChannel(None) 65 | 66 | WeakPipe(a, b) 67 | 68 | testChannelPair(t, "pipe", a, b) 69 | } 70 | 71 | func testMultiplex(t *testing.T, multi func(output SimpleInChannel, inputs ...SimpleOutChannel)) { 72 | a := NewNativeChannel(None) 73 | b := NewNativeChannel(None) 74 | 75 | multi(b, a) 76 | 77 | testChannelPair(t, "simple multiplex", a, b) 78 | 79 | a = NewNativeChannel(None) 80 | inputs := []Channel{ 81 | NewNativeChannel(None), 82 | NewNativeChannel(None), 83 | NewNativeChannel(None), 84 | NewNativeChannel(None), 85 | } 86 | 87 | multi(a, inputs[0], inputs[1], inputs[2], inputs[3]) 88 | 89 | go func() { 90 | rand.Seed(time.Now().Unix()) 91 | for i := 0; i < 1000; i++ { 92 | inputs[rand.Intn(len(inputs))].In() <- i 93 | } 94 | for i := range inputs { 95 | inputs[i].Close() 96 | } 97 | }() 98 | for i := 0; i < 1000; i++ { 99 | val := <-a.Out() 100 | if i != val.(int) { 101 | t.Fatal("multiplexing expected", i, "but got", val.(int)) 102 | } 103 | } 104 | } 105 | 106 | func TestMultiplex(t *testing.T) { 107 | testMultiplex(t, Multiplex) 108 | } 109 | 110 | func TestWeakMultiplex(t *testing.T) { 111 | testMultiplex(t, WeakMultiplex) 112 | } 113 | 114 | func testTee(t *testing.T, tee func(input SimpleOutChannel, outputs ...SimpleInChannel)) { 115 | a := NewNativeChannel(None) 116 | b := NewNativeChannel(None) 117 | 118 | tee(a, b) 119 | 120 | testChannelPair(t, "simple tee", a, b) 121 | 122 | a = NewNativeChannel(None) 123 | outputs := []Channel{ 124 | NewNativeChannel(None), 125 | NewNativeChannel(None), 126 | NewNativeChannel(None), 127 | NewNativeChannel(None), 128 | } 129 | 130 | tee(a, outputs[0], outputs[1], outputs[2], outputs[3]) 131 | 132 | go func() { 133 | for i := 0; i < 1000; i++ { 134 | a.In() <- i 135 | } 136 | a.Close() 137 | }() 138 | for i := 0; i < 1000; i++ { 139 | for _, output := range outputs { 140 | val := <-output.Out() 141 | if i != val.(int) { 142 | t.Fatal("teeing expected", i, "but got", val.(int)) 143 | } 144 | } 145 | } 146 | } 147 | 148 | func TestTee(t *testing.T) { 149 | testTee(t, Tee) 150 | } 151 | 152 | func TestWeakTee(t *testing.T) { 153 | testTee(t, WeakTee) 154 | } 155 | 156 | func testDistribute(t *testing.T, dist func(input SimpleOutChannel, outputs ...SimpleInChannel)) { 157 | a := NewNativeChannel(None) 158 | b := NewNativeChannel(None) 159 | 160 | dist(a, b) 161 | 162 | testChannelPair(t, "simple distribute", a, b) 163 | 164 | a = NewNativeChannel(None) 165 | outputs := []Channel{ 166 | NewNativeChannel(None), 167 | NewNativeChannel(None), 168 | NewNativeChannel(None), 169 | NewNativeChannel(None), 170 | } 171 | 172 | dist(a, outputs[0], outputs[1], outputs[2], outputs[3]) 173 | 174 | go func() { 175 | for i := 0; i < 1000; i++ { 176 | a.In() <- i 177 | } 178 | a.Close() 179 | }() 180 | 181 | received := make([]bool, 1000) 182 | for _ = range received { 183 | var val interface{} 184 | select { 185 | case val = <-outputs[0].Out(): 186 | case val = <-outputs[1].Out(): 187 | case val = <-outputs[2].Out(): 188 | case val = <-outputs[3].Out(): 189 | } 190 | if received[val.(int)] { 191 | t.Fatal("distribute got value twice", val.(int)) 192 | } 193 | received[val.(int)] = true 194 | } 195 | for i := range received { 196 | if !received[i] { 197 | t.Fatal("distribute missed", i) 198 | } 199 | } 200 | } 201 | 202 | func TestDistribute(t *testing.T) { 203 | testDistribute(t, Distribute) 204 | } 205 | 206 | func TestWeakDistribute(t *testing.T) { 207 | testDistribute(t, WeakDistribute) 208 | } 209 | 210 | func TestWrap(t *testing.T) { 211 | rawChan := make(chan int, 5) 212 | ch := Wrap(rawChan) 213 | 214 | for i := 0; i < 5; i++ { 215 | rawChan <- i 216 | } 217 | close(rawChan) 218 | 219 | for i := 0; i < 5; i++ { 220 | x := (<-ch.Out()).(int) 221 | if x != i { 222 | t.Error("Wrapped value", x, "was expecting", i) 223 | } 224 | } 225 | _, ok := <-ch.Out() 226 | if ok { 227 | t.Error("Wrapped channel didn't close") 228 | } 229 | } 230 | 231 | func TestUnwrap(t *testing.T) { 232 | rawChan := make(chan int) 233 | ch := NewNativeChannel(5) 234 | Unwrap(ch, rawChan) 235 | 236 | for i := 0; i < 5; i++ { 237 | ch.In() <- i 238 | } 239 | ch.Close() 240 | 241 | for i := 0; i < 5; i++ { 242 | x := <-rawChan 243 | if x != i { 244 | t.Error("Unwrapped value", x, "was expecting", i) 245 | } 246 | } 247 | _, ok := <-rawChan 248 | if ok { 249 | t.Error("Unwrapped channel didn't close") 250 | } 251 | } 252 | 253 | func ExampleChannel() { 254 | var ch Channel 255 | 256 | ch = NewInfiniteChannel() 257 | 258 | for i := 0; i < 10; i++ { 259 | ch.In() <- nil 260 | } 261 | 262 | for i := 0; i < 10; i++ { 263 | <-ch.Out() 264 | } 265 | } 266 | -------------------------------------------------------------------------------- /infinite_channel.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "github.com/eapache/queue" 4 | 5 | // InfiniteChannel implements the Channel interface with an infinite buffer between the input and the output. 6 | type InfiniteChannel struct { 7 | input, output chan interface{} 8 | length chan int 9 | buffer *queue.Queue 10 | } 11 | 12 | func NewInfiniteChannel() *InfiniteChannel { 13 | ch := &InfiniteChannel{ 14 | input: make(chan interface{}), 15 | output: make(chan interface{}), 16 | length: make(chan int), 17 | buffer: queue.New(), 18 | } 19 | go ch.infiniteBuffer() 20 | return ch 21 | } 22 | 23 | func (ch *InfiniteChannel) In() chan<- interface{} { 24 | return ch.input 25 | } 26 | 27 | func (ch *InfiniteChannel) Out() <-chan interface{} { 28 | return ch.output 29 | } 30 | 31 | func (ch *InfiniteChannel) Len() int { 32 | return <-ch.length 33 | } 34 | 35 | func (ch *InfiniteChannel) Cap() BufferCap { 36 | return Infinity 37 | } 38 | 39 | func (ch *InfiniteChannel) Close() { 40 | close(ch.input) 41 | } 42 | 43 | func (ch *InfiniteChannel) infiniteBuffer() { 44 | var input, output chan interface{} 45 | var next interface{} 46 | input = ch.input 47 | 48 | for input != nil || output != nil { 49 | select { 50 | case elem, open := <-input: 51 | if open { 52 | ch.buffer.Add(elem) 53 | } else { 54 | input = nil 55 | } 56 | case output <- next: 57 | ch.buffer.Remove() 58 | case ch.length <- ch.buffer.Length(): 59 | } 60 | 61 | if ch.buffer.Length() > 0 { 62 | output = ch.output 63 | next = ch.buffer.Peek() 64 | } else { 65 | output = nil 66 | next = nil 67 | } 68 | } 69 | 70 | close(ch.output) 71 | close(ch.length) 72 | } 73 | -------------------------------------------------------------------------------- /infinite_channel_test.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "testing" 4 | 5 | func TestInfiniteChannel(t *testing.T) { 6 | var ch Channel 7 | 8 | ch = NewInfiniteChannel() 9 | testChannel(t, "infinite channel", ch) 10 | 11 | ch = NewInfiniteChannel() 12 | testChannelPair(t, "infinite channel", ch, ch) 13 | 14 | ch = NewInfiniteChannel() 15 | testChannelConcurrentAccessors(t, "infinite channel", ch) 16 | } 17 | 18 | func BenchmarkInfiniteChannelSerial(b *testing.B) { 19 | ch := NewInfiniteChannel() 20 | for i := 0; i < b.N; i++ { 21 | ch.In() <- nil 22 | } 23 | for i := 0; i < b.N; i++ { 24 | <-ch.Out() 25 | } 26 | } 27 | 28 | func BenchmarkInfiniteChannelParallel(b *testing.B) { 29 | ch := NewInfiniteChannel() 30 | go func() { 31 | for i := 0; i < b.N; i++ { 32 | <-ch.Out() 33 | } 34 | ch.Close() 35 | }() 36 | for i := 0; i < b.N; i++ { 37 | ch.In() <- nil 38 | } 39 | <-ch.Out() 40 | } 41 | 42 | func BenchmarkInfiniteChannelTickTock(b *testing.B) { 43 | ch := NewInfiniteChannel() 44 | for i := 0; i < b.N; i++ { 45 | ch.In() <- nil 46 | <-ch.Out() 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /native_channel.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | // NativeInChannel implements the InChannel interface by wrapping a native go write-only channel. 4 | type NativeInChannel chan<- interface{} 5 | 6 | func (ch NativeInChannel) In() chan<- interface{} { 7 | return ch 8 | } 9 | 10 | func (ch NativeInChannel) Len() int { 11 | return len(ch) 12 | } 13 | 14 | func (ch NativeInChannel) Cap() BufferCap { 15 | return BufferCap(cap(ch)) 16 | } 17 | 18 | func (ch NativeInChannel) Close() { 19 | close(ch) 20 | } 21 | 22 | // NativeOutChannel implements the OutChannel interface by wrapping a native go read-only channel. 23 | type NativeOutChannel <-chan interface{} 24 | 25 | func (ch NativeOutChannel) Out() <-chan interface{} { 26 | return ch 27 | } 28 | 29 | func (ch NativeOutChannel) Len() int { 30 | return len(ch) 31 | } 32 | 33 | func (ch NativeOutChannel) Cap() BufferCap { 34 | return BufferCap(cap(ch)) 35 | } 36 | 37 | // NativeChannel implements the Channel interface by wrapping a native go channel. 38 | type NativeChannel chan interface{} 39 | 40 | // NewNativeChannel makes a new NativeChannel with the given buffer size. Just a convenience wrapper 41 | // to avoid having to cast the result of make(). 42 | func NewNativeChannel(size BufferCap) NativeChannel { 43 | return make(chan interface{}, size) 44 | } 45 | 46 | func (ch NativeChannel) In() chan<- interface{} { 47 | return ch 48 | } 49 | 50 | func (ch NativeChannel) Out() <-chan interface{} { 51 | return ch 52 | } 53 | 54 | func (ch NativeChannel) Len() int { 55 | return len(ch) 56 | } 57 | 58 | func (ch NativeChannel) Cap() BufferCap { 59 | return BufferCap(cap(ch)) 60 | } 61 | 62 | func (ch NativeChannel) Close() { 63 | close(ch) 64 | } 65 | 66 | // DeadChannel is a placeholder implementation of the Channel interface with no buffer 67 | // that is never ready for reading or writing. Closing a dead channel is a no-op. 68 | // Behaves almost like NativeChannel(nil) except that closing a nil NativeChannel will panic. 69 | type DeadChannel struct{} 70 | 71 | func NewDeadChannel() DeadChannel { 72 | return DeadChannel{} 73 | } 74 | 75 | func (ch DeadChannel) In() chan<- interface{} { 76 | return nil 77 | } 78 | 79 | func (ch DeadChannel) Out() <-chan interface{} { 80 | return nil 81 | } 82 | 83 | func (ch DeadChannel) Len() int { 84 | return 0 85 | } 86 | 87 | func (ch DeadChannel) Cap() BufferCap { 88 | return BufferCap(0) 89 | } 90 | 91 | func (ch DeadChannel) Close() { 92 | } 93 | -------------------------------------------------------------------------------- /native_channel_test.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "testing" 4 | 5 | func TestNativeChannels(t *testing.T) { 6 | var ch Channel 7 | 8 | ch = NewNativeChannel(None) 9 | testChannel(t, "bufferless native channel", ch) 10 | 11 | ch = NewNativeChannel(None) 12 | testChannelPair(t, "bufferless native channel", ch, ch) 13 | 14 | ch = NewNativeChannel(5) 15 | testChannel(t, "5-buffer native channel", ch) 16 | 17 | ch = NewNativeChannel(5) 18 | testChannelPair(t, "5-buffer native channel", ch, ch) 19 | 20 | ch = NewNativeChannel(None) 21 | testChannelConcurrentAccessors(t, "native channel", ch) 22 | } 23 | 24 | func TestNativeInOutChannels(t *testing.T) { 25 | ch1 := make(chan interface{}) 26 | ch2 := make(chan interface{}) 27 | 28 | Pipe(NativeOutChannel(ch1), NativeInChannel(ch2)) 29 | NativeInChannel(ch1).Close() 30 | } 31 | 32 | func TestDeadChannel(t *testing.T) { 33 | ch := NewDeadChannel() 34 | 35 | if ch.Len() != 0 { 36 | t.Error("dead channel length not 0") 37 | } 38 | if ch.Cap() != 0 { 39 | t.Error("dead channel cap not 0") 40 | } 41 | 42 | select { 43 | case <-ch.Out(): 44 | t.Error("read from a dead channel") 45 | default: 46 | } 47 | 48 | select { 49 | case ch.In() <- nil: 50 | t.Error("wrote to a dead channel") 51 | default: 52 | } 53 | 54 | ch.Close() 55 | 56 | ch = NewDeadChannel() 57 | testChannelConcurrentAccessors(t, "dead channel", ch) 58 | } 59 | -------------------------------------------------------------------------------- /overflowing_channel.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "github.com/eapache/queue" 4 | 5 | // OverflowingChannel implements the Channel interface in a way that never blocks the writer. 6 | // Specifically, if a value is written to an OverflowingChannel when its buffer is full 7 | // (or, in an unbuffered case, when the recipient is not ready) then that value is simply discarded. 8 | // Note that Go's scheduler can cause discarded values when they could be avoided, simply by scheduling 9 | // the writer before the reader, so caveat emptor. 10 | // For the opposite behaviour (discarding the oldest element, not the newest) see RingChannel. 11 | type OverflowingChannel struct { 12 | input, output chan interface{} 13 | length chan int 14 | buffer *queue.Queue 15 | size BufferCap 16 | } 17 | 18 | func NewOverflowingChannel(size BufferCap) *OverflowingChannel { 19 | if size < 0 && size != Infinity { 20 | panic("channels: invalid negative size in NewOverflowingChannel") 21 | } 22 | ch := &OverflowingChannel{ 23 | input: make(chan interface{}), 24 | output: make(chan interface{}), 25 | length: make(chan int), 26 | size: size, 27 | } 28 | if size == None { 29 | go ch.overflowingDirect() 30 | } else { 31 | ch.buffer = queue.New() 32 | go ch.overflowingBuffer() 33 | } 34 | return ch 35 | } 36 | 37 | func (ch *OverflowingChannel) In() chan<- interface{} { 38 | return ch.input 39 | } 40 | 41 | func (ch *OverflowingChannel) Out() <-chan interface{} { 42 | return ch.output 43 | } 44 | 45 | func (ch *OverflowingChannel) Len() int { 46 | if ch.size == None { 47 | return 0 48 | } else { 49 | return <-ch.length 50 | } 51 | } 52 | 53 | func (ch *OverflowingChannel) Cap() BufferCap { 54 | return ch.size 55 | } 56 | 57 | func (ch *OverflowingChannel) Close() { 58 | close(ch.input) 59 | } 60 | 61 | // for entirely unbuffered cases 62 | func (ch *OverflowingChannel) overflowingDirect() { 63 | for elem := range ch.input { 64 | // if we can't write it immediately, drop it and move on 65 | select { 66 | case ch.output <- elem: 67 | default: 68 | } 69 | } 70 | close(ch.output) 71 | } 72 | 73 | // for all buffered cases 74 | func (ch *OverflowingChannel) overflowingBuffer() { 75 | var input, output chan interface{} 76 | var next interface{} 77 | input = ch.input 78 | 79 | for input != nil || output != nil { 80 | select { 81 | // Prefer to write if possible, which is surprisingly effective in reducing 82 | // dropped elements due to overflow. The naive read/write select chooses randomly 83 | // when both channels are ready, which produces unnecessary drops 50% of the time. 84 | case output <- next: 85 | ch.buffer.Remove() 86 | default: 87 | select { 88 | case elem, open := <-input: 89 | if open { 90 | if ch.size == Infinity || ch.buffer.Length() < int(ch.size) { 91 | ch.buffer.Add(elem) 92 | } 93 | } else { 94 | input = nil 95 | } 96 | case output <- next: 97 | ch.buffer.Remove() 98 | case ch.length <- ch.buffer.Length(): 99 | } 100 | } 101 | 102 | if ch.buffer.Length() > 0 { 103 | output = ch.output 104 | next = ch.buffer.Peek() 105 | } else { 106 | output = nil 107 | next = nil 108 | } 109 | } 110 | 111 | close(ch.output) 112 | close(ch.length) 113 | } 114 | -------------------------------------------------------------------------------- /overflowing_channel_test.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "testing" 4 | 5 | func TestOverflowingChannel(t *testing.T) { 6 | var ch Channel 7 | 8 | ch = NewOverflowingChannel(Infinity) // yes this is rather silly, but it should work 9 | testChannel(t, "infinite overflowing channel", ch) 10 | 11 | ch = NewOverflowingChannel(None) 12 | go func() { 13 | for i := 0; i < 1000; i++ { 14 | ch.In() <- i 15 | } 16 | ch.Close() 17 | }() 18 | prev := -1 19 | for i := range ch.Out() { 20 | if prev >= i.(int) { 21 | t.Fatal("overflowing channel prev", prev, "but got", i.(int)) 22 | } 23 | } 24 | 25 | ch = NewOverflowingChannel(10) 26 | for i := 0; i < 1000; i++ { 27 | ch.In() <- i 28 | } 29 | ch.Close() 30 | for i := 0; i < 10; i++ { 31 | val := <-ch.Out() 32 | if i != val.(int) { 33 | t.Fatal("overflowing channel expected", i, "but got", val.(int)) 34 | } 35 | } 36 | if val, open := <-ch.Out(); open == true { 37 | t.Fatal("overflowing channel expected closed but got", val) 38 | } 39 | 40 | ch = NewOverflowingChannel(None) 41 | ch.In() <- 0 42 | ch.Close() 43 | if val, open := <-ch.Out(); open == true { 44 | t.Fatal("overflowing channel expected closed but got", val) 45 | } 46 | 47 | ch = NewOverflowingChannel(2) 48 | testChannelConcurrentAccessors(t, "overflowing channel", ch) 49 | } 50 | -------------------------------------------------------------------------------- /resizable_channel.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "github.com/eapache/queue" 4 | 5 | // ResizableChannel implements the Channel interface with a resizable buffer between the input and the output. 6 | // The channel initially has a buffer size of 1, but can be resized by calling Resize(). 7 | // 8 | // Resizing to a buffer capacity of None is, unfortunately, not supported and will panic 9 | // (see https://github.com/eapache/channels/issues/1). 10 | // Resizing back and forth between a finite and infinite buffer is fully supported. 11 | type ResizableChannel struct { 12 | input, output chan interface{} 13 | length chan int 14 | capacity, resize chan BufferCap 15 | size BufferCap 16 | buffer *queue.Queue 17 | } 18 | 19 | func NewResizableChannel() *ResizableChannel { 20 | ch := &ResizableChannel{ 21 | input: make(chan interface{}), 22 | output: make(chan interface{}), 23 | length: make(chan int), 24 | capacity: make(chan BufferCap), 25 | resize: make(chan BufferCap), 26 | size: 1, 27 | buffer: queue.New(), 28 | } 29 | go ch.magicBuffer() 30 | return ch 31 | } 32 | 33 | func (ch *ResizableChannel) In() chan<- interface{} { 34 | return ch.input 35 | } 36 | 37 | func (ch *ResizableChannel) Out() <-chan interface{} { 38 | return ch.output 39 | } 40 | 41 | func (ch *ResizableChannel) Len() int { 42 | return <-ch.length 43 | } 44 | 45 | func (ch *ResizableChannel) Cap() BufferCap { 46 | val, open := <-ch.capacity 47 | if open { 48 | return val 49 | } else { 50 | return ch.size 51 | } 52 | } 53 | 54 | func (ch *ResizableChannel) Close() { 55 | close(ch.input) 56 | } 57 | 58 | func (ch *ResizableChannel) Resize(newSize BufferCap) { 59 | if newSize == None { 60 | panic("channels: ResizableChannel does not support unbuffered behaviour") 61 | } 62 | if newSize < 0 && newSize != Infinity { 63 | panic("channels: invalid negative size trying to resize channel") 64 | } 65 | ch.resize <- newSize 66 | } 67 | 68 | func (ch *ResizableChannel) magicBuffer() { 69 | var input, output, nextInput chan interface{} 70 | var next interface{} 71 | nextInput = ch.input 72 | input = nextInput 73 | 74 | for input != nil || output != nil { 75 | select { 76 | case elem, open := <-input: 77 | if open { 78 | ch.buffer.Add(elem) 79 | } else { 80 | input = nil 81 | nextInput = nil 82 | } 83 | case output <- next: 84 | ch.buffer.Remove() 85 | case ch.size = <-ch.resize: 86 | case ch.length <- ch.buffer.Length(): 87 | case ch.capacity <- ch.size: 88 | } 89 | 90 | if ch.buffer.Length() == 0 { 91 | output = nil 92 | next = nil 93 | } else { 94 | output = ch.output 95 | next = ch.buffer.Peek() 96 | } 97 | 98 | if ch.size != Infinity && ch.buffer.Length() >= int(ch.size) { 99 | input = nil 100 | } else { 101 | input = nextInput 102 | } 103 | } 104 | 105 | close(ch.output) 106 | close(ch.resize) 107 | close(ch.length) 108 | close(ch.capacity) 109 | } 110 | -------------------------------------------------------------------------------- /resizable_channel_test.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | ) 7 | 8 | func TestResizableChannel(t *testing.T) { 9 | var ch *ResizableChannel 10 | 11 | ch = NewResizableChannel() 12 | testChannel(t, "default resizable channel", ch) 13 | 14 | ch = NewResizableChannel() 15 | testChannelPair(t, "default resizable channel", ch, ch) 16 | 17 | ch = NewResizableChannel() 18 | ch.Resize(Infinity) 19 | testChannel(t, "infinite resizable channel", ch) 20 | 21 | ch = NewResizableChannel() 22 | ch.Resize(Infinity) 23 | testChannelPair(t, "infinite resizable channel", ch, ch) 24 | 25 | ch = NewResizableChannel() 26 | ch.Resize(5) 27 | testChannel(t, "5-buffer resizable channel", ch) 28 | 29 | ch = NewResizableChannel() 30 | ch.Resize(5) 31 | testChannelPair(t, "5-buffer resizable channel", ch, ch) 32 | 33 | ch = NewResizableChannel() 34 | testChannelConcurrentAccessors(t, "resizable channel", ch) 35 | } 36 | 37 | func TestResizableChannelOnline(t *testing.T) { 38 | stopper := make(chan bool) 39 | ch := NewResizableChannel() 40 | go func() { 41 | for i := 0; i < 1000; i++ { 42 | ch.In() <- i 43 | } 44 | <-stopper 45 | ch.Close() 46 | }() 47 | 48 | go func() { 49 | for i := 0; i < 1000; i++ { 50 | ch.Resize(BufferCap(rand.Intn(50) + 1)) 51 | } 52 | close(stopper) 53 | }() 54 | 55 | for i := 0; i < 1000; i++ { 56 | val := <-ch.Out() 57 | if i != val.(int) { 58 | t.Fatal("resizable channel expected", i, "but got", val.(int)) 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /ring_channel.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "github.com/eapache/queue" 4 | 5 | // RingChannel implements the Channel interface in a way that never blocks the writer. 6 | // Specifically, if a value is written to a RingChannel when its buffer is full then the oldest 7 | // value in the buffer is discarded to make room (just like a standard ring-buffer). 8 | // Note that Go's scheduler can cause discarded values when they could be avoided, simply by scheduling 9 | // the writer before the reader, so caveat emptor. 10 | // For the opposite behaviour (discarding the newest element, not the oldest) see OverflowingChannel. 11 | type RingChannel struct { 12 | input, output chan interface{} 13 | length chan int 14 | buffer *queue.Queue 15 | size BufferCap 16 | } 17 | 18 | func NewRingChannel(size BufferCap) *RingChannel { 19 | if size < 0 && size != Infinity { 20 | panic("channels: invalid negative size in NewRingChannel") 21 | } 22 | ch := &RingChannel{ 23 | input: make(chan interface{}), 24 | output: make(chan interface{}), 25 | buffer: queue.New(), 26 | size: size, 27 | } 28 | if size == None { 29 | go ch.overflowingDirect() 30 | } else { 31 | ch.length = make(chan int) 32 | go ch.ringBuffer() 33 | } 34 | return ch 35 | } 36 | 37 | func (ch *RingChannel) In() chan<- interface{} { 38 | return ch.input 39 | } 40 | 41 | func (ch *RingChannel) Out() <-chan interface{} { 42 | return ch.output 43 | } 44 | 45 | func (ch *RingChannel) Len() int { 46 | if ch.size == None { 47 | return 0 48 | } else { 49 | return <-ch.length 50 | } 51 | } 52 | 53 | func (ch *RingChannel) Cap() BufferCap { 54 | return ch.size 55 | } 56 | 57 | func (ch *RingChannel) Close() { 58 | close(ch.input) 59 | } 60 | 61 | // for entirely unbuffered cases 62 | func (ch *RingChannel) overflowingDirect() { 63 | for elem := range ch.input { 64 | // if we can't write it immediately, drop it and move on 65 | select { 66 | case ch.output <- elem: 67 | default: 68 | } 69 | } 70 | close(ch.output) 71 | } 72 | 73 | // for all buffered cases 74 | func (ch *RingChannel) ringBuffer() { 75 | var input, output chan interface{} 76 | var next interface{} 77 | input = ch.input 78 | 79 | for input != nil || output != nil { 80 | select { 81 | // Prefer to write if possible, which is surprisingly effective in reducing 82 | // dropped elements due to overflow. The naive read/write select chooses randomly 83 | // when both channels are ready, which produces unnecessary drops 50% of the time. 84 | case output <- next: 85 | ch.buffer.Remove() 86 | default: 87 | select { 88 | case elem, open := <-input: 89 | if open { 90 | ch.buffer.Add(elem) 91 | if ch.size != Infinity && ch.buffer.Length() > int(ch.size) { 92 | ch.buffer.Remove() 93 | } 94 | } else { 95 | input = nil 96 | } 97 | case output <- next: 98 | ch.buffer.Remove() 99 | case ch.length <- ch.buffer.Length(): 100 | } 101 | } 102 | 103 | if ch.buffer.Length() > 0 { 104 | output = ch.output 105 | next = ch.buffer.Peek() 106 | } else { 107 | output = nil 108 | next = nil 109 | } 110 | } 111 | 112 | close(ch.output) 113 | close(ch.length) 114 | } 115 | -------------------------------------------------------------------------------- /ring_channel_test.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "testing" 4 | 5 | func TestRingChannel(t *testing.T) { 6 | var ch Channel 7 | 8 | ch = NewRingChannel(Infinity) // yes this is rather silly, but it should work 9 | testChannel(t, "infinite ring-buffer channel", ch) 10 | 11 | ch = NewRingChannel(None) 12 | go func() { 13 | for i := 0; i < 1000; i++ { 14 | ch.In() <- i 15 | } 16 | ch.Close() 17 | }() 18 | prev := -1 19 | for i := range ch.Out() { 20 | if prev >= i.(int) { 21 | t.Fatal("ring channel prev", prev, "but got", i.(int)) 22 | } 23 | } 24 | 25 | ch = NewRingChannel(10) 26 | for i := 0; i < 1000; i++ { 27 | ch.In() <- i 28 | } 29 | ch.Close() 30 | for i := 990; i < 1000; i++ { 31 | val := <-ch.Out() 32 | if i != val.(int) { 33 | t.Fatal("ring channel expected", i, "but got", val.(int)) 34 | } 35 | } 36 | if val, open := <-ch.Out(); open == true { 37 | t.Fatal("ring channel expected closed but got", val) 38 | } 39 | 40 | ch = NewRingChannel(None) 41 | ch.In() <- 0 42 | ch.Close() 43 | if val, open := <-ch.Out(); open == true { 44 | t.Fatal("ring channel expected closed but got", val) 45 | } 46 | 47 | ch = NewRingChannel(2) 48 | testChannelConcurrentAccessors(t, "ring channel", ch) 49 | } 50 | -------------------------------------------------------------------------------- /shared_buffer.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import ( 4 | "reflect" 5 | 6 | "github.com/eapache/queue" 7 | ) 8 | 9 | //sharedBufferChannel implements SimpleChannel and is created by the public 10 | //SharedBuffer type below 11 | type sharedBufferChannel struct { 12 | in chan interface{} 13 | out chan interface{} 14 | buf *queue.Queue 15 | closed bool 16 | } 17 | 18 | func (sch *sharedBufferChannel) In() chan<- interface{} { 19 | return sch.in 20 | } 21 | 22 | func (sch *sharedBufferChannel) Out() <-chan interface{} { 23 | return sch.out 24 | } 25 | 26 | func (sch *sharedBufferChannel) Close() { 27 | close(sch.in) 28 | } 29 | 30 | //SharedBuffer implements the Buffer interface, and permits multiple SimpleChannel instances to "share" a single buffer. 31 | //Each channel spawned by NewChannel has its own internal queue (so values flowing through do not get mixed up with 32 | //other channels) but the total number of elements buffered by all spawned channels is limited to a single capacity. This 33 | //means *all* such channels block and unblock for writing together. The primary use case is for implementing pipeline-style 34 | //parallelism with goroutines, limiting the total number of elements in the pipeline without limiting the number of elements 35 | //at any particular step. 36 | // Warning: this type has an unavoidable deadlock as implemented (see https://github.com/eapache/channels/issues/28). 37 | type SharedBuffer struct { 38 | cases []reflect.SelectCase // 2n+1 of these; [0] is for control, [1,3,5...] for recv, [2,4,6...] for send 39 | chans []*sharedBufferChannel // n of these 40 | count int 41 | size BufferCap 42 | in chan *sharedBufferChannel 43 | } 44 | 45 | func NewSharedBuffer(size BufferCap) *SharedBuffer { 46 | if size < 0 && size != Infinity { 47 | panic("channels: invalid negative size in NewSharedBuffer") 48 | } else if size == None { 49 | panic("channels: SharedBuffer does not support unbuffered behaviour") 50 | } 51 | 52 | buf := &SharedBuffer{ 53 | size: size, 54 | in: make(chan *sharedBufferChannel), 55 | } 56 | 57 | buf.cases = append(buf.cases, reflect.SelectCase{ 58 | Dir: reflect.SelectRecv, 59 | Chan: reflect.ValueOf(buf.in), 60 | }) 61 | 62 | go buf.mainLoop() 63 | 64 | return buf 65 | } 66 | 67 | //NewChannel spawns and returns a new channel sharing the underlying buffer. 68 | func (buf *SharedBuffer) NewChannel() SimpleChannel { 69 | ch := &sharedBufferChannel{ 70 | in: make(chan interface{}), 71 | out: make(chan interface{}), 72 | buf: queue.New(), 73 | } 74 | buf.in <- ch 75 | return ch 76 | } 77 | 78 | //Close shuts down the SharedBuffer. It is an error to call Close while channels are still using 79 | //the buffer (I'm not really sure what would happen if you do so). 80 | func (buf *SharedBuffer) Close() { 81 | // TODO: what if there are still active channels using this buffer? 82 | close(buf.in) 83 | } 84 | 85 | func (buf *SharedBuffer) mainLoop() { 86 | for { 87 | i, val, ok := reflect.Select(buf.cases) 88 | 89 | if i == 0 { 90 | if !ok { 91 | //Close was called on the SharedBuffer itself 92 | return 93 | } 94 | 95 | //NewChannel was called on the SharedBuffer 96 | ch := val.Interface().(*sharedBufferChannel) 97 | buf.chans = append(buf.chans, ch) 98 | buf.cases = append(buf.cases, 99 | reflect.SelectCase{Dir: reflect.SelectRecv}, 100 | reflect.SelectCase{Dir: reflect.SelectSend}, 101 | ) 102 | if buf.size == Infinity || buf.count < int(buf.size) { 103 | buf.cases[len(buf.cases)-2].Chan = reflect.ValueOf(ch.in) 104 | } 105 | } else if i%2 == 0 { 106 | //Send 107 | if buf.count == int(buf.size) { 108 | //room in the buffer again, re-enable all recv cases 109 | for j := range buf.chans { 110 | if !buf.chans[j].closed { 111 | buf.cases[(j*2)+1].Chan = reflect.ValueOf(buf.chans[j].in) 112 | } 113 | } 114 | } 115 | buf.count-- 116 | ch := buf.chans[(i-1)/2] 117 | if ch.buf.Length() > 0 { 118 | buf.cases[i].Send = reflect.ValueOf(ch.buf.Peek()) 119 | ch.buf.Remove() 120 | } else { 121 | //nothing left for this channel to send, disable sending 122 | buf.cases[i].Chan = reflect.Value{} 123 | buf.cases[i].Send = reflect.Value{} 124 | if ch.closed { 125 | // and it was closed, so close the output channel 126 | //TODO: shrink slice 127 | close(ch.out) 128 | } 129 | } 130 | } else { 131 | ch := buf.chans[i/2] 132 | if ok { 133 | //Receive 134 | buf.count++ 135 | if ch.buf.Length() == 0 && !buf.cases[i+1].Chan.IsValid() { 136 | //this channel now has something to send 137 | buf.cases[i+1].Chan = reflect.ValueOf(ch.out) 138 | buf.cases[i+1].Send = val 139 | } else { 140 | ch.buf.Add(val.Interface()) 141 | } 142 | if buf.count == int(buf.size) { 143 | //buffer full, disable recv cases 144 | for j := range buf.chans { 145 | buf.cases[(j*2)+1].Chan = reflect.Value{} 146 | } 147 | } 148 | } else { 149 | //Close 150 | buf.cases[i].Chan = reflect.Value{} 151 | ch.closed = true 152 | if ch.buf.Length() == 0 && !buf.cases[i+1].Chan.IsValid() { 153 | //nothing pending, close the out channel right away 154 | //TODO: shrink slice 155 | close(ch.out) 156 | } 157 | } 158 | } 159 | } 160 | } 161 | 162 | func (buf *SharedBuffer) Len() int { 163 | return buf.count 164 | } 165 | 166 | func (buf *SharedBuffer) Cap() BufferCap { 167 | return buf.size 168 | } 169 | -------------------------------------------------------------------------------- /shared_buffer_test.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "testing" 4 | 5 | func TestSharedBufferSingleton(t *testing.T) { 6 | buf := NewSharedBuffer(3) 7 | 8 | ch := buf.NewChannel() 9 | for i := 0; i < 5; i++ { 10 | ch.In() <- (*int)(nil) 11 | ch.In() <- (*int)(nil) 12 | ch.In() <- (*int)(nil) 13 | select { 14 | case ch.In() <- (*int)(nil): 15 | t.Error("Wrote to full shared-buffer") 16 | default: 17 | } 18 | 19 | <-ch.Out() 20 | <-ch.Out() 21 | <-ch.Out() 22 | select { 23 | case <-ch.Out(): 24 | t.Error("Read from empty shared-buffer") 25 | default: 26 | } 27 | } 28 | 29 | ch.Close() 30 | buf.Close() 31 | } 32 | 33 | func TestSharedBufferMultiple(t *testing.T) { 34 | buf := NewSharedBuffer(3) 35 | 36 | ch1 := buf.NewChannel() 37 | ch2 := buf.NewChannel() 38 | 39 | ch1.In() <- (*int)(nil) 40 | ch1.In() <- (*int)(nil) 41 | ch1.In() <- (*int)(nil) 42 | 43 | select { 44 | case ch2.In() <- (*int)(nil): 45 | t.Error("Wrote to full shared-buffer") 46 | case <-ch2.Out(): 47 | t.Error("Read from empty channel") 48 | default: 49 | } 50 | 51 | <-ch1.Out() 52 | 53 | for i := 0; i < 10; i++ { 54 | ch2.In() <- (*int)(nil) 55 | 56 | select { 57 | case ch1.In() <- (*int)(nil): 58 | t.Error("Wrote to full shared-buffer") 59 | case ch2.In() <- (*int)(nil): 60 | t.Error("Wrote to full shared-buffer") 61 | default: 62 | } 63 | 64 | <-ch2.Out() 65 | } 66 | 67 | <-ch1.Out() 68 | <-ch1.Out() 69 | 70 | ch1.Close() 71 | ch2.Close() 72 | buf.Close() 73 | } 74 | 75 | func TestSharedBufferConcurrent(t *testing.T) { 76 | const threads = 10 77 | const iters = 200 78 | 79 | buf := NewSharedBuffer(3) 80 | done := make(chan bool) 81 | 82 | for i := 0; i < threads; i++ { 83 | go func() { 84 | ch := buf.NewChannel() 85 | for i := 0; i < iters; i++ { 86 | ch.In() <- i 87 | val := <-ch.Out() 88 | if val.(int) != i { 89 | t.Error("Mismatched value out of channel") 90 | } 91 | } 92 | ch.Close() 93 | done <- true 94 | }() 95 | } 96 | 97 | for i := 0; i < threads; i++ { 98 | <-done 99 | } 100 | close(done) 101 | buf.Close() 102 | } 103 | 104 | func ExampleSharedBuffer() { 105 | // never more than 3 elements in the pipeline at once 106 | buf := NewSharedBuffer(3) 107 | 108 | ch1 := buf.NewChannel() 109 | ch2 := buf.NewChannel() 110 | 111 | // or, instead of a straight pipe, implement your pipeline step 112 | Pipe(ch1, ch2) 113 | 114 | // inputs 115 | go func() { 116 | for i := 0; i < 20; i++ { 117 | ch1.In() <- i 118 | } 119 | ch1.Close() 120 | }() 121 | 122 | for _ = range ch2.Out() { 123 | // outputs 124 | } 125 | 126 | buf.Close() 127 | } 128 | --------------------------------------------------------------------------------