├── .travis.yml ├── LICENSE ├── README.md ├── definitions.go ├── definitions_test.go ├── doc.go ├── examples_test.go ├── ratelimiter.go └── ratelimiter_test.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - tip 5 | 6 | env: 7 | - "PATH=/home/travis/gopath/bin:$PATH" 8 | 9 | before_script: 10 | - go get golang.org/x/tools/cmd/cover 11 | - go get github.com/mattn/goveralls 12 | 13 | script: 14 | - go test -v -covermode=count -coverprofile=coverage.out ./... 15 | - goveralls -coverprofile=coverage.out -service travis-ci -repotoken $COVERALLS_TOKEN 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Steven Bogacz & Michael T. Robinson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-pipeline 2 | 3 | [![Build Status](https://travis-ci.org/sbogacz/go-pipeline.svg?branch=master)](https://travis-ci.org/sbogacz/go-pipeline) [![Coverage Status](https://coveralls.io/repos/github/sbogacz/go-pipeline/badge.svg?branch=master)](https://coveralls.io/github/sbogacz/go-pipeline?branch=master) [![Go Report Card](https://goreportcard.com/badge/github.com/sbogacz/go-pipeline)](https://goreportcard.com/report/github.com/sbogacz/go-pipeline) [![GoDoc](https://godoc.org/github.com/sbogacz/go-pipeline?status.svg)](https://godoc.org/github.com/sbogacz/go-pipeline) 4 | 5 | go-pipeline is a Go library to provide some channel "middleware"-like functionality. 6 | It can lead to some clean code when processing various inputs that share a flow. 7 | 8 | ## Runner 9 | 10 | The `Runner` interface provides much of the functionality in the package. It's 11 | defined as 12 | ```go 13 | type Runner interface { 14 | Run(chan interface{}) chan interface{} 15 | } 16 | ``` 17 | 18 | The two implementing types provided are `Operator` and `Flow`, where the latter 19 | is just a collection of `Operator`s, and both provide a `Run` method. 20 | 21 | Example: 22 | 23 | ```go 24 | func multiplier(x int) Operator { 25 | return Operator(func(in chan interface{}, out chan interface{}) { 26 | for m := range in { 27 | n := m.(int) 28 | out <- (int(n) * x) 29 | } 30 | }) 31 | } 32 | ``` 33 | 34 | ## Rate Limiter 35 | 36 | This package also provides a `RateLimiter` function which takes a rate limiter 37 | from the "golang.org/x/time/rate" package, and returns an `Operator` which returns 38 | a channel whose input is throttled by the provided rate limiter. 39 | 40 | ## Examples 41 | 42 | More examples of how to use the pipeline package can be found in the test files 43 | 44 | ## Contributing 45 | 46 | If you'd like to contribute to this project, make sure that you're running [go vet](https://golang.org/cmd/vet/) 47 | and [go lint](https://github.com/golang/lint) before submitting a pull request. If 48 | adding a feature or fixing a bug, please also add a test case verify the new functionality/new fix. 49 | -------------------------------------------------------------------------------- /definitions.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "sync" 4 | 5 | // Runner interface exposes functions that take in a chan interface{} 6 | // and outputs to a chan interface{} 7 | type Runner interface { 8 | Run(<-chan interface{}) chan interface{} 9 | } 10 | 11 | // Operator aliases a function that takes one input channel and one output channel 12 | type Operator func(<-chan interface{}, chan interface{}) 13 | 14 | // Run takes an input channel, and a series of operators, and uses the output 15 | // of each successive operator as the input for the next. This makes the Operator 16 | // implement the Runner interface 17 | func (o Operator) Run(in <-chan interface{}) chan interface{} { 18 | out := make(chan interface{}) 19 | go func() { 20 | o(in, out) 21 | close(out) 22 | }() 23 | return out 24 | } 25 | 26 | // Flow is a slice of Operators that can be applied in sequence 27 | type Flow []Operator 28 | 29 | // NewFlow is syntactic sugar to create a Flow 30 | func NewFlow(ops ...Operator) Flow { 31 | return Flow(ops) 32 | } 33 | 34 | // Run takes an input channel and runs the operators in the slice in order. 35 | // This makes Flow implement the Runner interface 36 | func (f Flow) Run(in chan interface{}) chan interface{} { 37 | for _, m := range f { 38 | in = m.Run(in) 39 | } 40 | return in 41 | } 42 | 43 | // Split takes a single input channel, and broadcasts each item to each handler 44 | // function's channel. 45 | func Split(in chan interface{}, n int) []chan interface{} { 46 | outChans := make([]chan interface{}, n) 47 | for i := 0; i < n; i++ { 48 | outChans[i] = make(chan interface{}) 49 | } 50 | go func() { 51 | for n := range in { 52 | for _, out := range outChans { 53 | out <- n 54 | } 55 | } 56 | for _, out := range outChans { 57 | close(out) 58 | } 59 | }() 60 | return outChans 61 | } 62 | 63 | // Combine takes a variable number of channels and combines their output into 64 | // a single channel, that can still be used with an operator 65 | func Combine(ins ...chan interface{}) chan interface{} { 66 | out := make(chan interface{}) 67 | wg := &sync.WaitGroup{} 68 | wg.Add(len(ins)) 69 | for _, in := range ins { 70 | go func(in chan interface{}) { 71 | for n := range in { 72 | out <- n 73 | } 74 | wg.Done() 75 | }(in) 76 | } 77 | go func() { 78 | wg.Wait() 79 | close(out) 80 | }() 81 | return out 82 | } 83 | -------------------------------------------------------------------------------- /definitions_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/smartystreets/goconvey/convey" 7 | ) 8 | 9 | func TestPipeline(t *testing.T) { 10 | Convey("Make sure combining and applying operators works as expected", t, func() { 11 | input0 := make(chan interface{}) 12 | input1 := make(chan interface{}) 13 | // for every odd, multiply times two, and add the results 14 | oddFlow := NewFlow(ifOdd, multiplier(2), summer) 15 | out0 := oddFlow.Run(input0) 16 | 17 | evenFlow := NewFlow(ifEven, multiplier(3), summer) 18 | out1 := evenFlow.Run(input1) 19 | 20 | out := summer.Run(Combine(out0, out1)) 21 | 22 | go func() { 23 | for i := 0; i < 10; i++ { 24 | input0 <- i 25 | input1 <- i 26 | } 27 | close(input0) 28 | close(input1) 29 | }() 30 | total := <-out 31 | // 1 + 3 + 5 + 7 + 9 = 25... * 2 = 50 32 | // 2 + 4 + 6 + 8 = 20 * 3 = 60 33 | So(total, ShouldEqual, 110) 34 | }) 35 | Convey("Make sure we didn't break math", t, func() { 36 | input0 := make(chan interface{}) 37 | out := summer.Run(input0) 38 | input0 <- 1 39 | input0 <- 1 40 | close(input0) 41 | total := <-out 42 | So(total, ShouldEqual, 2) 43 | }) 44 | Convey("Make sure splitting works", t, func() { 45 | input := make(chan interface{}) 46 | go func() { 47 | for i := 0; i < 10; i++ { 48 | input <- i 49 | } 50 | close(input) 51 | }() 52 | 53 | out := Combine(Split(Combine(Split(input, 2)...), 2)...) 54 | total := 0 55 | for n := range out { 56 | total += n.(int) 57 | } 58 | So(total, ShouldEqual, 45*4) 59 | }) 60 | } 61 | 62 | func multiplier(x int) Operator { 63 | return Operator(func(in <-chan interface{}, out chan interface{}) { 64 | for m := range in { 65 | n := m.(int) 66 | out <- (int(n) * x) 67 | } 68 | }) 69 | } 70 | 71 | var ifOdd = Operator(func(in <-chan interface{}, out chan interface{}) { 72 | for m := range in { 73 | n := m.(int) 74 | if n%2 == 1 { 75 | out <- n 76 | } 77 | } 78 | }) 79 | 80 | var ifEven = Operator(func(in <-chan interface{}, out chan interface{}) { 81 | for m := range in { 82 | n := m.(int) 83 | if n%2 == 0 { 84 | out <- n 85 | } 86 | } 87 | }) 88 | 89 | var summer = Operator(func(in <-chan interface{}, out chan interface{}) { 90 | total := 0 91 | for m := range in { 92 | n := m.(int) 93 | total += n 94 | } 95 | out <- total 96 | }) 97 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package pipeline is intended to provided some helpful scaffolding for situations 3 | in which there are multiple data flows, or when multiple channels are used to 4 | operate on data. It's a library to facilitate patterns of "channel-middleware". 5 | The Flow example should cover most of the functionality in the package. 6 | */ 7 | package pipeline 8 | -------------------------------------------------------------------------------- /examples_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | 8 | pipeline "github.com/sbogacz/go-pipeline" 9 | 10 | "golang.org/x/time/rate" 11 | ) 12 | 13 | func ExampleRateLimiter() { 14 | // create new rate limiter allowing 10 ops/sec 15 | limiter := rate.NewLimiter(10, 1) 16 | 17 | in := make(chan interface{}, 21) 18 | 19 | // create a RateLimiter operator and run it on input channel 20 | out := pipeline.RateLimiter(limiter).Run(in) 21 | startTime := time.Now() 22 | for i := 0; i < 21; i++ { 23 | in <- i 24 | } 25 | close(in) 26 | 27 | for range out { 28 | } // this is just to flush the output channel 29 | // should have taken about 2 seconds 30 | fmt.Printf("After rate limiting, this took %d", int(time.Now().Sub(startTime).Seconds())) 31 | // Output: After rate limiting, this took 2 32 | } 33 | 34 | func ExampleFlow() { 35 | input0 := make(chan interface{}) 36 | input1 := make(chan interface{}) 37 | 38 | // multiplier takes an int and returns an Operator which multiplies the 39 | // input by the given int 40 | multiplier := func(x int) pipeline.Operator { 41 | return pipeline.Operator(func(in <-chan interface{}, out chan interface{}) { 42 | for m := range in { 43 | n := m.(int) 44 | out <- (int(n) * x) 45 | } 46 | }) 47 | } 48 | 49 | // ifEven is an operator which filters out odds and passes evens through 50 | ifEven := pipeline.Operator(func(in <-chan interface{}, out chan interface{}) { 51 | for m := range in { 52 | n := m.(int) 53 | if n%2 == 0 { 54 | out <- n 55 | } 56 | } 57 | }) 58 | 59 | // ifOdd is an operator which filters out evens and passes odds through 60 | ifOdd := pipeline.Operator(func(in <-chan interface{}, out chan interface{}) { 61 | for m := range in { 62 | n := m.(int) 63 | if n%2 == 1 { 64 | out <- n 65 | } 66 | } 67 | }) 68 | 69 | // summer is an operator which aggregates input integers, and outputs the 70 | // total once the input channel closes 71 | summer := pipeline.Operator(func(in <-chan interface{}, out chan interface{}) { 72 | total := 0 73 | for m := range in { 74 | n := m.(int) 75 | total += n 76 | } 77 | out <- total 78 | }) 79 | 80 | // for every odd, multiply times two, and add the results 81 | oddFlow := pipeline.NewFlow(ifOdd, multiplier(2), summer) 82 | out0 := oddFlow.Run(input0) 83 | 84 | // for every even, multiply times three and add the results 85 | evenFlow := pipeline.NewFlow(ifEven, multiplier(3), summer) 86 | out1 := evenFlow.Run(input1) 87 | 88 | // use the Combine helper to merge the output of out0 and out1 into 89 | // a single output channel out 90 | out := summer.Run(pipeline.Combine(out0, out1)) 91 | 92 | go func() { 93 | for i := 0; i < 10; i++ { 94 | input0 <- i 95 | input1 <- i 96 | } 97 | close(input0) 98 | close(input1) 99 | }() 100 | total := <-out 101 | // 1 + 3 + 5 + 7 + 9 = 25... * 2 = 50 102 | // 2 + 4 + 6 + 8 = 20 * 3 = 60 103 | fmt.Printf("The total is %d\n", total) // Should total 110 104 | // Output: The total is 110 105 | } 106 | 107 | func ExampleFlow_wordCount() { 108 | // Create a new intermediary type to operate on. 109 | // A tuple of the word and the number of times it occurred. 110 | type tuple struct { 111 | token string 112 | count int 113 | } 114 | 115 | // wordCount is an operator that takes in strings (words) and emits a tuple 116 | // of (word, 1) 117 | wordCount := pipeline.Operator(func(in <-chan interface{}, out chan interface{}) { 118 | for word := range in { 119 | out <- tuple{word.(string), 1} 120 | } 121 | }) 122 | 123 | // countAggregator takes in tuples and aggregates their counts. Outputs 124 | // the word and count output as a string. 125 | countAggregator := pipeline.Operator(func(in <-chan interface{}, out chan interface{}) { 126 | counts := make(map[string]int) 127 | for t := range in { 128 | counts[t.(tuple).token] += t.(tuple).count 129 | } 130 | for word, count := range counts { 131 | out <- fmt.Sprintf("%s appears %d times", word, count) 132 | } 133 | }) 134 | 135 | // Launch the word count Flow 136 | input := make(chan interface{}) 137 | wordCountFlow := pipeline.NewFlow(wordCount, countAggregator) 138 | output := wordCountFlow.Run(input) 139 | 140 | // Feed in the input document 141 | document := "the quick fox jumps over the lazy brown dog fox fox" 142 | for _, word := range strings.Split(document, " ") { 143 | input <- word 144 | } 145 | // Signal that we are done submitting input 146 | close(input) 147 | 148 | // Read the output 149 | for result := range output { 150 | fmt.Println(result) 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /ratelimiter.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | 6 | "golang.org/x/time/rate" 7 | ) 8 | 9 | // RateLimiter returns a new Operator that only lets items through at the rate 10 | // of the given `rate.Limiter`. Passing the limiter in allows you to share it 11 | // across multiple instances of this Operator. 12 | func RateLimiter(l *rate.Limiter) Operator { 13 | return func(in <-chan interface{}, out chan interface{}) { 14 | for n := range in { 15 | _ = l.Wait(context.Background()) 16 | out <- n 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /ratelimiter_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "testing" 5 | 6 | "golang.org/x/time/rate" 7 | 8 | . "github.com/smartystreets/goconvey/convey" 9 | ) 10 | 11 | func TestRateLimiter(t *testing.T) { 12 | Convey("RateLimiter works as expected", t, func() { 13 | limiter := rate.NewLimiter(10, 1) 14 | 15 | in := make(chan interface{}, 100) 16 | for i := 0; i < 100; i++ { 17 | in <- i 18 | } 19 | close(in) 20 | 21 | out := RateLimiter(limiter).Run(in) 22 | 23 | j := 0 24 | for n := range out { 25 | So(n, ShouldEqual, j) 26 | j++ 27 | } 28 | So(j, ShouldEqual, 100) 29 | }) 30 | Convey("RateLimiter with other operations", t, func() { 31 | limiter := rate.NewLimiter(100, 1) 32 | 33 | in := make(chan interface{}, 100) 34 | for i := 0; i < 100; i++ { 35 | in <- i 36 | } 37 | close(in) 38 | 39 | c := Split(in, 2) 40 | f1 := NewFlow(RateLimiter(limiter), multiplier(3), RateLimiter(limiter), summer) 41 | f2 := NewFlow(ifOdd, summer) 42 | out := Combine(f1.Run(c[0]), f2.Run(c[1])) 43 | 44 | j := 0 45 | for n := range out { 46 | So(n, ShouldBeGreaterThan, 0) 47 | j++ 48 | } 49 | So(j, ShouldEqual, 2) 50 | }) 51 | } 52 | --------------------------------------------------------------------------------