├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── example_test.go ├── examples └── example1.go ├── pipeline.go └── pipeline_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - "1.8" 5 | - "1.10.x" 6 | - master 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Nikhil Mungel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![GoDoc](https://godoc.org/github.com/hyfather/pipeline?status.svg)](https://godoc.org/github.com/hyfather/pipeline) 2 | [![Build Status](https://travis-ci.org/hyfather/pipeline.svg?branch=master)](https://travis-ci.org/hyfather/pipeline) 3 | [![cover.run](https://cover.run/go/github.com/hyfather/pipeline.svg?style=flat&tag=golang-1.10)](https://cover.run/go?tag=golang-1.10&repo=github.com%2Fhyfather%2Fpipeline) 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/hyfather/pipeline)](https://goreportcard.com/report/github.com/hyfather/pipeline) 5 | 6 | # pipeline 7 | 8 | This package provides a simplistic implementation of Go pipelines 9 | as outlined in [Go Concurrency Patterns: Pipelines and cancellation.](https://blog.golang.org/pipelines) 10 | 11 | # Docs 12 | GoDoc available [here.](https://godoc.org/github.com/hyfather/pipeline) 13 | 14 | # Example Usage 15 | 16 | ``` 17 | import "github.com/hyfather/pipeline" 18 | 19 | p := pipeline.New() 20 | p.AddStageWithFanOut(myStage, 10) 21 | p.AddStageWithFanOut(anotherStage, 100) 22 | doneChan := p.Run(inChan) 23 | 24 | <- doneChan 25 | ``` 26 | 27 | More comprehensive examples can be found [here.](./examples) 28 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hyfather/pipeline" 6 | ) 7 | 8 | func printStage(inObj interface{}) interface{} { 9 | fmt.Println(inObj) 10 | return inObj 11 | } 12 | 13 | func squareStage(inObj interface{}) interface{} { 14 | if v, ok := inObj.(int); ok { 15 | return v * v 16 | } 17 | return nil 18 | } 19 | 20 | var pipelineChan chan interface{} 21 | 22 | func Example() { 23 | p := pipeline.New() 24 | p.AddStageWithFanOut(squareStage, 1) 25 | p.AddStage(printStage) 26 | 27 | pipelineChan = make(chan interface{}, 10) 28 | pipelineChan <- 2 29 | pipelineChan <- 3 30 | close(pipelineChan) 31 | 32 | <-p.Run(pipelineChan) 33 | // Output: 4 34 | // 9 35 | } 36 | -------------------------------------------------------------------------------- /examples/example1.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hyfather/pipeline" 6 | "time" 7 | ) 8 | 9 | func add1(inObj interface{}) interface{} { 10 | if v, ok := inObj.(int); ok { 11 | return v + 1 12 | } 13 | return nil 14 | } 15 | 16 | func slowSquare(inObj interface{}) interface{} { 17 | time.Sleep(1 * time.Second) 18 | if v, ok := inObj.(int); ok { 19 | return v * v 20 | } 21 | return nil 22 | } 23 | 24 | func slowPrint(inObj interface{}) interface{} { 25 | time.Sleep(1 * time.Second) 26 | if v, ok := inObj.(int); ok { 27 | fmt.Println(v) 28 | return v * v 29 | } 30 | return nil 31 | } 32 | 33 | func main() { 34 | fmt.Println("Starting") 35 | ch := make(chan interface{}, 100) 36 | for i := 0; i < 10; i++ { 37 | ch <- i 38 | } 39 | close(ch) 40 | 41 | p := pipeline.New() 42 | p.AddStageWithFanOut(add1, 1) 43 | p.AddStageWithFanOut(slowSquare, 7) 44 | p.AddStageWithFanOut(slowPrint, 10) 45 | 46 | fmt.Println("Waiting") 47 | <-p.Run(ch) 48 | fmt.Println("Done") 49 | } 50 | -------------------------------------------------------------------------------- /pipeline.go: -------------------------------------------------------------------------------- 1 | // Package pipeline provides a simplistic implementation of pipelines 2 | // as outlined in https://blog.golang.org/pipelines 3 | package pipeline 4 | 5 | import ( 6 | "sync" 7 | ) 8 | 9 | // Pipeline type defines a pipeline to which processing "stages" can 10 | // be added and configured to fan-out. Pipelines are meant to be long 11 | // running as they continuously process data as it comes in. 12 | // 13 | // A pipeline can be simultaneously run multiple times with different 14 | // input channels by invoking the Run() method multiple times. 15 | // A running pipeline shouldn't be copied. 16 | type Pipeline []StageFn 17 | 18 | // StageFn is a lower level function type that chains together multiple 19 | // stages using channels. 20 | type StageFn func(inChan <-chan interface{}) (outChan chan interface{}) 21 | 22 | // ProcessFn are the primary function types defined by users of this 23 | // package and passed in to instantiate a meaningful pipeline. 24 | type ProcessFn func(inObj interface{}) (outObj interface{}) 25 | 26 | // New is a convenience method that creates a new Pipeline 27 | func New() Pipeline { 28 | return Pipeline{} 29 | } 30 | 31 | // AddStage is a convenience method for adding a stage with fanSize = 1. 32 | // See AddStageWithFanOut for more information. 33 | func (p *Pipeline) AddStage(inFunc ProcessFn) { 34 | *p = append(*p, fanningStageFnFactory(inFunc, 1)) 35 | } 36 | 37 | // AddStageWithFanOut adds a parallel fan-out ProcessFn to the pipeline. The 38 | // fanSize number indicates how many instances of this stage will read from the 39 | // previous stage and process the data flowing through simultaneously to take 40 | // advantage of parallel CPU scheduling. 41 | // 42 | // Most pipelines will have multiple stages, and the order in which AddStage() 43 | // and AddStageWithFanOut() is invoked matters -- the first invocation indicates 44 | // the first stage and so forth. 45 | // 46 | // Since discrete goroutines process the inChan for FanOut > 1, the order of 47 | // objects flowing through the FanOut stages can't be guaranteed. 48 | func (p *Pipeline) AddStageWithFanOut(inFunc ProcessFn, fanSize uint64) { 49 | *p = append(*p, fanningStageFnFactory(inFunc, fanSize)) 50 | } 51 | 52 | // AddRawStage simply adds a StageFn type to the pipeline without any further 53 | // processing or parsing. This is meant for extensibility and customizations. 54 | func (p *Pipeline) AddRawStage(inFunc StageFn) { 55 | *p = append(*p, inFunc) 56 | } 57 | 58 | // Run starts the pipeline with all the stages that have been added. Run is not 59 | // a blocking function and will return immediately with a doneChan. Consumers 60 | // can wait on the doneChan for an indication of when the pipeline has completed 61 | // processing. 62 | // 63 | // The pipeline runs until its `inChan` channel is open. Once the `inChan` is closed, 64 | // the pipeline stages will sequentially complete from the first stage to the last. 65 | // Once all stages are complete, the last outChan is drained and the doneChan is closed. 66 | // 67 | // Run() can be invoked multiple times to start multiple instances of a pipeline 68 | // that will typically process different incoming channels. 69 | func (p *Pipeline) Run(inChan <-chan interface{}) (doneChan chan struct{}) { 70 | for _, stage := range *p { 71 | inChan = stage(inChan) 72 | } 73 | 74 | doneChan = make(chan struct{}) 75 | go func() { 76 | defer close(doneChan) 77 | for range inChan { 78 | // pull objects from inChan so that the gc marks them 79 | } 80 | }() 81 | return 82 | } 83 | 84 | // stageFnFactory makes a standard stage function from a given ProcessFn. 85 | // StageFn functions types accept an inChan and return an outChan, allowing 86 | // us to chain multiple functions into a pipeline. 87 | func stageFnFactory(inFunc ProcessFn) (outFunc StageFn) { 88 | return func(inChan <-chan interface{}) (outChan chan interface{}) { 89 | outChan = make(chan interface{}) 90 | go func() { 91 | defer close(outChan) 92 | for inObj := range inChan { 93 | if outObj := inFunc(inObj); outObj != nil { 94 | outChan <- outObj 95 | } 96 | } 97 | }() 98 | return 99 | } 100 | } 101 | 102 | // fanningStageFnFactory makes a stage function that fans into multiple 103 | // goroutines increasing the stage throughput depending on the CPU. 104 | func fanningStageFnFactory(inFunc ProcessFn, fanSize uint64) (outFunc StageFn) { 105 | return func(inChan <-chan interface{}) (outChan chan interface{}) { 106 | var channels []chan interface{} 107 | for i := uint64(0); i < fanSize; i++ { 108 | channels = append(channels, stageFnFactory(inFunc)(inChan)) 109 | } 110 | outChan = MergeChannels(channels) 111 | return 112 | } 113 | } 114 | 115 | // MergeChannels merges an array of channels into a single channel. This utility 116 | // function can also be used independently outside of a pipeline. 117 | func MergeChannels(inChans []chan interface{}) (outChan chan interface{}) { 118 | var wg sync.WaitGroup 119 | wg.Add(len(inChans)) 120 | 121 | outChan = make(chan interface{}) 122 | for _, inChan := range inChans { 123 | go func(ch <-chan interface{}) { 124 | defer wg.Done() 125 | for obj := range ch { 126 | outChan <- obj 127 | } 128 | }(inChan) 129 | } 130 | 131 | go func() { 132 | defer close(outChan) 133 | wg.Wait() 134 | }() 135 | return 136 | } 137 | -------------------------------------------------------------------------------- /pipeline_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hyfather/pipeline" 6 | "sort" 7 | ) 8 | 9 | func ExampleMergeChannels() { 10 | inChan1 := make(chan interface{}, 10) 11 | inChan2 := make(chan interface{}, 10) 12 | 13 | inChan1 <- 1 14 | inChan1 <- 3 15 | inChan1 <- 5 16 | inChan1 <- 7 17 | inChan1 <- 9 18 | close(inChan1) 19 | 20 | inChan2 <- 2 21 | inChan2 <- 4 22 | inChan2 <- 6 23 | inChan2 <- 8 24 | inChan2 <- 10 25 | close(inChan2) 26 | 27 | outChan := pipeline.MergeChannels([]chan interface{}{inChan1, inChan2}) 28 | 29 | var ints []int 30 | for e := range outChan { 31 | ints = append(ints, e.(int)) 32 | } 33 | sort.Ints(ints) 34 | fmt.Println(ints) 35 | 36 | // Output: [1 2 3 4 5 6 7 8 9 10] 37 | } 38 | --------------------------------------------------------------------------------