├── go.sum ├── CODEOWNERS ├── go.mod ├── drain.go ├── .golangci.yml ├── Makefile ├── split.go ├── buffer.go ├── merge_example_test.go ├── sequence.go ├── .github ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md └── workflows │ └── ci.yml ├── emit.go ├── delay.go ├── join.go ├── apply.go ├── pipeline.go ├── SECURITY.md ├── merge.go ├── cancel.go ├── cancel_example_test.go ├── split_test.go ├── LICENSE ├── apply_example_test.go ├── sequence_test.go ├── apply_test.go ├── join_test.go ├── processor.go ├── mocks_test.go ├── collect.go ├── cancel_test.go ├── process.go ├── process_example_test.go ├── semaphore └── semaphore.go ├── process_batch_example_test.go ├── delay_test.go ├── process_batch.go ├── collect_test.go ├── pipeline_example_test.go ├── merge_test.go ├── process_batch_test.go ├── process_test.go └── README.md /go.sum: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * mark.salpeter@deliveryhero.com 2 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/deliveryhero/pipeline/v2 2 | 3 | go 1.18 4 | -------------------------------------------------------------------------------- /drain.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | // Drain empties the input and blocks until the channel is closed 4 | func Drain[Item any](in <-chan Item) { 5 | for range in { 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | disable-all: true 3 | enable: 4 | - gofmt 5 | - revive 6 | - govet 7 | - misspell 8 | - deadcode 9 | - gosec 10 | linters-settings: 11 | misspell: 12 | locale: US 13 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | phony: readme lint 2 | 3 | readme: #generates the README.md file 4 | @goreadme -functions -badge-godoc -badge-github=ci.yaml -badge-goreportcard -credit=false -skip-sub-packages > README.md 5 | 6 | lint: #runs the go linter 7 | @go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.45.2 8 | @golangci-lint run -------------------------------------------------------------------------------- /split.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | // Split takes an interface from Collect and splits it back out into individual elements 4 | func Split[Item any](in <-chan []Item) <-chan Item { 5 | out := make(chan Item) 6 | go func() { 7 | defer close(out) 8 | for is := range in { 9 | for _, i := range is { 10 | out <- i 11 | } 12 | } 13 | }() 14 | return out 15 | } 16 | -------------------------------------------------------------------------------- /buffer.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | // Buffer creates a buffered channel that will close after the input 4 | // is closed and the buffer is fully drained 5 | func Buffer[Item any](size int, in <-chan Item) <-chan Item { 6 | buffer := make(chan Item, size) 7 | go func() { 8 | for i := range in { 9 | buffer <- i 10 | } 11 | close(buffer) 12 | }() 13 | return buffer 14 | } 15 | -------------------------------------------------------------------------------- /merge_example_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/deliveryhero/pipeline/v2" 7 | ) 8 | 9 | func ExampleMerge() { 10 | one := pipeline.Emit(1) 11 | two := pipeline.Emit(2, 2) 12 | three := pipeline.Emit(3, 3, 3) 13 | 14 | for i := range pipeline.Merge(one, two, three) { 15 | fmt.Printf("output: %d\n", i) 16 | } 17 | 18 | fmt.Println("done") 19 | 20 | // Example Output: 21 | // Output:: 1 22 | // Output:: 2 23 | // Output:: 2 24 | // Output:: 3 25 | // Output:: 3 26 | // Output:: 3 27 | // done 28 | } 29 | -------------------------------------------------------------------------------- /sequence.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "context" 4 | 5 | type sequence[A any] []Processor[A, A] 6 | 7 | func (s sequence[A]) Process(ctx context.Context, a A) (A, error) { 8 | var zero A 9 | var in = a 10 | for _, p := range s { 11 | out, err := p.Process(ctx, in) 12 | if err != nil { 13 | p.Cancel(in, err) 14 | return zero, err 15 | } 16 | in = out 17 | } 18 | return in, nil 19 | } 20 | 21 | func (s sequence[A]) Cancel(_ A, _ error) {} 22 | 23 | // Sequence connects many processors sequentially where the inputs are the same outputs 24 | func Sequence[A any](ps ...Processor[A, A]) Processor[A, A] { 25 | return sequence[A](ps) 26 | } 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "⭐️ " 5 | labels: "⭐️ enhancement" 6 | assignees: marksalpeter 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /emit.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "context" 4 | 5 | // Emit fans `is ...Item`` out to a `<-chan Item` 6 | func Emit[Item any](is ...Item) <-chan Item { 7 | out := make(chan Item) 8 | go func() { 9 | defer close(out) 10 | for _, i := range is { 11 | out <- i 12 | } 13 | }() 14 | return out 15 | } 16 | 17 | // Emitter continuously emits new items generated by the next func 18 | // until the context is canceled 19 | func Emitter[Item any](ctx context.Context, next func() Item) <-chan Item { 20 | out := make(chan Item) 21 | go func() { 22 | defer close(out) 23 | for { 24 | select { 25 | case <-ctx.Done(): 26 | return 27 | default: 28 | out <- next() 29 | } 30 | } 31 | }() 32 | return out 33 | } 34 | -------------------------------------------------------------------------------- /delay.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | // Delay delays reading each input by `duration`. 9 | // If the context is canceled, the delay will not be applied. 10 | func Delay[Item any](ctx context.Context, duration time.Duration, in <-chan Item) <-chan Item { 11 | out := make(chan Item) 12 | go func() { 13 | defer close(out) 14 | // Keep reading from in until its closed 15 | for i := range in { 16 | // Take one element from in and pass it to out 17 | out <- i 18 | select { 19 | // Wait duration before reading another input 20 | case <-time.After(duration): 21 | // Don't wait if the context is canceled 22 | case <-ctx.Done(): 23 | } 24 | } 25 | }() 26 | return out 27 | } 28 | -------------------------------------------------------------------------------- /join.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "context" 4 | 5 | type join[A, B, C any] struct { 6 | a Processor[A, B] 7 | b Processor[B, C] 8 | } 9 | 10 | func (j *join[A, B, C]) Process(ctx context.Context, a A) (C, error) { 11 | var zero C 12 | if b, err := j.a.Process(ctx, a); err != nil { 13 | j.a.Cancel(a, err) 14 | return zero, err 15 | } else if c, err := j.b.Process(ctx, b); err != nil { 16 | j.b.Cancel(b, err) 17 | return zero, err 18 | } else { 19 | return c, nil 20 | } 21 | } 22 | 23 | func (j *join[A, B, C]) Cancel(_ A, _ error) {} 24 | 25 | // Join connects two processes where the output of the first is the input of the second 26 | func Join[A, B, C any](a Processor[A, B], b Processor[B, C]) Processor[A, C] { 27 | return &join[A, B, C]{a, b} 28 | } 29 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "\U0001F41E " 5 | labels: "\U0001F41E bug" 6 | assignees: marksalpeter 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Console output** 24 | ```bash 25 | $ go run main.go 26 | errors! 27 | ``` 28 | 29 | **Client info (please complete the following information):** 30 | - Device: [e.g. Docker, macOS, etc] 31 | - OS: [e.g. alpine, busybox, Catalina] 32 | - Version [e.g. 22] 33 | 34 | **Additional context** 35 | Add any other context about the problem here. 36 | -------------------------------------------------------------------------------- /apply.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "context" 4 | 5 | type apply[A, B, C any] struct { 6 | a Processor[A, []B] 7 | b Processor[B, C] 8 | } 9 | 10 | func (j *apply[A, B, C]) Process(ctx context.Context, a A) ([]C, error) { 11 | bs, err := j.a.Process(ctx, a) 12 | if err != nil { 13 | j.a.Cancel(a, err) 14 | return []C{}, err 15 | } 16 | 17 | cs := make([]C, 0, len(bs)) 18 | 19 | for i := range bs { 20 | c, err := j.b.Process(ctx, bs[i]) 21 | if err != nil { 22 | j.b.Cancel(bs[i], err) 23 | return cs, err 24 | } 25 | 26 | cs = append(cs, c) 27 | } 28 | 29 | return cs, nil 30 | } 31 | 32 | func (j *apply[A, B, C]) Cancel(_ A, _ error) {} 33 | 34 | // Apply connects two processes, applying the second to each item of the first output 35 | func Apply[A, B, C any]( 36 | a Processor[A, []B], 37 | b Processor[B, C], 38 | ) Processor[A, []C] { 39 | return &apply[A, B, C]{a, b} 40 | } 41 | -------------------------------------------------------------------------------- /pipeline.go: -------------------------------------------------------------------------------- 1 | // Pipeline is a go library that helps you build pipelines without worrying about channel management and concurrency. 2 | // It contains common fan-in and fan-out operations as well as useful utility funcs for batch processing and scaling. 3 | // 4 | // If you have another common use case you would like to see covered by this package, please (open a feature request) https://github.com/deliveryhero/pipeline/issues. 5 | // 6 | // Cookbook 7 | // 8 | // * (How to run a pipeline until the container is killed) https://github.com/deliveryhero/pipeline#PipelineShutsDownWhenContainerIsKilled 9 | // * (How to shut down a pipeline when there is a error) https://github.com/deliveryhero/pipeline#PipelineShutsDownOnError 10 | // * (How to shut down a pipeline after it has finished processing a batch of data) https://github.com/deliveryhero/pipeline#PipelineShutsDownWhenInputChannelIsClosed 11 | // 12 | package pipeline 13 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Delivery Hero Pipeline Security Issues 2 | 3 | The following are a list of known security issues: 4 | 5 | | Version | Issue | 6 | | ------------ | ------------ | 7 | | 0.2.X | :white_check_mark: | 8 | 9 | 10 | ## Reporting Security Issues 11 | The Delivery Hero security team takes security bugs seriously. We appreciate your efforts to responsibly disclose your findings and will make every effort to acknowledge your contributions. 12 | To report a security issue, please use the [responsible disclosure form on our web page][Link]. 13 | 14 | The Delivery Hero team will send a response indicating the next steps in handling your report. After the initial reply to your report, the team will keep you informed of the progress towards a fix and full announcement and may ask for additional information or guidance. You can follow the public announcements on the repository's security page. 15 | 16 | [Link]: https://www.deliveryhero.com/security/ -------------------------------------------------------------------------------- /merge.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "sync" 4 | 5 | // Merge fans multiple channels in to a single channel 6 | func Merge[Item any](ins ...<-chan Item) <-chan Item { 7 | // Don't merge anything if we don't have to 8 | if l := len(ins); l == 0 { 9 | out := make(chan Item) 10 | close(out) 11 | return out 12 | } else if l == 1 { 13 | return ins[0] 14 | } 15 | out := make(chan Item) 16 | // Create a WaitGroup that waits for all of the ins to close 17 | var wg sync.WaitGroup 18 | wg.Add(len(ins)) 19 | go func() { 20 | // When all of the ins are closed, close the out 21 | wg.Wait() 22 | close(out) 23 | }() 24 | for i := range ins { 25 | go func(in <-chan Item) { 26 | // Wait for each in to close 27 | for i := range in { 28 | // Fan the contents of each in into the out 29 | out <- i 30 | } 31 | // Tell the WaitGroup that one of the channels is closed 32 | wg.Done() 33 | }(ins[i]) 34 | } 35 | return out 36 | } 37 | -------------------------------------------------------------------------------- /cancel.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | // Cancel passes an `Item any` from the `in <-chan Item` directly to the out `<-chan Item` until the `Context` is canceled. 8 | // After the context is canceled, everything from `in <-chan Item` is sent to the `cancel` func instead with the `ctx.Err()`. 9 | func Cancel[Item any](ctx context.Context, cancel func(Item, error), in <-chan Item) <-chan Item { 10 | out := make(chan Item) 11 | go func() { 12 | defer close(out) 13 | for { 14 | select { 15 | // When the context isn't canceld, pass everything to the out chan 16 | // until in is closed 17 | case i, open := <-in: 18 | if !open { 19 | return 20 | } 21 | out <- i 22 | // When the context is canceled, pass all ins to the 23 | // cancel fun until in is closed 24 | case <-ctx.Done(): 25 | for i := range in { 26 | cancel(i, ctx.Err()) 27 | } 28 | return 29 | } 30 | } 31 | }() 32 | return out 33 | } 34 | -------------------------------------------------------------------------------- /cancel_example_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/deliveryhero/pipeline/v2" 9 | ) 10 | 11 | func ExampleCancel() { 12 | // Create a context that lasts for 1 second 13 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) 14 | defer cancel() 15 | 16 | // Create a basic pipeline that emits one int every 250ms 17 | p := pipeline.Delay(ctx, time.Second/4, 18 | pipeline.Emit(1, 2, 3, 4, 5), 19 | ) 20 | 21 | // If the context is canceled, pass the ints to the cancel func for teardown 22 | p = pipeline.Cancel(ctx, func(i int, err error) { 23 | fmt.Printf("%+v could not be processed, %s\n", i, err) 24 | }, p) 25 | 26 | // Otherwise, process the inputs 27 | for out := range p { 28 | fmt.Printf("process: %+v\n", out) 29 | } 30 | 31 | // Output: 32 | // process: 1 33 | // process: 2 34 | // process: 3 35 | // process: 4 36 | // 5 could not be processed, context deadline exceeded 37 | } 38 | -------------------------------------------------------------------------------- /split_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestSplit(t *testing.T) { 9 | t.Parallel() 10 | 11 | type args struct { 12 | in []int 13 | } 14 | type want struct { 15 | out []int 16 | } 17 | tests := []struct { 18 | name string 19 | args args 20 | want want 21 | }{ 22 | { 23 | "splits slices if interfaces into individual interfaces", 24 | args{ 25 | in: []int{1, 2, 3, 4, 5}, 26 | }, 27 | want{ 28 | out: []int{1, 2, 3, 4, 5}, 29 | }, 30 | }, 31 | } 32 | for _, test := range tests { 33 | t.Run(test.name, func(t *testing.T) { 34 | t.Parallel() 35 | 36 | // Create the in channel 37 | in := Emit(test.args.in) 38 | 39 | // Collect the output 40 | var outs []int 41 | for o := range Split(in) { 42 | outs = append(outs, o) 43 | } 44 | 45 | // Expected out 46 | if !reflect.DeepEqual(test.want.out, outs) { 47 | t.Errorf("%+v != %+v", test.want.out, outs) 48 | } 49 | }) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 Delivery Hero 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /apply_example_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/deliveryhero/pipeline/v2" 9 | ) 10 | 11 | func ExampleApply() { 12 | transform := pipeline.NewProcessor(func(_ context.Context, s string) ([]string, error) { 13 | return strings.Split(s, ","), nil 14 | }, nil) 15 | 16 | double := pipeline.NewProcessor(func(_ context.Context, s string) (string, error) { 17 | return s + s, nil 18 | }, nil) 19 | 20 | addLeadingZero := pipeline.NewProcessor(func(_ context.Context, s string) (string, error) { 21 | return "0" + s, nil 22 | }, nil) 23 | 24 | apply := pipeline.Apply( 25 | transform, 26 | pipeline.Sequence( 27 | double, 28 | addLeadingZero, 29 | double, 30 | ), 31 | ) 32 | 33 | input := "1,2,3,4,5" 34 | 35 | for out := range pipeline.Process(context.Background(), apply, pipeline.Emit(input)) { 36 | for j := range out { 37 | fmt.Printf("process: %s\n", out[j]) 38 | } 39 | } 40 | 41 | // Output: 42 | // process: 011011 43 | // process: 022022 44 | // process: 033033 45 | // process: 044044 46 | // process: 055055 47 | } 48 | -------------------------------------------------------------------------------- /sequence_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | ) 8 | 9 | func TestSequence(t *testing.T) { 10 | t.Parallel() 11 | 12 | // Create a step that increments the integer by 1 13 | inc := NewProcessor(func(_ context.Context, i int) (int, error) { 14 | i++ 15 | return i, nil 16 | }, nil) 17 | // Add 5 to every number 18 | inc5 := Sequence(inc, inc, inc, inc, inc) 19 | // Verify the sequence ads 5 to each number 20 | var i int 21 | want := []int{5, 6, 7, 8, 9} 22 | for o := range Process(context.Background(), inc5, Emit(0, 1, 2, 3, 4)) { 23 | if want[i] != o { 24 | t.Fatalf("[%d] = %d, want %d", i, o, want[i]) 25 | } 26 | i++ 27 | } 28 | } 29 | 30 | func SequenceExample() { 31 | // Create a step that increments the integer by 1 32 | inc := NewProcessor(func(_ context.Context, i int) (int, error) { 33 | i++ 34 | return i, nil 35 | }, nil) 36 | // Add 5 to every input 37 | inputs := Emit(0, 1, 2, 3, 4) 38 | addFive := Process(context.Background(), Sequence(inc, inc, inc, inc, inc), inputs) 39 | // Print the output 40 | for o := range addFive { 41 | fmt.Println(o) 42 | } 43 | // Output: 44 | // 5 45 | // 6 46 | // 7 47 | // 8 48 | // 9 49 | } 50 | -------------------------------------------------------------------------------- /apply_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "testing" 7 | ) 8 | 9 | func TestLoopApply(t *testing.T) { 10 | t.Parallel() 11 | 12 | transform := NewProcessor(func(_ context.Context, s string) ([]string, error) { 13 | return strings.Split(s, ","), nil 14 | }, nil) 15 | 16 | double := NewProcessor(func(_ context.Context, s string) (string, error) { 17 | return s + s, nil 18 | }, nil) 19 | 20 | addLeadingZero := NewProcessor(func(_ context.Context, s string) (string, error) { 21 | return "0" + s, nil 22 | }, nil) 23 | 24 | looper := Apply( 25 | transform, 26 | Sequence( 27 | double, 28 | addLeadingZero, 29 | double, 30 | ), 31 | ) 32 | 33 | gotCount := 0 34 | input := "1,2,3,4,5" 35 | want := []string{"011011", "022022", "033033", "044044", "055055"} 36 | 37 | for out := range Process(context.Background(), looper, Emit(input)) { 38 | for j := range out { 39 | gotCount++ 40 | if !contains(want, out[j]) { 41 | t.Errorf("does not contains got=%v, want=%v", out[j], want) 42 | } 43 | } 44 | } 45 | 46 | if gotCount != len(want) { 47 | t.Errorf("total results got=%v, want=%v", gotCount, len(want)) 48 | } 49 | } 50 | 51 | func contains(s []string, e string) bool { 52 | for i := range s { 53 | if s[i] == e { 54 | return true 55 | } 56 | } 57 | 58 | return false 59 | } 60 | -------------------------------------------------------------------------------- /join_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "testing" 8 | ) 9 | 10 | func TestJoin(t *testing.T) { 11 | t.Parallel() 12 | 13 | // Emit 10 numbers 14 | want := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} 15 | ins := Emit(want...) 16 | // Join two steps, one that converts the number to a string, the other that converts it back to a number 17 | join := Join(NewProcessor(func(_ context.Context, i int) (string, error) { 18 | return strconv.Itoa(i), nil 19 | }, nil), NewProcessor(func(_ context.Context, i string) (int, error) { 20 | return strconv.Atoi(i) 21 | }, nil)) 22 | // Compare inputs and outputs 23 | var idx int 24 | for got := range Process(context.Background(), join, ins) { 25 | if want[idx] != got { 26 | t.Fatalf("[%d] = %d, want = %d", idx, got, want[idx]) 27 | } 28 | idx++ 29 | } 30 | } 31 | 32 | func JoinExample() { 33 | // Emit 10 numbers 34 | inputs := Emit(0, 1, 2, 3, 4, 5) 35 | // Join two steps, one that converts the number to a string, the other that converts it back to a number 36 | convertToStringThenBackToInt := Process(context.Background(), Join(NewProcessor(func(_ context.Context, i int) (string, error) { 37 | return strconv.Itoa(i), nil 38 | }, nil), NewProcessor(func(_ context.Context, i string) (int, error) { 39 | return strconv.Atoi(i) 40 | }, nil)), inputs) 41 | // Print the output 42 | for o := range convertToStringThenBackToInt { 43 | fmt.Println(o) 44 | } 45 | // Output: 46 | // 0 47 | // 1 48 | // 2 49 | // 3 50 | // 4 51 | // 5 52 | } 53 | -------------------------------------------------------------------------------- /processor.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "context" 4 | 5 | // Processor represents a blocking operation in a pipeline. Implementing `Processor` will allow you to add 6 | // business logic to your pipelines without directly managing channels. This simplifies your unit tests 7 | // and eliminates channel management related bugs. 8 | type Processor[Input, Output any] interface { 9 | // Process processes an input and returns an output or an error, if the output could not be processed. 10 | // When the context is canceled, process should stop all blocking operations and return the `Context.Err()`. 11 | Process(ctx context.Context, i Input) (Output, error) 12 | 13 | // Cancel is called if process returns an error or if the context is canceled while there are still items in the `in <-chan Input`. 14 | Cancel(i Input, err error) 15 | } 16 | 17 | // NewProcessor creates a process and cancel func 18 | func NewProcessor[Input, Output any]( 19 | process func(ctx context.Context, i Input) (Output, error), 20 | cancel func(i Input, err error), 21 | ) Processor[Input, Output] { 22 | return &processor[Input, Output]{process, cancel} 23 | } 24 | 25 | // processor implements Processor 26 | type processor[Input, Output any] struct { 27 | process func(ctx context.Context, i Input) (Output, error) 28 | cancel func(i Input, err error) 29 | } 30 | 31 | func (p *processor[Input, Output]) Process(ctx context.Context, i Input) (Output, error) { 32 | return p.process(ctx, i) 33 | } 34 | 35 | func (p *processor[Input, Output]) Cancel(i Input, err error) { 36 | p.cancel(i, err) 37 | } 38 | -------------------------------------------------------------------------------- /mocks_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | // mockProcess is a mock of the Processor interface 10 | type mockProcessor[Item any] struct { 11 | processDuration time.Duration 12 | cancelDuration time.Duration 13 | processReturnsErrs bool 14 | processed []Item 15 | canceled []Item 16 | errs []string 17 | } 18 | 19 | // Process waits processDuration before returning its input as its output 20 | func (m *mockProcessor[Item]) Process(ctx context.Context, i Item) (Item, error) { 21 | var zero Item 22 | select { 23 | case <-ctx.Done(): 24 | return zero, ctx.Err() 25 | case <-time.After(m.processDuration): 26 | break 27 | } 28 | if m.processReturnsErrs { 29 | return zero, fmt.Errorf("process error: %v", i) 30 | } 31 | m.processed = append(m.processed, i) 32 | return i, nil 33 | } 34 | 35 | // Cancel collects all inputs that were canceled in m.canceled 36 | func (m *mockProcessor[Item]) Cancel(i Item, err error) { 37 | time.Sleep(m.cancelDuration) 38 | m.canceled = append(m.canceled, i) 39 | m.errs = append(m.errs, err.Error()) 40 | } 41 | 42 | // containsAll returns true if a and b contain all of the same elements 43 | // in any order or if both are empty / nil 44 | func containsAll[Item comparable](a, b []Item) bool { 45 | if len(a) != len(b) { 46 | return false 47 | } else if len(a) == 0 { 48 | return true 49 | } 50 | aMap := make(map[Item]bool) 51 | for _, i := range a { 52 | aMap[i] = true 53 | } 54 | for _, i := range b { 55 | if !aMap[i] { 56 | return false 57 | } 58 | } 59 | return true 60 | } 61 | -------------------------------------------------------------------------------- /collect.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | // Collect collects `[Item any]`s from its in channel and returns `[]Item` from its out channel. 9 | // It will collect up to `maxSize` inputs from the `in <-chan Item` over up to `maxDuration` before returning them as `[]Item`. 10 | // That means when `maxSize` is reached before `maxDuration`, `[maxSize]Item` will be passed to the out channel. 11 | // But if `maxDuration` is reached before `maxSize` inputs are collected, `[< maxSize]Item` will be passed to the out channel. 12 | // When the `context` is canceled, everything in the buffer will be flushed to the out channel. 13 | func Collect[Item any](ctx context.Context, maxSize int, maxDuration time.Duration, in <-chan Item) <-chan []Item { 14 | out := make(chan []Item) 15 | go func() { 16 | for { 17 | is, open := collect[Item](ctx, maxSize, maxDuration, in) 18 | if is != nil { 19 | out <- is 20 | } 21 | if !open { 22 | close(out) 23 | return 24 | } 25 | } 26 | }() 27 | return out 28 | } 29 | 30 | func collect[Item any](ctx context.Context, maxSize int, maxDuration time.Duration, in <-chan Item) ([]Item, bool) { 31 | var buffer []Item 32 | timeout := time.After(maxDuration) 33 | for { 34 | lenBuffer := len(buffer) 35 | select { 36 | case <-ctx.Done(): 37 | // Reduce the timeout to 1/10th of a second 38 | bs, open := collect(context.Background(), maxSize-lenBuffer, 100*time.Millisecond, in) 39 | return append(buffer, bs...), open 40 | case <-timeout: 41 | return buffer, true 42 | case i, open := <-in: 43 | if !open { 44 | return buffer, false 45 | } else if lenBuffer == maxSize-1 { 46 | // There is no room left in the buffer 47 | return append(buffer, i), true 48 | } 49 | // There is still room in the buffer 50 | buffer = append(buffer, i) 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /cancel_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math" 7 | "strconv" 8 | "strings" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func TestCancel(t *testing.T) { 14 | t.Parallel() 15 | 16 | const testDuration = time.Second 17 | 18 | // Collect logs 19 | var logs []string 20 | logf := func(v string, is ...interface{}) { 21 | logs = append(logs, fmt.Sprintf(v, is...)) 22 | } 23 | 24 | // Send a stream of ints through the in chan 25 | in := make(chan int) 26 | go func() { 27 | defer close(in) 28 | i := 0 29 | endAt := time.Now().Add(testDuration) 30 | for now := time.Now(); now.Before(endAt); now = time.Now() { 31 | in <- i 32 | i++ 33 | time.Sleep(testDuration / 100) 34 | } 35 | logf("ended") 36 | }() 37 | 38 | // Create a logger for the cancel func 39 | canceled := func(i int, err error) { 40 | logf("canceled: %d because %s\n", i, err) 41 | } 42 | 43 | // Start canceling the pipeline about half way through the test 44 | ctx, cancel := context.WithTimeout(context.Background(), testDuration/2) 45 | defer cancel() 46 | for i := range Cancel(ctx, canceled, in) { 47 | logf("%d", i) 48 | } 49 | 50 | // There should be some logs 51 | lenLogs := len(logs) 52 | if lenLogs < 2 { 53 | t.Errorf("len(logs) = %d, wanted > 2", lenLogs) 54 | t.Log(logs) 55 | return 56 | } 57 | 58 | // The first half of the logs (+-20%) should be a string representation of the numbers in order 59 | var iCanceled int 60 | for i, log := range logs { 61 | if strconv.Itoa(i) != log { 62 | iCanceled = i 63 | if isAboutHalfWay := math.Abs(float64((lenLogs/2)-i)) <= .2*float64(lenLogs); isAboutHalfWay { 64 | break 65 | } 66 | t.Errorf("got %d, wanted %s", i, log) 67 | for _, l := range logs { 68 | t.Error(l) 69 | } 70 | } 71 | } 72 | 73 | // The remaining logs should be prefixed with "canceled:" 74 | for i, log := range logs[iCanceled : lenLogs-1] { 75 | if !strings.Contains(log, "canceled:") { 76 | t.Errorf("got '%s', wanted 'canceled: %d because context deadline exceeded'", log, i+lenLogs/2) 77 | } 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /process.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/deliveryhero/pipeline/v2/semaphore" 7 | ) 8 | 9 | // Process takes each input from the `in <-chan Input` and calls `Processor.Process` on it. 10 | // When `Processor.Process` returns an `Output`, it will be sent to the output `<-chan Output`. 11 | // If `Processor.Process` returns an error, `Processor.Cancel` will be called with the corresponding input and error message. 12 | // Finally, if the `Context` is canceled, all inputs remaining in the `in <-chan Input` will go directly to `Processor.Cancel`. 13 | func Process[Input, Output any](ctx context.Context, processor Processor[Input, Output], in <-chan Input) <-chan Output { 14 | out := make(chan Output) 15 | go func() { 16 | for i := range in { 17 | process(ctx, processor, i, out) 18 | } 19 | close(out) 20 | }() 21 | return out 22 | } 23 | 24 | // ProcessConcurrently fans the in channel out to multiple Processors running concurrently, 25 | // then it fans the out channels of the Processors back into a single out chan 26 | func ProcessConcurrently[Input, Output any](ctx context.Context, concurrently int, p Processor[Input, Output], in <-chan Input) <-chan Output { 27 | // Create the out chan 28 | out := make(chan Output) 29 | go func() { 30 | // Perform Process concurrently times 31 | sem := semaphore.New(concurrently) 32 | for i := range in { 33 | sem.Add(1) 34 | go func(i Input) { 35 | process(ctx, p, i, out) 36 | sem.Done() 37 | }(i) 38 | } 39 | // Close the out chan after all of the Processors finish executing 40 | sem.Wait() 41 | close(out) 42 | }() 43 | return out 44 | } 45 | 46 | func process[A, B any]( 47 | ctx context.Context, 48 | processor Processor[A, B], 49 | i A, 50 | out chan<- B, 51 | ) { 52 | select { 53 | // When the context is canceled, Cancel all inputs 54 | case <-ctx.Done(): 55 | processor.Cancel(i, ctx.Err()) 56 | // Otherwise, Process all inputs 57 | default: 58 | result, err := processor.Process(ctx, i) 59 | if err != nil { 60 | processor.Cancel(i, err) 61 | return 62 | } 63 | out <- result 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /process_example_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "time" 8 | 9 | "github.com/deliveryhero/pipeline/v2" 10 | ) 11 | 12 | func ExampleProcess() { 13 | // Create a context that times out after 5 seconds 14 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 15 | defer cancel() 16 | 17 | // Create a pipeline that emits 1-6 at a rate of one int per second 18 | p := pipeline.Delay(ctx, time.Second, pipeline.Emit(1, 2, 3, 4, 5, 6)) 19 | 20 | // Multiply each number by 10 21 | p = pipeline.Process(ctx, pipeline.NewProcessor(func(ctx context.Context, in int) (int, error) { 22 | return in * 10, nil 23 | }, func(i int, err error) { 24 | fmt.Printf("error: could not multiply %v, %s\n", i, err) 25 | }), p) 26 | 27 | // Finally, lets print the results and see what happened 28 | for result := range p { 29 | fmt.Printf("result: %d\n", result) 30 | } 31 | 32 | // Example Output: 33 | // result: 10 34 | // result: 20 35 | // result: 30 36 | // result: 40 37 | // result: 50 38 | // error: could not multiply 6, context deadline exceeded 39 | } 40 | 41 | func ExampleProcessConcurrently() { 42 | // Create a context that times out after 5 seconds 43 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 44 | defer cancel() 45 | 46 | // Create a pipeline that emits 1-7 47 | p := pipeline.Emit(1, 2, 3, 4, 5, 6, 7) 48 | 49 | // Add a two second delay to each number 50 | p = pipeline.Delay(ctx, 2*time.Second, p) 51 | 52 | // Add two concurrent processors that pass input numbers to the output 53 | p = pipeline.ProcessConcurrently(ctx, 2, pipeline.NewProcessor(func(ctx context.Context, in int) (int, error) { 54 | return in, nil 55 | }, func(i int, err error) { 56 | fmt.Printf("error: could not process %v, %s\n", i, err) 57 | }), p) 58 | 59 | // Finally, lets print the results and see what happened 60 | for result := range p { 61 | log.Printf("result: %d\n", result) 62 | } 63 | 64 | // Example Output: 65 | // result: 2 66 | // result: 1 67 | // result: 4 68 | // result: 3 69 | // error: could not process 6, process was canceled 70 | // error: could not process 5, process was canceled 71 | // error: could not process 7, context deadline exceeded 72 | } 73 | -------------------------------------------------------------------------------- /semaphore/semaphore.go: -------------------------------------------------------------------------------- 1 | // package semaphore is like a sync.WaitGroup with an upper limit. 2 | // It's useful for limiting concurrent operations. 3 | // 4 | // Example Usage 5 | // 6 | // // startMultiplying is a pipeline step that concurrently multiplies input numbers by a factor 7 | // func startMultiplying(concurrency, factor int, in <-chan int) <-chan int { 8 | // out := make(chan int) 9 | // go func() { 10 | // sem := semaphore.New(concurrency) 11 | // for i := range in { 12 | // // Multiply up to 'concurrency' inputs at once 13 | // sem.Add(1) 14 | // go func() { 15 | // out <- factor * i 16 | // sem.Done() 17 | // }() 18 | // } 19 | // // Wait for all multiplications to finish before closing the output chan 20 | // sem.Wait() 21 | // close(out) 22 | // }() 23 | // return out 24 | // } 25 | // 26 | package semaphore 27 | 28 | // Semaphore is like a sync.WaitGroup, except it has a maximum 29 | // number of items that can be added. If that maximum is reached, 30 | // Add will block until Done is called. 31 | type Semaphore chan struct{} 32 | 33 | // New returns a new Semaphore 34 | func New(max int) Semaphore { 35 | // There are probably more memory efficient ways to implement 36 | // a semaphore using runtime primitives like runtime_SemacquireMutex 37 | return make(Semaphore, max) 38 | } 39 | 40 | // Add adds delta, which may be negative, to the semaphore buffer. 41 | // If the buffer becomes 0, all goroutines blocked by Wait are released. 42 | // If the buffer goes negative, Add will block until another goroutine makes it positive. 43 | // If the buffer exceeds max, Add will block until another goroutine decrements the buffer. 44 | func (s Semaphore) Add(delta int) { 45 | // Increment the semaphore 46 | for i := delta; i > 0; i-- { 47 | s <- struct{}{} 48 | } 49 | // Decrement the semaphore 50 | for i := delta; i < 0; i++ { 51 | <-s 52 | } 53 | } 54 | 55 | // Done decrements the semaphore by 1 56 | func (s Semaphore) Done() { 57 | s.Add(-1) 58 | } 59 | 60 | // Wait blocks until the semaphore is buffer is empty 61 | func (s Semaphore) Wait() { 62 | // Filling the buffered channel ensures that its empty 63 | s.Add(cap(s)) 64 | // Free the buffer before closing (unsure if this matters) 65 | s.Add(-cap(s)) 66 | close(s) 67 | } 68 | -------------------------------------------------------------------------------- /process_batch_example_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/deliveryhero/pipeline/v2" 9 | ) 10 | 11 | func ExampleProcessBatch() { 12 | // Create a context that times out after 5 seconds 13 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 14 | defer cancel() 15 | 16 | // Create a pipeline that emits 1-6 at a rate of one int per second 17 | p := pipeline.Delay(ctx, time.Second, pipeline.Emit(1, 2, 3, 4, 5, 6)) 18 | 19 | // Multiply every 2 adjacent numbers together 20 | p = pipeline.ProcessBatch(ctx, 2, time.Minute, pipeline.NewProcessor(func(ctx context.Context, is []int) ([]int, error) { 21 | o := 1 22 | for _, i := range is { 23 | o *= i 24 | } 25 | return []int{o}, nil 26 | }, func(is []int, err error) { 27 | fmt.Printf("error: could not multiply %v, %s\n", is, err) 28 | }), p) 29 | 30 | // Finally, lets print the results and see what happened 31 | for result := range p { 32 | fmt.Printf("result: %d\n", result) 33 | } 34 | 35 | // Output: 36 | // result: 2 37 | // result: 12 38 | // error: could not multiply [5 6], context deadline exceeded 39 | } 40 | 41 | func ExampleProcessBatchConcurrently() { 42 | // Create a context that times out after 5 seconds 43 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 44 | defer cancel() 45 | 46 | // Create a pipeline that emits 1-9 47 | p := pipeline.Emit(1, 2, 3, 4, 5, 6, 7, 8, 9) 48 | 49 | // Add a 1 second delay to each number 50 | p = pipeline.Delay(ctx, time.Second, p) 51 | 52 | // Group two inputs at a time 53 | p = pipeline.ProcessBatchConcurrently(ctx, 2, 2, time.Minute, pipeline.NewProcessor(func(ctx context.Context, ins []int) ([]int, error) { 54 | return ins, nil 55 | }, func(i []int, err error) { 56 | fmt.Printf("error: could not process %v, %s\n", i, err) 57 | }), p) 58 | 59 | // Finally, lets print the results and see what happened 60 | for result := range p { 61 | fmt.Printf("result: %d\n", result) 62 | } 63 | 64 | // Example Output 65 | // result: 1 66 | // result: 2 67 | // result: 3 68 | // result: 5 69 | // error: could not process [7 8], context deadline exceeded 70 | // error: could not process [4 6], context deadline exceeded 71 | // error: could not process [9], context deadline exceeded 72 | } 73 | -------------------------------------------------------------------------------- /delay_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestDelay(t *testing.T) { 11 | t.Parallel() 12 | 13 | const maxTestDuration = time.Second 14 | type args struct { 15 | ctxTimeout time.Duration 16 | duration time.Duration 17 | in []int 18 | } 19 | type want struct { 20 | out []int 21 | open bool 22 | } 23 | for _, test := range []struct { 24 | name string 25 | args args 26 | want want 27 | }{{ 28 | name: "out closes after duration when in closes", 29 | args: args{ 30 | ctxTimeout: maxTestDuration, 31 | duration: maxTestDuration - 100*time.Millisecond, 32 | in: []int{1}, 33 | }, 34 | want: want{ 35 | out: []int{1}, 36 | open: false, 37 | }, 38 | }, { 39 | name: "delay is not applied when the context is canceled", 40 | args: args{ 41 | ctxTimeout: 10 * time.Millisecond, 42 | duration: maxTestDuration, 43 | in: []int{1, 2, 3, 4, 5}, 44 | }, 45 | want: want{ 46 | out: []int{1, 2, 3, 4, 5}, 47 | open: false, 48 | }, 49 | }, { 50 | name: "out is delayed by duration", 51 | args: args{ 52 | ctxTimeout: maxTestDuration, 53 | duration: maxTestDuration / 4, 54 | in: []int{1, 2, 3, 4, 5}, 55 | }, 56 | want: want{ 57 | out: []int{1, 2, 3, 4}, 58 | open: true, 59 | }, 60 | }} { 61 | t.Run(test.name, func(t *testing.T) { 62 | t.Parallel() 63 | 64 | // Create in channel 65 | in := Emit(test.args.in...) 66 | 67 | // Create a context with a timeut 68 | ctx, cancel := context.WithTimeout(context.Background(), test.args.ctxTimeout) 69 | defer cancel() 70 | 71 | // Start reading from in 72 | delay := Delay(ctx, test.args.duration, in) 73 | timeout := time.After(maxTestDuration) 74 | var isOpen bool 75 | var outs []int 76 | loop: 77 | for { 78 | select { 79 | case i, open := <-delay: 80 | isOpen = open 81 | if !open { 82 | break loop 83 | } 84 | outs = append(outs, i) 85 | case <-timeout: 86 | break loop 87 | } 88 | } 89 | 90 | // Expecting the out channel to be open or closed 91 | if test.want.open != isOpen { 92 | t.Errorf("%t != %t", test.want.open, isOpen) 93 | } 94 | 95 | // Expecting processed outputs 96 | if !reflect.DeepEqual(test.want.out, outs) { 97 | t.Errorf("%+v != %+v", test.want.out, outs) 98 | } 99 | }) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /process_batch.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/deliveryhero/pipeline/v2/semaphore" 8 | ) 9 | 10 | // ProcessBatch collects up to maxSize elements over maxDuration and processes them together as a slice of `Input`s. 11 | // It passed an []Output to the `Processor.Process` method and expects a []Input back. 12 | // It passes []Input batches of inputs to the `Processor.Cancel` method. 13 | // If the receiver is backed up, ProcessBatch can holds up to 2x maxSize. 14 | func ProcessBatch[Input, Output any]( 15 | ctx context.Context, 16 | maxSize int, 17 | maxDuration time.Duration, 18 | processor Processor[[]Input, []Output], 19 | in <-chan Input, 20 | ) <-chan Output { 21 | out := make(chan Output) 22 | go func() { 23 | for { 24 | if !processOneBatch(ctx, maxSize, maxDuration, processor, in, out) { 25 | break 26 | } 27 | } 28 | close(out) 29 | }() 30 | return out 31 | } 32 | 33 | // ProcessBatchConcurrently fans the in channel out to multiple batch Processors running concurrently, 34 | // then it fans the out channels of the batch Processors back into a single out chan 35 | func ProcessBatchConcurrently[Input, Output any]( 36 | ctx context.Context, 37 | concurrently, 38 | maxSize int, 39 | maxDuration time.Duration, 40 | processor Processor[[]Input, []Output], 41 | in <-chan Input, 42 | ) <-chan Output { 43 | // Create the out chan 44 | out := make(chan Output) 45 | go func() { 46 | // Perform Process concurrently times 47 | sem := semaphore.New(concurrently) 48 | lctx, done := context.WithCancel(context.Background()) 49 | for !isDone(lctx) { 50 | sem.Add(1) 51 | go func() { 52 | if !processOneBatch(ctx, maxSize, maxDuration, processor, in, out) { 53 | done() 54 | } 55 | sem.Done() 56 | }() 57 | } 58 | // Close the out chan after all of the Processors finish executing 59 | sem.Wait() 60 | close(out) 61 | done() // Satisfy go-vet 62 | }() 63 | return out 64 | } 65 | 66 | // isDone returns true if the context is canceled 67 | func isDone(ctx context.Context) bool { 68 | select { 69 | case <-ctx.Done(): 70 | return true 71 | default: 72 | return false 73 | } 74 | } 75 | 76 | // processOneBatch processes one batch of inputs from the in chan. 77 | // It returns true if the in chan is still open. 78 | func processOneBatch[Input, Output any]( 79 | ctx context.Context, 80 | maxSize int, 81 | maxDuration time.Duration, 82 | processor Processor[[]Input, []Output], 83 | in <-chan Input, 84 | out chan<- Output, 85 | ) (open bool) { 86 | // Collect interfaces for batch processing 87 | is, open := collect(ctx, maxSize, maxDuration, in) 88 | if is != nil { 89 | select { 90 | // Cancel all inputs during shutdown 91 | case <-ctx.Done(): 92 | processor.Cancel(is, ctx.Err()) 93 | // Otherwise Process the inputs 94 | default: 95 | results, err := processor.Process(ctx, is) 96 | if err != nil { 97 | processor.Cancel(is, err) 98 | return open 99 | } 100 | // Split the results back into interfaces 101 | for _, result := range results { 102 | out <- result 103 | } 104 | } 105 | } 106 | return open 107 | } 108 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: push 4 | 5 | jobs: 6 | lint: 7 | runs-on: ubuntu-latest 8 | continue-on-error: true 9 | steps: 10 | - name: Checkout Repo 11 | uses: actions/checkout@v2 12 | 13 | - name: Set up Go 14 | uses: actions/setup-go@v2 15 | with: 16 | go-version: 1.18 17 | 18 | - name: Lint 19 | if: always() 20 | uses: golangci/golangci-lint-action@v3 21 | with: 22 | version: v1.45 23 | 24 | test: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - name: Checkout Repo 28 | uses: actions/checkout@v2 29 | 30 | - name: Set up Go 31 | uses: actions/setup-go@v2 32 | with: 33 | go-version: 1.18 34 | 35 | - name: Test 36 | run: go test -coverprofile=coverage.txt -json ./... > test.json 37 | 38 | - name: Annotate tests 39 | if: always() 40 | uses: guyarb/golang-test-annoations@v0.3.0 41 | with: 42 | test-results: test.json 43 | 44 | - name: Report coverage 45 | uses: codecov/codecov-action@v1 46 | with: 47 | file: coverage.txt 48 | fail_ci_if_error: true 49 | 50 | build: 51 | needs: test 52 | runs-on: ubuntu-latest 53 | steps: 54 | - name: Checkout Repo 55 | uses: actions/checkout@v2 56 | 57 | - name: Set up Go 58 | uses: actions/setup-go@v2 59 | with: 60 | go-version: 1.18 61 | 62 | - name: Build 63 | run: go build -v ./... 64 | 65 | # docs: 66 | # needs: build 67 | # runs-on: ubuntu-latest 68 | # steps: 69 | # - name: Check out repository 70 | # uses: actions/checkout@v2 71 | # - name: Update readme according to Go doc 72 | # uses: posener/goreadme 73 | # with: 74 | # email: 'marksalpeter@gmail.com' 75 | # badge-codecov: true 76 | # badge-godoc: true 77 | # badge-github: "ci.yml" 78 | # badge-goreportcard: true 79 | # badge-travisci: false 80 | # commit-message: 'improvement(docs): updated docs' 81 | # types: true 82 | # functions: true 83 | # github-token: '${{ secrets.GITHUB_TOKEN }}' 84 | 85 | version: 86 | # needs: docs 87 | if: github.ref == 'refs/heads/main' # only version on push to master 88 | runs-on: ubuntu-latest 89 | steps: 90 | - name: Checkout Repo 91 | uses: actions/checkout@v2 92 | 93 | - name: Conventional Commits Versioning + Changelog 94 | id: changelog 95 | uses: TriPSs/conventional-changelog-action@v3 96 | with: 97 | github-token: ${{ secrets.github_token }} 98 | skip-version-file: true 99 | output-file: false 100 | skip-commit: true 101 | 102 | - name: Tag + Release 103 | uses: actions/create-release@v1 104 | if: ${{ steps.changelog.outputs.skipped == 'false' }} 105 | env: 106 | GITHUB_TOKEN: ${{ secrets.github_token }} 107 | with: 108 | tag_name: ${{ steps.changelog.outputs.tag }} 109 | release_name: ${{ steps.changelog.outputs.tag }} 110 | body: ${{ steps.changelog.outputs.clean_changelog }} -------------------------------------------------------------------------------- /collect_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | // TestCollect tests the following cases of the Collect func 11 | // 1. Closes when in closes 12 | // 2. Remains Open if in remains open 13 | // 3. Collects max and returns immediately 14 | // 4. Returns everything passed in if less than max after duration 15 | // 5. After duration with nothing in the buffer, nothing is returned, channel remains open 16 | // 6. Flushes the buffer if the context is canceled 17 | func TestCollect(t *testing.T) { 18 | t.Parallel() 19 | 20 | const maxTestDuration = time.Second 21 | type args struct { 22 | maxSize int 23 | maxDuration time.Duration 24 | in []int 25 | inDelay time.Duration 26 | ctxTimeout time.Duration 27 | } 28 | type want struct { 29 | out [][]int 30 | open bool 31 | } 32 | for _, test := range []struct { 33 | name string 34 | args args 35 | want want 36 | }{{ 37 | name: "out closes when in closes", 38 | args: args{ 39 | maxSize: 20, 40 | maxDuration: maxTestDuration, 41 | in: nil, 42 | inDelay: 0, 43 | ctxTimeout: maxTestDuration, 44 | }, 45 | want: want{ 46 | out: nil, 47 | open: false, 48 | }, 49 | }, { 50 | name: "out remains open if in remains open", 51 | args: args{ 52 | maxSize: 2, 53 | maxDuration: maxTestDuration, 54 | in: []int{1, 2, 3}, 55 | inDelay: (maxTestDuration / 2) - (10 * time.Millisecond), 56 | ctxTimeout: maxTestDuration, 57 | }, 58 | want: want{ 59 | out: [][]int{{1, 2}}, 60 | open: true, 61 | }, 62 | }, { 63 | name: "collects maxSize inputs and returns", 64 | args: args{ 65 | maxSize: 2, 66 | maxDuration: maxTestDuration / 10 * 9, 67 | inDelay: maxTestDuration / 10, 68 | in: []int{1, 2, 3, 4, 5}, 69 | ctxTimeout: maxTestDuration / 10 * 9, 70 | }, 71 | want: want{ 72 | out: [][]int{ 73 | {1, 2}, 74 | {3, 4}, 75 | {5}, 76 | }, 77 | open: false, 78 | }, 79 | }, { 80 | name: "collection returns after maxDuration with < maxSize", 81 | args: args{ 82 | maxSize: 10, 83 | maxDuration: maxTestDuration / 4, 84 | inDelay: (maxTestDuration / 4) - (25 * time.Millisecond), 85 | in: []int{1, 2, 3, 4, 5}, 86 | ctxTimeout: maxTestDuration / 4, 87 | }, 88 | want: want{ 89 | out: [][]int{ 90 | {1}, 91 | {2}, 92 | {3}, 93 | {4}, 94 | }, 95 | open: true, 96 | }, 97 | }, { 98 | name: "collection flushes buffer when the context is canceled", 99 | args: args{ 100 | maxSize: 10, 101 | maxDuration: maxTestDuration, 102 | inDelay: 0, 103 | in: []int{1, 2, 3, 4, 5}, 104 | ctxTimeout: 0, 105 | }, 106 | want: want{ 107 | out: [][]int{ 108 | {1, 2, 3, 4, 5}, 109 | }, 110 | open: false, 111 | }, 112 | }} { 113 | t.Run(test.name, func(t *testing.T) { 114 | t.Parallel() 115 | 116 | // Create the in channel 117 | in := make(chan int) 118 | go func() { 119 | defer close(in) 120 | for _, i := range test.args.in { 121 | time.Sleep(test.args.inDelay) 122 | in <- i 123 | } 124 | }() 125 | 126 | // Create the context 127 | ctx, cancel := context.WithTimeout(context.Background(), test.args.ctxTimeout) 128 | defer cancel() 129 | 130 | // Collect responses 131 | collect := Collect(ctx, test.args.maxSize, test.args.maxDuration, in) 132 | timeout := time.After(maxTestDuration) 133 | var outs [][]int 134 | var isOpen bool 135 | loop: 136 | for { 137 | select { 138 | case out, open := <-collect: 139 | if !open { 140 | isOpen = false 141 | break loop 142 | } 143 | isOpen = true 144 | outs = append(outs, out) 145 | case <-timeout: 146 | break loop 147 | } 148 | } 149 | 150 | // Expecting to close or stay open 151 | if test.want.open != isOpen { 152 | t.Errorf("open = %t, want %t", isOpen, test.want.open) 153 | } 154 | 155 | // Expecting outputs 156 | if !reflect.DeepEqual(test.want.out, outs) { 157 | t.Errorf("out = %v, want %v", outs, test.want.out) 158 | } 159 | }) 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /pipeline_example_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | "time" 10 | 11 | "github.com/deliveryhero/pipeline/v2" 12 | ) 13 | 14 | // The following example shows how you can shutdown a pipeline 15 | // gracefully when it receives an error message 16 | func Example_pipelineShutsDownOnError() { 17 | // Create a context that can be canceled 18 | ctx, cancel := context.WithCancel(context.Background()) 19 | defer cancel() 20 | 21 | // Create a pipeline that emits 1-10 22 | p := pipeline.Emit(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) 23 | 24 | // A step that will shutdown the pipeline if the number is greater than 1 25 | p = pipeline.Process(ctx, pipeline.NewProcessor(func(ctx context.Context, i int) (int, error) { 26 | // Shut down the pipeline by canceling the context 27 | if i != 1 { 28 | cancel() 29 | return i, fmt.Errorf("%d caused the shutdown", i) 30 | } 31 | return i, nil 32 | }, func(i int, err error) { 33 | // The cancel func is called when an error is returned by the process func or the context is canceled 34 | fmt.Printf("could not process %d: %s\n", i, err) 35 | }), p) 36 | 37 | // Finally, lets print the results and see what happened 38 | for result := range p { 39 | fmt.Printf("result: %d\n", result) 40 | } 41 | 42 | fmt.Println("exiting the pipeline after all data is processed") 43 | 44 | // Example Output: 45 | // could not process 2: 2 caused the shutdown 46 | // result: 1 47 | // could not process 3: context canceled 48 | // could not process 4: context canceled 49 | // could not process 5: context canceled 50 | // could not process 6: context canceled 51 | // could not process 7: context canceled 52 | // could not process 8: context canceled 53 | // could not process 9: context canceled 54 | // could not process 10: context canceled 55 | // exiting the pipeline after all data is processed 56 | } 57 | 58 | // The following example demonstrates a pipeline 59 | // that naturally finishes its run when the input channel is closed 60 | func Example_pipelineShutsDownWhenInputChannelIsClosed() { 61 | // Create a context that can be canceled 62 | ctx, cancel := context.WithCancel(context.Background()) 63 | defer cancel() 64 | 65 | // Create a pipeline that emits 1-10 and then closes its output channel 66 | p := pipeline.Emit(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) 67 | 68 | // Multiply every number by 2 69 | p = pipeline.Process(ctx, pipeline.NewProcessor(func(ctx context.Context, i int) (int, error) { 70 | return i * 2, nil 71 | }, func(i int, err error) { 72 | fmt.Printf("could not multiply %d: %s\n", i, err) 73 | }), p) 74 | 75 | // Finally, lets print the results and see what happened 76 | for result := range p { 77 | fmt.Printf("result: %d\n", result) 78 | } 79 | 80 | fmt.Println("exiting after the input channel is closed") 81 | 82 | // Example Output: 83 | // result: 2 84 | // result: 4 85 | // result: 6 86 | // result: 8 87 | // result: 10 88 | // result: 12 89 | // result: 14 90 | // result: 16 91 | // result: 18 92 | // result: 20 93 | // exiting after the input channel is closed 94 | } 95 | 96 | // This example demonstrates a pipline 97 | // that runs until the os / container the pipline is running in kills it 98 | func Example_pipelineShutsDownWhenContainerIsKilled() { 99 | // Gracefully shutdown the pipeline when the the system is shutting down 100 | // by canceling the context when os.Kill or os.Interrupt signal is sent 101 | ctx, cancel := signal.NotifyContext(context.Background(), os.Kill, os.Interrupt) 102 | defer cancel() 103 | 104 | // Create a pipeline that keeps emitting numbers sequentially until the context is canceled 105 | var count int 106 | p := pipeline.Emitter(ctx, func() int { 107 | count++ 108 | return count 109 | }) 110 | 111 | // Filter out only even numbers 112 | p = pipeline.Process(ctx, pipeline.NewProcessor(func(ctx context.Context, i int) (int, error) { 113 | if i%2 == 0 { 114 | return i, nil 115 | } 116 | return i, fmt.Errorf("'%d' is an odd number", i) 117 | }, func(i int, err error) { 118 | fmt.Printf("error processing '%v': %s\n", i, err) 119 | }), p) 120 | 121 | // Wait a few nanoseconds an simulate the os.Interrupt signal 122 | go func() { 123 | time.Sleep(time.Millisecond / 10) 124 | fmt.Print("\n--- os kills the app ---\n\n") 125 | syscall.Kill(syscall.Getpid(), syscall.SIGINT) 126 | }() 127 | 128 | // Finally, lets print the results and see what happened 129 | for result := range p { 130 | fmt.Printf("result: %d\n", result) 131 | } 132 | 133 | fmt.Println("exiting after the input channel is closed") 134 | 135 | // Example Output: 136 | // error processing '1': '1' is an odd number 137 | // result: 2 138 | // 139 | // --- os kills the app --- 140 | // 141 | // error processing '3': '3' is an odd number 142 | // error processing '4': context canceled 143 | // exiting after the input channel is closed 144 | } 145 | -------------------------------------------------------------------------------- /merge_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | // task will wait for a specified duration of time before returning a certain number of errors 11 | type task struct { 12 | id string 13 | errorCount int 14 | waitFor time.Duration 15 | } 16 | 17 | // do performs the task 18 | func (t task) do() <-chan error { 19 | out := make(chan error) 20 | go func() { 21 | defer close(out) 22 | time.Sleep(t.waitFor) 23 | for i := 0; i < t.errorCount; i++ { 24 | out <- fmt.Errorf("[task %s] error %d", t.id, i) 25 | } 26 | }() 27 | return out 28 | } 29 | 30 | // TestMerge makes sure that the merged chan 31 | // 1. Closes after all of its child chans close 32 | // 2. Receives all error messages from the error chans 33 | // 3. Stays open if one of its child chans never closes 34 | func TestMerge(t *testing.T) { 35 | t.Parallel() 36 | 37 | maxTestDuration := time.Second 38 | for _, test := range []struct { 39 | description string 40 | finishBefore time.Duration 41 | expectedErrors []error 42 | tasks []task 43 | }{{ 44 | description: "Closes after all of its error chans close", 45 | finishBefore: time.Second, 46 | tasks: []task{{ 47 | id: "a", 48 | waitFor: 250 * time.Millisecond, 49 | }, { 50 | id: "b", 51 | waitFor: 500 * time.Millisecond, 52 | }, { 53 | id: "c", 54 | waitFor: 750 * time.Millisecond, 55 | }}, 56 | }, { 57 | description: "Receives all errors from all of its error chans", 58 | finishBefore: time.Second, 59 | expectedErrors: []error{ 60 | errors.New("[task a] error 0"), 61 | errors.New("[task c] error 0"), 62 | errors.New("[task c] error 1"), 63 | errors.New("[task c] error 2"), 64 | errors.New("[task b] error 0"), 65 | errors.New("[task b] error 1"), 66 | }, 67 | tasks: []task{{ 68 | id: "a", 69 | waitFor: 250 * time.Millisecond, 70 | errorCount: 1, 71 | }, { 72 | id: "b", 73 | waitFor: 750 * time.Millisecond, 74 | errorCount: 2, 75 | }, { 76 | id: "c", 77 | waitFor: 500 * time.Millisecond, 78 | errorCount: 3, 79 | }}, 80 | }, { 81 | description: "Stays open if one of its chans never closes", 82 | expectedErrors: []error{ 83 | errors.New("[task c] error 0"), 84 | errors.New("[task b] error 0"), 85 | errors.New("[task b] error 1"), 86 | }, 87 | tasks: []task{{ 88 | id: "a", 89 | waitFor: 2 * maxTestDuration, 90 | // We shoud expect to 'never' receive this error, because it will emit after the maxTestDuration 91 | errorCount: 1, 92 | }, { 93 | id: "b", 94 | waitFor: 750 * time.Millisecond, 95 | errorCount: 2, 96 | }, { 97 | id: "c", 98 | waitFor: 500 * time.Millisecond, 99 | errorCount: 1, 100 | }}, 101 | }, { 102 | description: "Single channel passes through", 103 | expectedErrors: []error{ 104 | errors.New("[task a] error 0"), 105 | errors.New("[task a] error 1"), 106 | errors.New("[task a] error 2"), 107 | }, 108 | tasks: []task{{ 109 | id: "a", 110 | waitFor: 0, 111 | // We shoud expect to 'never' receive this error, because it will emit after the maxTestDuration 112 | errorCount: 3, 113 | }}, 114 | }, { 115 | description: "Closed channel returned", 116 | expectedErrors: []error{}, 117 | tasks: []task{}, 118 | }} { 119 | t.Run(test.description, func(t *testing.T) { 120 | t.Parallel() 121 | 122 | // Start doing all of the tasks 123 | var errChans []<-chan error 124 | for _, task := range test.tasks { 125 | errChans = append(errChans, task.do()) 126 | } 127 | 128 | // Merge all of their error channels together 129 | var errs []error 130 | merged := Merge[error](errChans...) 131 | 132 | // Create the timeout 133 | timeout := time.After(maxTestDuration) 134 | if test.finishBefore > 0 { 135 | timeout = time.After(test.finishBefore) 136 | } 137 | 138 | loop: 139 | for { 140 | select { 141 | case i, ok := <-merged: 142 | if !ok { 143 | // The chan has closed 144 | break loop 145 | } else if err, ok := i.(error); ok { 146 | errs = append(errs, err) 147 | } else { 148 | t.Errorf("'%+v' is not an error!", i) 149 | return 150 | } 151 | case <-timeout: 152 | if isExpected := test.finishBefore == 0; isExpected { 153 | // We're testing that open channels cause a timeout 154 | break loop 155 | } 156 | t.Error("timed out!") 157 | } 158 | } 159 | 160 | // Check that all of the expected errors match the errors we received 161 | lenErrs, lenExpectedErros := len(errs), len(test.expectedErrors) 162 | for i, expectedError := range test.expectedErrors { 163 | if i >= lenErrs { 164 | t.Errorf("expectedErrors[%d]: '%s' != ", i, expectedError) 165 | } else if err := errs[i]; expectedError.Error() != err.Error() { 166 | t.Errorf("expectedErrors[%d]: '%s' != %s", i, expectedError, err) 167 | } 168 | } 169 | 170 | // Check that we have no additional error messages other than the ones we expected 171 | if hasTooManyErrors := lenErrs > lenExpectedErros; hasTooManyErrors { 172 | for _, err := range errs[lenExpectedErros-1:] { 173 | t.Errorf("'%s' is unexpected!", err) 174 | } 175 | } 176 | }) 177 | } 178 | 179 | } 180 | -------------------------------------------------------------------------------- /process_batch_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestProcessBatch(t *testing.T) { 11 | t.Parallel() 12 | 13 | const maxTestDuration = time.Second 14 | type args struct { 15 | ctxTimeout time.Duration 16 | maxSize int 17 | maxDuration time.Duration 18 | processor *mockProcessor[[]int] 19 | in <-chan int 20 | } 21 | tests := []struct { 22 | name string 23 | args args 24 | wantOpen bool 25 | }{{ 26 | name: "out stays open if in is open", 27 | args: args{ 28 | // Cancel the pipeline context half way through the test 29 | ctxTimeout: maxTestDuration / 2, 30 | maxDuration: maxTestDuration, 31 | // Process 2 elements 33% of the total test duration 32 | maxSize: 2, 33 | processor: &mockProcessor[[]int]{ 34 | processDuration: maxTestDuration / 3, 35 | cancelDuration: maxTestDuration / 3, 36 | }, 37 | // * 10 elements = 165% of the test duration 38 | in: Emit(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), 39 | }, 40 | // Therefore the out chan should still be open when the test times out 41 | wantOpen: true, 42 | }, { 43 | name: "out closes if in is closed", 44 | args: args{ 45 | // Cancel the pipeline context half way through the test 46 | ctxTimeout: maxTestDuration / 2, 47 | maxDuration: maxTestDuration, 48 | // Process 5 elements 33% of the total test duration 49 | maxSize: 5, 50 | processor: &mockProcessor[[]int]{ 51 | processDuration: maxTestDuration / 3, 52 | cancelDuration: maxTestDuration / 3, 53 | }, 54 | // * 10 elements = 66% of the test duration 55 | in: Emit(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), 56 | }, 57 | // Therefore the out channel should be closed when the test ends 58 | wantOpen: false, 59 | }} 60 | for i := range tests { 61 | tt := tests[i] 62 | 63 | t.Run(tt.name, func(t *testing.T) { 64 | t.Parallel() 65 | 66 | ctx, cancel := context.WithTimeout(context.Background(), tt.args.ctxTimeout) 67 | defer cancel() 68 | 69 | // Process the batch with a timeout of maxTestDuration 70 | open := true 71 | outChan := ProcessBatch[int, int](ctx, tt.args.maxSize, tt.args.maxDuration, tt.args.processor, tt.args.in) 72 | timeout := time.After(maxTestDuration) 73 | loop: 74 | for { 75 | select { 76 | case <-timeout: 77 | break loop 78 | case _, ok := <-outChan: 79 | if !ok { 80 | open = false 81 | break loop 82 | } 83 | } 84 | } 85 | // Expecting the channels open state 86 | if open != tt.wantOpen { 87 | t.Errorf("open = %t, wanted %t", open, tt.wantOpen) 88 | } 89 | }) 90 | } 91 | } 92 | 93 | func TestProcessBatchConcurrently(t *testing.T) { 94 | t.Parallel() 95 | 96 | const maxTestDuration = time.Second 97 | type args struct { 98 | ctxTimeout time.Duration 99 | concurrently int 100 | maxSize int 101 | maxDuration time.Duration 102 | processor *mockProcessor[[]int] 103 | in <-chan int 104 | } 105 | tests := []struct { 106 | name string 107 | args args 108 | wantOpen bool 109 | }{{ 110 | name: "out stays open if in is open", 111 | args: args{ 112 | ctxTimeout: maxTestDuration / 2, 113 | maxDuration: maxTestDuration, 114 | // Process 1 element for 33% of the total test duration 115 | maxSize: 1, 116 | // * 2x concurrently 117 | concurrently: 2, 118 | processor: &mockProcessor[[]int]{ 119 | processDuration: maxTestDuration / 3, 120 | cancelDuration: maxTestDuration / 3, 121 | }, 122 | // * 10 elements = 165% of the test duration 123 | in: Emit(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), 124 | }, 125 | // Therefore the out chan should still be open when the test times out 126 | wantOpen: true, 127 | }, { 128 | name: "out closes if in is closed", 129 | args: args{ 130 | ctxTimeout: maxTestDuration / 2, 131 | maxDuration: maxTestDuration, 132 | // Process 1 element for 33% of the total test duration 133 | maxSize: 1, 134 | // * 5x concurrently 135 | concurrently: 5, 136 | processor: &mockProcessor[[]int]{ 137 | processDuration: maxTestDuration / 3, 138 | cancelDuration: maxTestDuration / 3, 139 | }, 140 | // * 10 elements = 66% of the test duration 141 | in: Emit(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), 142 | }, 143 | // Therefore the out channel should be closed by the end of the test 144 | wantOpen: false, 145 | }} 146 | for i := range tests { 147 | tt := tests[i] 148 | 149 | t.Run(tt.name, func(t *testing.T) { 150 | t.Parallel() 151 | 152 | ctx, cancel := context.WithTimeout(context.Background(), tt.args.ctxTimeout) 153 | defer cancel() 154 | 155 | // Process the batch with a timeout of maxTestDuration 156 | open := true 157 | out := ProcessBatchConcurrently[int, int](ctx, tt.args.concurrently, tt.args.maxSize, tt.args.maxDuration, tt.args.processor, tt.args.in) 158 | timeout := time.After(maxTestDuration) 159 | loop: 160 | for { 161 | select { 162 | case <-timeout: 163 | break loop 164 | case _, ok := <-out: 165 | if !ok { 166 | open = false 167 | break loop 168 | } 169 | } 170 | } 171 | // Expecting the channels open state 172 | if open != tt.wantOpen { 173 | t.Errorf("open = %t, wanted %t", open, tt.wantOpen) 174 | } 175 | }) 176 | } 177 | } 178 | 179 | func Test_processBatch(t *testing.T) { 180 | t.Parallel() 181 | 182 | drain := make(chan int, 10000) 183 | const maxTestDuration = time.Second 184 | type args struct { 185 | ctxTimeout time.Duration 186 | maxSize int 187 | maxDuration time.Duration 188 | processor *mockProcessor[[]int] 189 | in <-chan int 190 | out chan<- int 191 | } 192 | type want struct { 193 | open bool 194 | processed [][]int 195 | canceled [][]int 196 | errs []string 197 | } 198 | tests := []struct { 199 | name string 200 | args args 201 | want want 202 | }{{ 203 | name: "returns instantly if in is closed", 204 | args: args{ 205 | ctxTimeout: maxTestDuration, 206 | maxSize: 20, 207 | maxDuration: maxTestDuration, 208 | processor: new(mockProcessor[[]int]), 209 | in: func() <-chan int { 210 | in := make(chan int) 211 | close(in) 212 | return in 213 | }(), 214 | out: drain, 215 | }, 216 | want: want{ 217 | open: false, 218 | }, 219 | }, { 220 | name: "processes slices of inputs", 221 | args: args{ 222 | ctxTimeout: maxTestDuration, 223 | maxSize: 2, 224 | maxDuration: maxTestDuration, 225 | processor: new(mockProcessor[[]int]), 226 | in: Emit(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), 227 | out: drain, 228 | }, 229 | want: want{ 230 | open: false, 231 | processed: [][]int{{ 232 | 1, 2, 233 | }, { 234 | 3, 4, 235 | }, { 236 | 5, 6, 237 | }, { 238 | 7, 8, 239 | }, { 240 | 9, 10, 241 | }}, 242 | }, 243 | }, { 244 | name: "cancels slices of inputs if process returns an error", 245 | args: args{ 246 | ctxTimeout: maxTestDuration / 2, 247 | maxSize: 5, 248 | maxDuration: maxTestDuration, 249 | processor: &mockProcessor[[]int]{ 250 | processReturnsErrs: true, 251 | }, 252 | in: Emit(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), 253 | out: drain, 254 | }, 255 | want: want{ 256 | open: false, 257 | canceled: [][]int{{ 258 | 1, 2, 3, 4, 5, 259 | }, { 260 | 6, 7, 8, 9, 10, 261 | }}, 262 | errs: []string{ 263 | "process error: [1 2 3 4 5]", 264 | "process error: [6 7 8 9 10]", 265 | }, 266 | }, 267 | }, { 268 | name: "cancels slices of inputs when the context is canceled", 269 | args: args{ 270 | ctxTimeout: maxTestDuration / 2, 271 | maxSize: 1, 272 | maxDuration: maxTestDuration, 273 | processor: &mockProcessor[[]int]{ 274 | // this will take longer to complete than the maxTestDuration by a few micro seconds 275 | processDuration: maxTestDuration / 10, // 5 calls to Process > maxTestDuration / 2 276 | cancelDuration: maxTestDuration/10 + 25*time.Millisecond, // 5 calls to Cancel > maxTestDuration / 2 277 | }, 278 | in: Emit(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), 279 | out: drain, 280 | }, 281 | want: want{ 282 | open: true, 283 | processed: [][]int{ 284 | {1}, {2}, {3}, {4}, 285 | }, 286 | canceled: [][]int{ 287 | {5}, {6}, {7}, {8}, 288 | }, 289 | errs: []string{ 290 | "context deadline exceeded", 291 | "context deadline exceeded", 292 | "context deadline exceeded", 293 | "context deadline exceeded", 294 | }, 295 | }, 296 | }} 297 | for i := range tests { 298 | tt := tests[i] 299 | 300 | t.Run(tt.name, func(t *testing.T) { 301 | t.Parallel() 302 | 303 | ctx, cancel := context.WithTimeout(context.Background(), tt.args.ctxTimeout) 304 | defer cancel() 305 | 306 | // Process the batch with a timeout of maxTestDuration 307 | timeout := time.After(maxTestDuration) 308 | open := true 309 | loop: 310 | for { 311 | select { 312 | case <-timeout: 313 | break loop 314 | default: 315 | open = processOneBatch[int, int](ctx, tt.args.maxSize, tt.args.maxDuration, tt.args.processor, tt.args.in, tt.args.out) 316 | if !open { 317 | break loop 318 | } 319 | } 320 | } 321 | 322 | // Processing took longer than expected 323 | if open != tt.want.open { 324 | t.Errorf("open = %t, wanted %t", open, tt.want.open) 325 | } 326 | // Expecting processed inputs 327 | if !reflect.DeepEqual(tt.args.processor.processed, tt.want.processed) { 328 | t.Errorf("processed = %+v, want %+v", tt.args.processor.processed, tt.want.processed) 329 | } 330 | // Expecting canceled inputs 331 | if !reflect.DeepEqual(tt.args.processor.canceled, tt.want.canceled) { 332 | t.Errorf("canceled = %+v, want %+v", tt.args.processor.canceled, tt.want.canceled) 333 | } 334 | // Expecting canceled errors 335 | if !reflect.DeepEqual(tt.args.processor.errs, tt.want.errs) { 336 | t.Errorf("errs = %+v, want %+v", tt.args.processor.errs, tt.want.errs) 337 | } 338 | }) 339 | } 340 | } 341 | -------------------------------------------------------------------------------- /process_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestProcess(t *testing.T) { 11 | t.Parallel() 12 | 13 | const maxTestDuration = time.Second 14 | type args struct { 15 | ctxTimeout time.Duration 16 | processDuration time.Duration 17 | processReturnsErrors bool 18 | cancelDuration time.Duration 19 | in []int 20 | } 21 | type want struct { 22 | open bool 23 | out []int 24 | canceled []int 25 | canceledErrs []string 26 | } 27 | tests := []struct { 28 | name string 29 | args args 30 | want want 31 | }{ 32 | { 33 | name: "out closes if in closes but the context isn't canceled", 34 | args: args{ 35 | ctxTimeout: 2 * maxTestDuration, 36 | processDuration: 0, 37 | in: []int{1, 2, 3}, 38 | }, 39 | want: want{ 40 | open: false, 41 | out: []int{1, 2, 3}, 42 | canceled: nil, 43 | }, 44 | }, { 45 | name: "cancel is called on elements after the context is canceled", 46 | args: args{ 47 | ctxTimeout: maxTestDuration / 2, 48 | processDuration: maxTestDuration / 11, 49 | in: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, 50 | }, 51 | want: want{ 52 | open: false, 53 | out: []int{1, 2, 3, 4, 5}, 54 | canceled: []int{6, 7, 8, 9, 10}, 55 | canceledErrs: []string{ 56 | "context deadline exceeded", 57 | "context deadline exceeded", 58 | "context deadline exceeded", 59 | "context deadline exceeded", 60 | "context deadline exceeded", 61 | }, 62 | }, 63 | }, { 64 | name: "out stays open as long as in is open", 65 | args: args{ 66 | ctxTimeout: maxTestDuration / 2, 67 | processDuration: (maxTestDuration / 2) - (100 * time.Millisecond), 68 | cancelDuration: (maxTestDuration / 2) - (100 * time.Millisecond), 69 | in: []int{1, 2, 3}, 70 | }, 71 | want: want{ 72 | open: true, 73 | out: []int{1}, 74 | canceled: []int{2}, 75 | canceledErrs: []string{ 76 | "context deadline exceeded", 77 | }, 78 | }, 79 | }, { 80 | name: "when an error is returned during process, it is passed to cancel", 81 | args: args{ 82 | ctxTimeout: maxTestDuration - 100*time.Millisecond, 83 | processDuration: (maxTestDuration - 200*time.Millisecond) / 2, 84 | processReturnsErrors: true, 85 | cancelDuration: 0, 86 | in: []int{1, 2, 3}, 87 | }, 88 | want: want{ 89 | open: false, 90 | out: nil, 91 | canceled: []int{1, 2, 3}, 92 | canceledErrs: []string{ 93 | "process error: 1", 94 | "process error: 2", 95 | "context deadline exceeded", 96 | }, 97 | }, 98 | }, 99 | } 100 | for _, test := range tests { 101 | t.Run(test.name, func(t *testing.T) { 102 | t.Parallel() 103 | 104 | // Create the in channel 105 | in := make(chan int) 106 | go func() { 107 | defer close(in) 108 | for _, i := range test.args.in { 109 | in <- i 110 | } 111 | }() 112 | 113 | // Setup the Processor 114 | ctx, cancel := context.WithTimeout(context.Background(), test.args.ctxTimeout) 115 | defer cancel() 116 | processor := &mockProcessor[int]{ 117 | processDuration: test.args.processDuration, 118 | processReturnsErrs: test.args.processReturnsErrors, 119 | cancelDuration: test.args.cancelDuration, 120 | } 121 | out := Process[int, int](ctx, processor, in) 122 | 123 | // Collect the outputs 124 | timeout := time.After(maxTestDuration) 125 | var outs []int 126 | var isOpen bool 127 | loop: 128 | for { 129 | select { 130 | case o, open := <-out: 131 | if !open { 132 | isOpen = false 133 | break loop 134 | } 135 | isOpen = true 136 | outs = append(outs, o) 137 | case <-timeout: 138 | break loop 139 | } 140 | } 141 | 142 | // Expecting the out channel to be open or closed 143 | if test.want.open != isOpen { 144 | t.Errorf("%t != %t", test.want.open, isOpen) 145 | } 146 | 147 | // Expecting processed outputs 148 | if !reflect.DeepEqual(test.want.out, outs) { 149 | t.Errorf("%+v != %+v", test.want.out, outs) 150 | } 151 | 152 | // Expecting canceled inputs 153 | if !reflect.DeepEqual(test.want.canceled, processor.canceled) { 154 | t.Errorf("%+v != %+v", test.want.canceled, processor.canceled) 155 | } 156 | 157 | // Expecting canceled errors 158 | if !reflect.DeepEqual(test.want.canceledErrs, processor.errs) { 159 | t.Errorf("%+v != %+v", test.want.canceledErrs, processor.errs) 160 | } 161 | }) 162 | } 163 | } 164 | 165 | func TestProcessConcurrently(t *testing.T) { 166 | t.Parallel() 167 | 168 | const maxTestDuration = time.Second 169 | type args struct { 170 | ctxTimeout time.Duration 171 | processDuration time.Duration 172 | processReturnsErrors bool 173 | cancelDuration time.Duration 174 | concurrently int 175 | in []int 176 | } 177 | type want struct { 178 | open bool 179 | out []int 180 | canceled []int 181 | canceledErrs []string 182 | } 183 | tests := []struct { 184 | name string 185 | args args 186 | want want 187 | }{ 188 | { 189 | name: "out closes if in closes but the context isn't canceled", 190 | args: args{ 191 | ctxTimeout: 2 * maxTestDuration, // context never times out 192 | processDuration: maxTestDuration/3 - (100 * time.Millisecond), // 3 processed per processor 193 | concurrently: 2, // * 2 processors = 6 processed, pipe closes 194 | in: []int{1, 2, 3, 4, 5, 6}, 195 | }, 196 | want: want{ 197 | open: false, 198 | out: []int{1, 2, 3, 4, 5, 6}, 199 | canceled: nil, 200 | }, 201 | }, { 202 | name: "cancel is called on elements after the context is canceled", 203 | args: args{ 204 | ctxTimeout: maxTestDuration / 2, // context times out before the test ends 205 | processDuration: (maxTestDuration / 4) - (10 * time.Millisecond), // 2 processed per processor before timeout 206 | concurrently: 3, // * 3 processors = 6 processed, 4 canceled, pipe closes 207 | in: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, 208 | }, 209 | want: want{ 210 | open: false, 211 | out: []int{1, 2, 3, 4, 5, 6}, 212 | canceled: []int{7, 8, 9, 10}, 213 | canceledErrs: []string{ 214 | "context deadline exceeded", 215 | "context deadline exceeded", 216 | "context deadline exceeded", 217 | "context deadline exceeded", 218 | }, 219 | }, 220 | }, { 221 | name: "out stays open as long as in is open", 222 | args: args{ 223 | ctxTimeout: maxTestDuration / 2, // context times out half way through the test 224 | processDuration: (maxTestDuration / 2) - (100 * time.Millisecond), // process fires onces per processor 225 | cancelDuration: (maxTestDuration / 2) - (100 * time.Millisecond), // cancel fires once per process 226 | concurrently: 3, // * 3 proceses = 3 canceled, 3 processed, 1 still in the pipe 227 | in: []int{1, 2, 3, 4, 5, 6, 7}, 228 | }, 229 | want: want{ 230 | open: true, 231 | out: []int{1, 2, 3}, 232 | canceled: []int{4, 5, 6}, 233 | canceledErrs: []string{ 234 | "context deadline exceeded", 235 | "context deadline exceeded", 236 | "context deadline exceeded", 237 | }, 238 | }, 239 | }, { 240 | name: "when an error is returned during process, it is passed to cancel", 241 | args: args{ 242 | ctxTimeout: maxTestDuration - 100*time.Millisecond, 243 | processDuration: (maxTestDuration - 200*time.Millisecond) / 2, 244 | processReturnsErrors: true, 245 | cancelDuration: 0, 246 | concurrently: 1, 247 | in: []int{1, 2, 3}, 248 | }, 249 | want: want{ 250 | open: false, 251 | out: nil, 252 | canceled: []int{1, 2, 3}, 253 | canceledErrs: []string{ 254 | "process error: 1", 255 | "process error: 2", 256 | "context deadline exceeded", 257 | }, 258 | }, 259 | }, 260 | } 261 | for _, test := range tests { 262 | t.Run(test.name, func(t *testing.T) { 263 | t.Parallel() 264 | 265 | // Create the in channel 266 | in := Emit(test.args.in...) 267 | 268 | // Setup the Processor 269 | ctx, cancel := context.WithTimeout(context.Background(), test.args.ctxTimeout) 270 | defer cancel() 271 | processor := &mockProcessor[int]{ 272 | processDuration: test.args.processDuration, 273 | processReturnsErrs: test.args.processReturnsErrors, 274 | cancelDuration: test.args.cancelDuration, 275 | } 276 | out := ProcessConcurrently[int, int](ctx, test.args.concurrently, processor, in) 277 | 278 | var outs []int 279 | var isOpen bool 280 | timeout := time.After(maxTestDuration) 281 | loop: 282 | for { 283 | select { 284 | case i, open := <-out: 285 | isOpen = open 286 | if !open { 287 | break loop 288 | } 289 | outs = append(outs, i) 290 | 291 | case <-timeout: 292 | break loop 293 | } 294 | } 295 | 296 | // Expecting the out channel to be open or closed 297 | if test.want.open != isOpen { 298 | t.Errorf("open = %t, want %t", isOpen, test.want.open) 299 | } 300 | 301 | // Expecting canceled inputs 302 | if !containsAll(test.want.out, outs) { 303 | t.Errorf("out = %+v, want %+v", outs, test.want.out) 304 | } 305 | 306 | // Expecting canceled inputs 307 | if !containsAll(test.want.canceled, processor.canceled) { 308 | t.Errorf("canceled = %+v, want %+v", processor.canceled, test.want.canceled) 309 | } 310 | 311 | // Expecting canceled errors 312 | if !containsAll(test.want.canceledErrs, processor.errs) { 313 | t.Errorf("canceledErrs = %+v, want %+v", processor.errs, test.want.canceledErrs) 314 | } 315 | }) 316 | } 317 | } 318 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pipeline 2 | 3 | [![Build](https://github.com/deliveryhero/pipeline/actions/workflows/ci.yml/badge.svg)](https://github.com/deliveryhero/pipeline/actions/workflows/ci.yml) 4 | [![GoDoc](https://img.shields.io/badge/pkg.go.dev-doc-blue)](http://pkg.go.dev/github.com/deliveryhero/pipeline/v2) 5 | [![Go Report Card](https://goreportcard.com/badge/deliveryhero/pipeline)](https://goreportcard.com/report/deliveryhero/pipeline) 6 | 7 | Pipeline is a go library that helps you build pipelines without worrying about channel management and concurrency. 8 | It contains common fan-in and fan-out operations as well as useful utility funcs for batch processing and scaling. 9 | 10 | If you have another common use case you would like to see covered by this package, please [open a feature request](https://github.com/deliveryhero/pipeline/issues). 11 | 12 | ## Cookbook 13 | 14 | * [How to run a pipeline until the container is killed](https://github.com/deliveryhero/pipeline#PipelineShutsDownWhenContainerIsKilled) 15 | * [How to shut down a pipeline when there is a error](https://github.com/deliveryhero/pipeline#PipelineShutsDownOnError) 16 | * [How to shut down a pipeline after it has finished processing a batch of data](https://github.com/deliveryhero/pipeline#PipelineShutsDownWhenInputChannelIsClosed) 17 | 18 | ## Functions 19 | 20 | ### func [Apply](/apply.go#L34) 21 | 22 | `func Apply[A, B, C any](a Processor[A, []B], b Processor[B, C]) Processor[A, []C]` 23 | 24 | Apply connects two processes, applying the second to each item of the first output 25 | 26 | ```golang 27 | transform := pipeline.NewProcessor(func(_ context.Context, s string) ([]string, error) { 28 | return strings.Split(s, ","), nil 29 | }, nil) 30 | 31 | double := pipeline.NewProcessor(func(_ context.Context, s string) (string, error) { 32 | return s + s, nil 33 | }, nil) 34 | 35 | addLeadingZero := pipeline.NewProcessor(func(_ context.Context, s string) (string, error) { 36 | return "0" + s, nil 37 | }, nil) 38 | 39 | apply := pipeline.Apply( 40 | transform, 41 | pipeline.Sequence( 42 | double, 43 | addLeadingZero, 44 | double, 45 | ), 46 | ) 47 | 48 | input := "1,2,3,4,5" 49 | 50 | for out := range pipeline.Process(context.Background(), apply, pipeline.Emit(input)) { 51 | for j := range out { 52 | fmt.Printf("process: %s\n", out[j]) 53 | } 54 | } 55 | ``` 56 | 57 | Output: 58 | 59 | ``` 60 | process: 011011 61 | process: 022022 62 | process: 033033 63 | process: 044044 64 | process: 055055 65 | ``` 66 | 67 | ### func [Buffer](/buffer.go#L5) 68 | 69 | `func Buffer[Item any](size int, in <-chan Item) <-chan Item` 70 | 71 | Buffer creates a buffered channel that will close after the input 72 | is closed and the buffer is fully drained 73 | 74 | ### func [Cancel](/cancel.go#L9) 75 | 76 | `func Cancel[Item any](ctx context.Context, cancel func(Item, error), in <-chan Item) <-chan Item` 77 | 78 | Cancel passes an `Item any` from the `in <-chan Item` directly to the out `<-chan Item` until the `Context` is canceled. 79 | After the context is canceled, everything from `in <-chan Item` is sent to the `cancel` func instead with the `ctx.Err()`. 80 | 81 | ```golang 82 | // Create a context that lasts for 1 second 83 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) 84 | defer cancel() 85 | 86 | // Create a basic pipeline that emits one int every 250ms 87 | p := pipeline.Delay(ctx, time.Second/4, 88 | pipeline.Emit(1, 2, 3, 4, 5), 89 | ) 90 | 91 | // If the context is canceled, pass the ints to the cancel func for teardown 92 | p = pipeline.Cancel(ctx, func(i int, err error) { 93 | fmt.Printf("%+v could not be processed, %s\n", i, err) 94 | }, p) 95 | 96 | // Otherwise, process the inputs 97 | for out := range p { 98 | fmt.Printf("process: %+v\n", out) 99 | } 100 | ``` 101 | 102 | Output: 103 | 104 | ``` 105 | process: 1 106 | process: 2 107 | process: 3 108 | process: 4 109 | 5 could not be processed, context deadline exceeded 110 | ``` 111 | 112 | ### func [Collect](/collect.go#L13) 113 | 114 | `func Collect[Item any](ctx context.Context, maxSize int, maxDuration time.Duration, in <-chan Item) <-chan []Item` 115 | 116 | Collect collects `[Item any]`s from its in channel and returns `[]Item` from its out channel. 117 | It will collect up to `maxSize` inputs from the `in <-chan Item` over up to `maxDuration` before returning them as `[]Item`. 118 | That means when `maxSize` is reached before `maxDuration`, `[maxSize]Item` will be passed to the out channel. 119 | But if `maxDuration` is reached before `maxSize` inputs are collected, `[< maxSize]Item` will be passed to the out channel. 120 | When the `context` is canceled, everything in the buffer will be flushed to the out channel. 121 | 122 | ### func [Delay](/delay.go#L10) 123 | 124 | `func Delay[Item any](ctx context.Context, duration time.Duration, in <-chan Item) <-chan Item` 125 | 126 | Delay delays reading each input by `duration`. 127 | If the context is canceled, the delay will not be applied. 128 | 129 | ### func [Drain](/drain.go#L4) 130 | 131 | `func Drain[Item any](in <-chan Item)` 132 | 133 | Drain empties the input and blocks until the channel is closed 134 | 135 | ### func [Emit](/emit.go#L6) 136 | 137 | `func Emit[Item any](is ...Item) <-chan Item` 138 | 139 | Emit fans `is ...Item`` out to a `<-chan Item` 140 | 141 | ### func [Emitter](/emit.go#L19) 142 | 143 | `func Emitter[Item any](ctx context.Context, next func() Item) <-chan Item` 144 | 145 | Emitter continuously emits new items generated by the next func 146 | until the context is canceled 147 | 148 | ### func [Merge](/merge.go#L6) 149 | 150 | `func Merge[Item any](ins ...<-chan Item) <-chan Item` 151 | 152 | Merge fans multiple channels in to a single channel 153 | 154 | ```golang 155 | one := pipeline.Emit(1) 156 | two := pipeline.Emit(2, 2) 157 | three := pipeline.Emit(3, 3, 3) 158 | 159 | for i := range pipeline.Merge(one, two, three) { 160 | fmt.Printf("output: %d\n", i) 161 | } 162 | 163 | fmt.Println("done") 164 | ``` 165 | 166 | Output: 167 | 168 | ``` 169 | Output:: 1 170 | Output:: 3 171 | Output:: 2 172 | Output:: 2 173 | Output:: 3 174 | Output:: 3 175 | done 176 | ``` 177 | 178 | ### func [Process](/process.go#L13) 179 | 180 | `func Process[Input, Output any](ctx context.Context, processor Processor[Input, Output], in <-chan Input) <-chan Output` 181 | 182 | Process takes each input from the `in <-chan Input` and calls `Processor.Process` on it. 183 | When `Processor.Process` returns an `Output`, it will be sent to the output `<-chan Output`. 184 | If `Processor.Process` returns an error, `Processor.Cancel` will be called with the corresponding input and error message. 185 | Finally, if the `Context` is canceled, all inputs remaining in the `in <-chan Input` will go directly to `Processor.Cancel`. 186 | 187 | ```golang 188 | // Create a context that times out after 5 seconds 189 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 190 | defer cancel() 191 | 192 | // Create a pipeline that emits 1-6 at a rate of one int per second 193 | p := pipeline.Delay(ctx, time.Second, pipeline.Emit(1, 2, 3, 4, 5, 6)) 194 | 195 | // Multiply each number by 10 196 | p = pipeline.Process(ctx, pipeline.NewProcessor(func(ctx context.Context, in int) (int, error) { 197 | return in * 10, nil 198 | }, func(i int, err error) { 199 | fmt.Printf("error: could not multiply %v, %s\n", i, err) 200 | }), p) 201 | 202 | // Finally, lets print the results and see what happened 203 | for result := range p { 204 | fmt.Printf("result: %d\n", result) 205 | } 206 | ``` 207 | 208 | Output: 209 | 210 | ``` 211 | result: 10 212 | result: 20 213 | result: 30 214 | result: 40 215 | result: 50 216 | error: could not multiply 6, context deadline exceeded 217 | ``` 218 | 219 | ### func [ProcessBatch](/process_batch.go#L14) 220 | 221 | `func ProcessBatch[Input, Output any]( 222 | ctx context.Context, 223 | maxSize int, 224 | maxDuration time.Duration, 225 | processor Processor[[]Input, []Output], 226 | in <-chan Input, 227 | ) <-chan Output` 228 | 229 | ProcessBatch collects up to maxSize elements over maxDuration and processes them together as a slice of `Input`s. 230 | It passed an []Output to the `Processor.Process` method and expects a []Input back. 231 | It passes []Input batches of inputs to the `Processor.Cancel` method. 232 | If the receiver is backed up, ProcessBatch can holds up to 2x maxSize. 233 | 234 | ```golang 235 | // Create a context that times out after 5 seconds 236 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 237 | defer cancel() 238 | 239 | // Create a pipeline that emits 1-6 at a rate of one int per second 240 | p := pipeline.Delay(ctx, time.Second, pipeline.Emit(1, 2, 3, 4, 5, 6)) 241 | 242 | // Multiply every 2 adjacent numbers together 243 | p = pipeline.ProcessBatch(ctx, 2, time.Minute, pipeline.NewProcessor(func(ctx context.Context, is []int) ([]int, error) { 244 | o := 1 245 | for _, i := range is { 246 | o *= i 247 | } 248 | return []int{o}, nil 249 | }, func(is []int, err error) { 250 | fmt.Printf("error: could not multiply %v, %s\n", is, err) 251 | }), p) 252 | 253 | // Finally, lets print the results and see what happened 254 | for result := range p { 255 | fmt.Printf("result: %d\n", result) 256 | } 257 | ``` 258 | 259 | Output: 260 | 261 | ``` 262 | result: 2 263 | result: 12 264 | error: could not multiply [5 6], context deadline exceeded 265 | ``` 266 | 267 | ### func [ProcessBatchConcurrently](/process_batch.go#L35) 268 | 269 | `func ProcessBatchConcurrently[Input, Output any]( 270 | ctx context.Context, 271 | concurrently, 272 | maxSize int, 273 | maxDuration time.Duration, 274 | processor Processor[[]Input, []Output], 275 | in <-chan Input, 276 | ) <-chan Output` 277 | 278 | ProcessBatchConcurrently fans the in channel out to multiple batch Processors running concurrently, 279 | then it fans the out channels of the batch Processors back into a single out chan 280 | 281 | ```golang 282 | // Create a context that times out after 5 seconds 283 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 284 | defer cancel() 285 | 286 | // Create a pipeline that emits 1-9 287 | p := pipeline.Emit(1, 2, 3, 4, 5, 6, 7, 8, 9) 288 | 289 | // Add a 1 second delay to each number 290 | p = pipeline.Delay(ctx, time.Second, p) 291 | 292 | // Group two inputs at a time 293 | p = pipeline.ProcessBatchConcurrently(ctx, 2, 2, time.Minute, pipeline.NewProcessor(func(ctx context.Context, ins []int) ([]int, error) { 294 | return ins, nil 295 | }, func(i []int, err error) { 296 | fmt.Printf("error: could not process %v, %s\n", i, err) 297 | }), p) 298 | 299 | // Finally, lets print the results and see what happened 300 | for result := range p { 301 | fmt.Printf("result: %d\n", result) 302 | } 303 | ``` 304 | 305 | Output: 306 | 307 | ``` 308 | result: 1 309 | result: 2 310 | result: 3 311 | result: 5 312 | error: could not process [7 8], context deadline exceeded 313 | error: could not process [4 6], context deadline exceeded 314 | error: could not process [9], context deadline exceeded 315 | ``` 316 | 317 | ### func [ProcessConcurrently](/process.go#L26) 318 | 319 | `func ProcessConcurrently[Input, Output any](ctx context.Context, concurrently int, p Processor[Input, Output], in <-chan Input) <-chan Output` 320 | 321 | ProcessConcurrently fans the in channel out to multiple Processors running concurrently, 322 | then it fans the out channels of the Processors back into a single out chan 323 | 324 | ```golang 325 | // Create a context that times out after 5 seconds 326 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 327 | defer cancel() 328 | 329 | // Create a pipeline that emits 1-7 330 | p := pipeline.Emit(1, 2, 3, 4, 5, 6, 7) 331 | 332 | // Add a two second delay to each number 333 | p = pipeline.Delay(ctx, 2*time.Second, p) 334 | 335 | // Add two concurrent processors that pass input numbers to the output 336 | p = pipeline.ProcessConcurrently(ctx, 2, pipeline.NewProcessor(func(ctx context.Context, in int) (int, error) { 337 | return in, nil 338 | }, func(i int, err error) { 339 | fmt.Printf("error: could not process %v, %s\n", i, err) 340 | }), p) 341 | 342 | // Finally, lets print the results and see what happened 343 | for result := range p { 344 | log.Printf("result: %d\n", result) 345 | } 346 | ``` 347 | 348 | Output: 349 | 350 | ``` 351 | result: 2 352 | result: 1 353 | result: 4 354 | result: 3 355 | error: could not process 6, process was canceled 356 | error: could not process 5, process was canceled 357 | error: could not process 7, context deadline exceeded 358 | ``` 359 | 360 | ### func [Split](/split.go#L4) 361 | 362 | `func Split[Item any](in <-chan []Item) <-chan Item` 363 | 364 | Split takes an interface from Collect and splits it back out into individual elements 365 | 366 | ## Examples 367 | 368 | ### PipelineShutsDownOnError 369 | 370 | The following example shows how you can shutdown a pipeline 371 | gracefully when it receives an error message 372 | 373 | ```golang 374 | // Create a context that can be canceled 375 | ctx, cancel := context.WithCancel(context.Background()) 376 | defer cancel() 377 | 378 | // Create a pipeline that emits 1-10 379 | p := pipeline.Emit(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) 380 | 381 | // A step that will shutdown the pipeline if the number is greater than 1 382 | p = pipeline.Process(ctx, pipeline.NewProcessor(func(ctx context.Context, i int) (int, error) { 383 | // Shut down the pipeline by canceling the context 384 | if i != 1 { 385 | cancel() 386 | return i, fmt.Errorf("%d caused the shutdown", i) 387 | } 388 | return i, nil 389 | }, func(i int, err error) { 390 | // The cancel func is called when an error is returned by the process func or the context is canceled 391 | fmt.Printf("could not process %d: %s\n", i, err) 392 | }), p) 393 | 394 | // Finally, lets print the results and see what happened 395 | for result := range p { 396 | fmt.Printf("result: %d\n", result) 397 | } 398 | 399 | fmt.Println("exiting the pipeline after all data is processed") 400 | ``` 401 | 402 | Output: 403 | 404 | ``` 405 | could not process 2: 2 caused the shutdown 406 | result: 1 407 | could not process 3: context canceled 408 | could not process 4: context canceled 409 | could not process 5: context canceled 410 | could not process 6: context canceled 411 | could not process 7: context canceled 412 | could not process 8: context canceled 413 | could not process 9: context canceled 414 | could not process 10: context canceled 415 | exiting the pipeline after all data is processed 416 | ``` 417 | 418 | ### PipelineShutsDownWhenContainerIsKilled 419 | 420 | This example demonstrates a pipline 421 | that runs until the os / container the pipline is running in kills it 422 | 423 | ```golang 424 | // Gracefully shutdown the pipeline when the the system is shutting down 425 | // by canceling the context when os.Kill or os.Interrupt signal is sent 426 | ctx, cancel := signal.NotifyContext(context.Background(), os.Kill, os.Interrupt) 427 | defer cancel() 428 | 429 | // Create a pipeline that keeps emitting numbers sequentially until the context is canceled 430 | var count int 431 | p := pipeline.Emitter(ctx, func() int { 432 | count++ 433 | return count 434 | }) 435 | 436 | // Filter out only even numbers 437 | p = pipeline.Process(ctx, pipeline.NewProcessor(func(ctx context.Context, i int) (int, error) { 438 | if i%2 == 0 { 439 | return i, nil 440 | } 441 | return i, fmt.Errorf("'%d' is an odd number", i) 442 | }, func(i int, err error) { 443 | fmt.Printf("error processing '%v': %s\n", i, err) 444 | }), p) 445 | 446 | // Wait a few nanoseconds an simulate the os.Interrupt signal 447 | go func() { 448 | time.Sleep(time.Millisecond / 10) 449 | fmt.Print("\n--- os kills the app ---\n\n") 450 | syscall.Kill(syscall.Getpid(), syscall.SIGINT) 451 | }() 452 | 453 | // Finally, lets print the results and see what happened 454 | for result := range p { 455 | fmt.Printf("result: %d\n", result) 456 | } 457 | 458 | fmt.Println("exiting after the input channel is closed") 459 | ``` 460 | 461 | Output: 462 | 463 | ``` 464 | error processing '1': '1' is an odd number 465 | result: 2 466 | 467 | --- os kills the app --- 468 | 469 | error processing '3': '3' is an odd number 470 | error processing '4': context canceled 471 | exiting after the input channel is closed 472 | ``` 473 | 474 | ### PipelineShutsDownWhenInputChannelIsClosed 475 | 476 | The following example demonstrates a pipeline 477 | that naturally finishes its run when the input channel is closed 478 | 479 | ```golang 480 | // Create a context that can be canceled 481 | ctx, cancel := context.WithCancel(context.Background()) 482 | defer cancel() 483 | 484 | // Create a pipeline that emits 1-10 and then closes its output channel 485 | p := pipeline.Emit(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) 486 | 487 | // Multiply every number by 2 488 | p = pipeline.Process(ctx, pipeline.NewProcessor(func(ctx context.Context, i int) (int, error) { 489 | return i * 2, nil 490 | }, func(i int, err error) { 491 | fmt.Printf("could not multiply %d: %s\n", i, err) 492 | }), p) 493 | 494 | // Finally, lets print the results and see what happened 495 | for result := range p { 496 | fmt.Printf("result: %d\n", result) 497 | } 498 | 499 | fmt.Println("exiting after the input channel is closed") 500 | ``` 501 | 502 | Output: 503 | 504 | ``` 505 | result: 2 506 | result: 4 507 | result: 6 508 | result: 8 509 | result: 10 510 | result: 12 511 | result: 14 512 | result: 16 513 | result: 18 514 | result: 20 515 | exiting after the input channel is closed 516 | ``` 517 | --------------------------------------------------------------------------------