├── internal └── context.go ├── go.mod ├── wrap_err.go ├── parallel.go ├── wrap_err_test.go ├── retry ├── retry_test.go └── retry.go ├── timeout.go ├── pipego.go ├── examples ├── simple │ └── main.go ├── streaming │ └── main.go └── aggregation │ └── main.go ├── timeout_test.go ├── chan_test.go ├── LICENSE ├── pipego_test.go ├── parallel_test.go ├── slices.go ├── chan.go ├── go.sum ├── slices_test.go └── README.md /internal/context.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "reflect" 5 | "runtime" 6 | ) 7 | 8 | type key int 9 | 10 | var SectionKey = key(0) 11 | var AutomaticSectionKey = key(1) 12 | 13 | func GetFunctionName(i interface{}) string { 14 | return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() 15 | } 16 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/sonalys/pipego 2 | 3 | go 1.23 4 | 5 | require ( 6 | github.com/stretchr/testify v1.8.2 7 | golang.org/x/sync v0.8.0 8 | ) 9 | 10 | require ( 11 | github.com/davecgh/go-spew v1.1.1 // indirect 12 | github.com/kr/text v0.2.0 // indirect 13 | github.com/pmezard/go-difflib v1.0.0 // indirect 14 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 15 | gopkg.in/yaml.v3 v3.0.1 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /wrap_err.go: -------------------------------------------------------------------------------- 1 | package pp 2 | 3 | import "context" 4 | 5 | type ErrorWrapper func(error) error 6 | 7 | // WrapErr encapsulates all given steps errors, if an error is returned, it will be wrapped by ErrorWrapper's error. 8 | func WrapErr(wrapper ErrorWrapper, steps ...Step) (out Steps) { 9 | out = make(Steps, 0, len(steps)) 10 | for _, step := range steps { 11 | out = append(out, func(ctx context.Context) (err error) { 12 | err = step(ctx) 13 | if err != nil { 14 | return wrapper(err) 15 | } 16 | return nil 17 | }) 18 | } 19 | return 20 | } 21 | -------------------------------------------------------------------------------- /parallel.go: -------------------------------------------------------------------------------- 1 | package pp 2 | 3 | import ( 4 | "context" 5 | 6 | "golang.org/x/sync/errgroup" 7 | ) 8 | 9 | // Parallel runs all the given steps in parallel, 10 | // It cancels context for the first non-nil error and returns. 11 | // It runs 'n' go-routines at a time. 12 | func Parallel(n uint16, steps ...Step) Step { 13 | return func(ctx context.Context) (err error) { 14 | if n <= 0 { 15 | n = uint16(len(steps)) 16 | } 17 | 18 | errgrp, ctx := errgroup.WithContext(ctx) 19 | errgrp.SetLimit(int(n)) 20 | 21 | for _, step := range steps { 22 | errgrp.Go(func() error { 23 | return step(ctx) 24 | }) 25 | } 26 | 27 | return errgrp.Wait() 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /wrap_err_test.go: -------------------------------------------------------------------------------- 1 | package pp 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestWrapErr(t *testing.T) { 12 | ctx := context.Background() 13 | type args struct { 14 | wrapper ErrorWrapper 15 | steps Steps 16 | } 17 | tests := []struct { 18 | name string 19 | args args 20 | exp error 21 | }{ 22 | { 23 | name: "simple case", 24 | exp: fmt.Errorf("mock"), 25 | args: args{ 26 | wrapper: func(err error) error { 27 | return fmt.Errorf("mock") 28 | }, 29 | steps: Steps{ 30 | func(ctx context.Context) (err error) { 31 | return fmt.Errorf("test") 32 | }, 33 | }, 34 | }, 35 | }, 36 | } 37 | for _, tt := range tests { 38 | t.Run(tt.name, func(t *testing.T) { 39 | gotOut := WrapErr(tt.args.wrapper, tt.args.steps...)[0](ctx) 40 | require.Equal(t, tt.exp, gotOut) 41 | }) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /retry/retry_test.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func Test_ConstantRetry(t *testing.T) { 11 | cr := constantRetry{time.Second} 12 | for i := 1; i <= 10; i++ { 13 | require.Equal(t, cr.Retry(i), time.Second) 14 | } 15 | } 16 | 17 | func Test_LinearRetry(t *testing.T) { 18 | lr := linearRetry{time.Second} 19 | for i := 1; i <= 10; i++ { 20 | require.Equal(t, lr.Retry(i), time.Duration(i)*time.Second) 21 | } 22 | } 23 | 24 | func Test_ExpRetry(t *testing.T) { 25 | er := expRetry{time.Second, 10 * time.Second, 2} 26 | expSlice := []time.Duration{ 27 | 1 * time.Second, // 0 ^ 2 + 1 = 1s 28 | 2 * time.Second, // 1 ^ 2 + 1 = 2s 29 | 5 * time.Second, // 2 ˆ 2 + 1 = 5s 30 | 10 * time.Second, // 3 ^ 2 + 1 = 10s 31 | 10 * time.Second, // maxDelay 32 | } 33 | for i := range expSlice { 34 | require.Equal(t, expSlice[i], er.Retry(i)) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /timeout.go: -------------------------------------------------------------------------------- 1 | package pp 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | // Timeout limits all children steps to execute in the given duration, 10 | // the timer starts when the first step is run. 11 | // All steps shares the same timeout. 12 | func Timeout(d time.Duration, steps ...Step) (out Steps) { 13 | out = make(Steps, 0, len(steps)) 14 | getTimer := sync.OnceValue(func() *time.Timer { return time.NewTimer(d) }) 15 | for _, step := range steps { 16 | enclosedStep := func(ctx context.Context) (err error) { 17 | // Sets a cancellable context bounded to a unique timer, started when the first step is run. 18 | ctx, cancel := context.WithCancelCause(ctx) 19 | defer cancel(context.DeadlineExceeded) 20 | 21 | resultCh := make(chan error, 1) 22 | defer close(resultCh) 23 | 24 | go func() { 25 | resultCh <- step(ctx) 26 | }() 27 | 28 | select { 29 | case <-ctx.Done(): 30 | return ctx.Err() 31 | case <-getTimer().C: 32 | cancel(context.DeadlineExceeded) 33 | return context.DeadlineExceeded 34 | case err := <-resultCh: 35 | return err 36 | } 37 | } 38 | out = append(out, enclosedStep) 39 | } 40 | return 41 | } 42 | -------------------------------------------------------------------------------- /pipego.go: -------------------------------------------------------------------------------- 1 | package pp 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type ( 8 | // Step is a function signature, 9 | // It is used to padronize function calls, making it possible to create a set of generic behaviors. 10 | // It's created on the pipeline initialization, and runs during the `Run` function. 11 | // A Step is a job, that might never run, or might run until it succeeds. 12 | Step func(ctx context.Context) (err error) 13 | 14 | Steps []Step 15 | ) 16 | 17 | // Run receives a context, and runs all pipeline functions. 18 | // It runs until the first non-nil error or completion. 19 | func Run(ctx context.Context, steps ...Step) error { 20 | return runSteps(ctx, steps...) 21 | } 22 | 23 | func runSteps(ctx context.Context, steps ...Step) error { 24 | var err error 25 | for _, step := range steps { 26 | if err = ctx.Err(); err != nil { 27 | return err 28 | } 29 | if err = step(ctx); err != nil { 30 | return err 31 | } 32 | } 33 | return err 34 | } 35 | 36 | func (s Steps) Group() func(context.Context) error { 37 | return func(ctx context.Context) (err error) { 38 | return runSteps(ctx, s...) 39 | } 40 | } 41 | 42 | func (s Steps) Parallel(n uint16) Step { 43 | return Parallel(n, s...) 44 | } 45 | -------------------------------------------------------------------------------- /examples/simple/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | pp "github.com/sonalys/pipego" 8 | "github.com/sonalys/pipego/retry" 9 | ) 10 | 11 | type api struct{} 12 | 13 | func (a api) fetch(ctx context.Context, id string) (int, error) { 14 | return 4, nil 15 | } 16 | 17 | type pipeline struct { 18 | API interface { 19 | fetch(ctx context.Context, id string) (int, error) 20 | } 21 | 22 | input int 23 | sum int 24 | square int 25 | } 26 | 27 | func (p *pipeline) fetchInput(id string) pp.Step { 28 | return func(ctx context.Context) (err error) { 29 | p.input, err = p.API.fetch(ctx, id) 30 | return 31 | } 32 | } 33 | 34 | func (p *pipeline) sumInput(ctx context.Context) (err error) { 35 | p.sum = p.input + p.input 36 | return 37 | } 38 | 39 | func (p *pipeline) sqrInput(ctx context.Context) (err error) { 40 | p.square = p.input * p.input 41 | return 42 | } 43 | 44 | func main() { 45 | ctx := context.Background() 46 | p := pipeline{ 47 | API: api{}, 48 | } 49 | err := pp.Run(ctx, 50 | retry.Constant(3, time.Second, 51 | p.fetchInput("id"), 52 | ), 53 | pp.Parallel(2, 54 | p.sumInput, 55 | p.sqrInput, 56 | ), 57 | ) 58 | if err != nil { 59 | println(err.Error()) 60 | } 61 | // main.pipeline{API:main.api{}, input:4, sum:8, square:16} 62 | } 63 | -------------------------------------------------------------------------------- /timeout_test.go: -------------------------------------------------------------------------------- 1 | package pp 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestTimeout(t *testing.T) { 12 | ctx := context.Background() 13 | tests := []struct { 14 | name string 15 | run func(t *testing.T) 16 | }{ 17 | { 18 | name: "empty", 19 | run: func(t *testing.T) { 20 | require.NotPanics(t, func() { 21 | resp := Timeout(0) 22 | require.Empty(t, resp) 23 | }) 24 | }, 25 | }, 26 | { 27 | name: "dont timeout", 28 | run: func(t *testing.T) { 29 | a := 0 30 | f := func(_ context.Context) (err error) { 31 | a++ 32 | return 33 | } 34 | steps := Timeout(time.Second, 35 | f, f, f, 36 | ) 37 | err := Run(ctx, steps...) 38 | require.NoError(t, err) 39 | require.Equal(t, 3, a) 40 | }, 41 | }, 42 | { 43 | name: "timeout", 44 | run: func(t *testing.T) { 45 | a := 0 46 | f := func(ctx context.Context) (err error) { 47 | time.Sleep(400 * time.Millisecond) 48 | if ctx.Err() != nil { 49 | return 50 | } 51 | a++ 52 | return 53 | } 54 | steps := Timeout(time.Second, 55 | f, f, f, 56 | ) 57 | err := Run(ctx, steps...) 58 | require.Error(t, err) 59 | require.Equal(t, 2, a) 60 | }, 61 | }, 62 | } 63 | for _, tt := range tests { 64 | t.Run(tt.name, func(t *testing.T) { 65 | tt.run(t) 66 | }) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /chan_test.go: -------------------------------------------------------------------------------- 1 | package pp 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestChanDivide(t *testing.T) { 12 | ctx := context.Background() 13 | // Since we are obliged to work with channel pointers in this context, we need this tricky function to get 14 | // the casting right. 15 | getCh := func() (chan int, *<-chan int) { 16 | ch := make(chan int, 1) 17 | var recv <-chan int = ch 18 | return ch, &recv 19 | } 20 | t.Run("empty", func(t *testing.T) { 21 | ch, recv := getCh() 22 | step := ChanDivide(recv, func(_ context.Context, i int) error { 23 | return fmt.Errorf("failed") 24 | }) 25 | close(ch) 26 | err := step(ctx) 27 | 28 | require.NoError(t, err) 29 | }) 30 | t.Run("context cancelled", func(t *testing.T) { 31 | ch, recv := getCh() 32 | ctx, cancel := context.WithCancel(ctx) 33 | value := 0 34 | step := ChanDivide(recv, func(_ context.Context, _ int) error { 35 | value = 1 36 | return fmt.Errorf("failed") 37 | }) 38 | cancel() 39 | err := step(ctx) 40 | require.NoError(t, err) 41 | require.Equal(t, 0, value) 42 | close(ch) 43 | }) 44 | t.Run("success", func(t *testing.T) { 45 | ch, recv := getCh() 46 | value := 0 47 | step := ChanDivide(recv, func(_ context.Context, i int) error { 48 | value = i 49 | return nil 50 | }) 51 | ch <- 1 52 | close(ch) 53 | err := step(ctx) 54 | require.NoError(t, err) 55 | require.Equal(t, 1, value) 56 | }) 57 | } 58 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2023, Alysson Ribeiro 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products derived from 17 | this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /pipego_test.go: -------------------------------------------------------------------------------- 1 | package pp_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | pp "github.com/sonalys/pipego" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func Test_Run(t *testing.T) { 14 | ctx := context.Background() 15 | t.Run("no steps", func(t *testing.T) { 16 | err := pp.Run(ctx) 17 | require.NoError(t, err) 18 | }) 19 | t.Run("with steps", func(t *testing.T) { 20 | run := false 21 | err := pp.Run(ctx, func(_ context.Context) (err error) { 22 | run = true 23 | return 24 | }) 25 | require.NoError(t, err) 26 | require.True(t, run) 27 | }) 28 | t.Run("with duration", func(t *testing.T) { 29 | run := false 30 | delay := 100 * time.Millisecond 31 | err := pp.Run(ctx, func(_ context.Context) (err error) { 32 | run = true 33 | time.Sleep(delay) 34 | return 35 | }) 36 | require.NoError(t, err) 37 | require.True(t, run) 38 | }) 39 | t.Run("keep step order", func(t *testing.T) { 40 | var i int 41 | err := pp.Run(ctx, 42 | func(_ context.Context) (err error) { 43 | require.Equal(t, 0, i) 44 | i++ 45 | return err 46 | }, 47 | func(_ context.Context) (err error) { 48 | require.Equal(t, 1, i) 49 | i++ 50 | return err 51 | }, 52 | ) 53 | require.NoError(t, err) 54 | require.Equal(t, 2, i) 55 | }) 56 | t.Run("stop on error", func(t *testing.T) { 57 | err := pp.Run(ctx, 58 | func(_ context.Context) (err error) { 59 | return fmt.Errorf("mock") 60 | }, 61 | func(_ context.Context) (err error) { 62 | require.Fail(t, "should not run") 63 | return 64 | }, 65 | ) 66 | require.Equal(t, fmt.Errorf("mock"), err) 67 | }) 68 | } 69 | -------------------------------------------------------------------------------- /parallel_test.go: -------------------------------------------------------------------------------- 1 | package pp_test 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "testing" 7 | 8 | pp "github.com/sonalys/pipego" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func Test_Parallel(t *testing.T) { 13 | ctx := context.Background() 14 | t.Run("empty", func(t *testing.T) { 15 | err := pp.Parallel(5)(ctx) 16 | require.NoError(t, err) 17 | }) 18 | t.Run("is parallel", func(t *testing.T) { 19 | var wg, ready sync.WaitGroup 20 | wg.Add(1) 21 | ready.Add(2) 22 | type state struct { 23 | a, b int 24 | } 25 | var s state 26 | err := pp.Parallel(2, 27 | func(_ context.Context) (err error) { 28 | s.a = 1 29 | ready.Done() 30 | wg.Wait() 31 | return nil 32 | }, 33 | func(_ context.Context) (err error) { 34 | s.b = 2 35 | ready.Done() 36 | wg.Wait() 37 | return nil 38 | }, 39 | )(ctx) 40 | ready.Wait() 41 | require.NoError(t, err) 42 | require.Equal(t, 1, s.a) 43 | require.Equal(t, 2, s.b) 44 | wg.Done() 45 | }) 46 | t.Run("runs at the specified parallelism number", func(t *testing.T) { 47 | var wg, ready sync.WaitGroup 48 | wg.Add(1) 49 | ready.Add(1) 50 | type state struct { 51 | a, b int 52 | } 53 | var s state 54 | go require.NotPanics(t, func() { 55 | err := pp.Parallel(1, 56 | func(_ context.Context) (err error) { 57 | s.a = 1 58 | ready.Done() 59 | wg.Wait() 60 | return nil 61 | }, 62 | func(_ context.Context) (err error) { 63 | s.b = 1 64 | ready.Done() // If you set parallelism = 2 you will see this panics, because weight is 1. 65 | wg.Wait() 66 | return nil 67 | }, 68 | )(ctx) 69 | require.NoError(t, err) 70 | }) 71 | ready.Wait() 72 | require.NotEqual(t, s.b, s.a) 73 | ready.Add(1) 74 | wg.Done() 75 | }) 76 | } 77 | -------------------------------------------------------------------------------- /slices.go: -------------------------------------------------------------------------------- 1 | package pp 2 | 3 | import ( 4 | "slices" 5 | ) 6 | 7 | // DivideSliceInSize receives a slice `s` and divide it into groups with `n` elements each, 8 | // then it uses a step factory to generate steps for each group. 9 | // `n` must be greater than 0 or it will panic. 10 | func DivideSliceInSize[T any](s []T, n int, stepFactory func(T) Step) (steps Steps) { 11 | for chunk := range slices.Chunk(s, n) { 12 | batch := make(Steps, 0, len(chunk)) 13 | for _, v := range chunk { 14 | batch = append(batch, stepFactory(v)) 15 | } 16 | steps = append(steps, batch.Group()) 17 | } 18 | return steps 19 | } 20 | 21 | // divideSliceInGroups receive a slice `s` and breaks it into `n` sub-slices. 22 | func divideSliceInGroups[T any](s []T, n int) [][]T { 23 | length := float64(len(s)) 24 | if length == 0 { 25 | return nil 26 | } 27 | var out [][]T 28 | for segment := 0; segment < n; segment++ { 29 | startIndex := int(float64(segment) / float64(n) * length) 30 | endIndex := int(float64(segment+1) / float64(n) * length) 31 | if startIndex == endIndex { 32 | continue 33 | } 34 | out = append(out, s[startIndex:endIndex]) 35 | } 36 | return out 37 | } 38 | 39 | // DivideSliceInGroups receives a slice `s` and divide it into `n` groups, 40 | // then it uses a step factory to generate steps for each group. 41 | func DivideSliceInGroups[T any](s []T, n int, stepFactory func(T) Step) (steps Steps) { 42 | for _, chunk := range divideSliceInGroups(s, n) { 43 | batch := make(Steps, 0, len(chunk)) 44 | for _, v := range chunk { 45 | batch = append(batch, stepFactory(v)) 46 | } 47 | steps = append(steps, batch.Group()) 48 | } 49 | return steps 50 | } 51 | 52 | // ForEach takes a slice `s` and a stepFactory, and creates a step for each element inside. 53 | func ForEach[T any](s []T, stepFactory func(T, int) Step) Step { 54 | batch := make(Steps, 0, len(s)) 55 | for i := range s { 56 | batch = append(batch, stepFactory(s[i], i)) 57 | } 58 | return batch.Group() 59 | } 60 | -------------------------------------------------------------------------------- /chan.go: -------------------------------------------------------------------------------- 1 | package pp 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | ) 7 | 8 | // ChanWorker defines a function signature to process values returned from a channel. 9 | type ChanWorker[T any] func(context.Context, T) error 10 | 11 | // ChanDivide divides the input of a channel between all the given workers, 12 | // they process load as they are free to do so. 13 | // We only accept *<-chan T because during the initialization of the pipeline the channel 14 | // field will still be unset. 15 | // Please provide an initialized channel pointer, using nil pointers will result in panic. 16 | // ChanDivide must be used inside a `parallel` section, 17 | // unless the channel providing values is in another go-routine. 18 | // ChanDivide and the provided chan in the same go-routine will dead-lock. 19 | func ChanDivide[T any](ch *<-chan T, workers ...ChanWorker[T]) Step { 20 | if ch == nil { 21 | panic("cannot use nil chan pointer") 22 | } 23 | // We define a waitGroup to wait for all worker's routines to end. 24 | var wg sync.WaitGroup 25 | wg.Add(len(workers)) 26 | // We also define an errChan to get the first error to happen and return it. 27 | errChan := make(chan error, len(workers)) 28 | return func(ctx context.Context) (err error) { 29 | ctx, cancel := context.WithCancel(ctx) 30 | defer cancel() 31 | for i := range workers { 32 | // Spawns 1 routine for each worker, making them consume from job channel. 33 | go func(i int) { 34 | defer wg.Done() 35 | for { 36 | select { 37 | // Case for worker waiting for a job. 38 | case v, ok := <-*ch: 39 | // Job channel is closed, all waiting workers should end. 40 | if !ok { 41 | return 42 | } 43 | // Execute job and cancel other jobs in case of error. 44 | if err := workers[i](ctx, v); err != nil { 45 | errChan <- err 46 | cancel() 47 | return 48 | } 49 | // context.Context cancellation, all jobs must end. 50 | case <-ctx.Done(): 51 | return 52 | } 53 | } 54 | }(i) 55 | } 56 | wg.Wait() 57 | close(errChan) 58 | return <-errChan 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /examples/streaming/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math/rand" 7 | "time" 8 | 9 | pp "github.com/sonalys/pipego" 10 | "github.com/sonalys/pipego/retry" 11 | ) 12 | 13 | // API is a generic API implementation. 14 | type API struct{} 15 | 16 | // fetchData implements a generic data fetcher signature. 17 | func (a API) fetchData(ctx context.Context, id string) <-chan int { 18 | ch := make(chan int) 19 | ctx, cancel := context.WithTimeout(ctx, 10*time.Second) 20 | // Note that if this wasn't async, both fetch and chanDivide would need to be insire parallel stage. 21 | go func() { 22 | defer cancel() 23 | defer close(ch) 24 | for ctx.Err() == nil { 25 | ch <- rand.Intn(10) 26 | time.Sleep(time.Second) 27 | } 28 | }() 29 | return ch 30 | } 31 | 32 | type PipelineDependencies struct { 33 | API interface { 34 | fetchData(ctx context.Context, id string) <-chan int 35 | } 36 | } 37 | 38 | type Pipeline struct { 39 | dep PipelineDependencies 40 | // We need to use pointers with ChanDivide func because at initialization, the field is not set yet. 41 | values <-chan int 42 | } 43 | 44 | func newPipeline(dep PipelineDependencies) Pipeline { 45 | return Pipeline{ 46 | dep: dep, 47 | values: make(<-chan int), 48 | } 49 | } 50 | 51 | func (s *Pipeline) fetchValues(id string) pp.Step { 52 | return func(ctx context.Context) (err error) { 53 | s.values = s.dep.API.fetchData(ctx, id) 54 | return 55 | } 56 | } 57 | 58 | func main() { 59 | ctx := context.Background() 60 | api := API{} 61 | pipeline := newPipeline(PipelineDependencies{ 62 | API: api, 63 | }) 64 | err := pp.Run(ctx, 65 | // Setup a simple example of a streaming response. 66 | retry.Constant(retry.Inf, time.Second, 67 | pipeline.fetchValues("objectID"), 68 | ), 69 | pp.ChanDivide(&pipeline.values, 70 | func(ctx context.Context, i int) (err error) { 71 | fmt.Printf("got value %d", i) 72 | return 73 | }, 74 | // Slower worker that will take longer to execute values. 75 | func(ctx context.Context, i int) (err error) { 76 | fmt.Printf("got value %d", i) 77 | time.Sleep(2 * time.Second) 78 | return 79 | }, 80 | ), 81 | ) 82 | if err != nil { 83 | println("could not execute pipeline: ", err.Error()) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /examples/aggregation/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "math/rand" 7 | "time" 8 | 9 | pp "github.com/sonalys/pipego" 10 | "github.com/sonalys/pipego/retry" 11 | ) 12 | 13 | // API is a generic API implementation. 14 | type API struct{} 15 | 16 | // fetchData implements a generic data fetcher signature. 17 | func (a API) fetchData(_ context.Context, id string) ([]int, error) { 18 | // Here we are simply implementing a failure mechanism to test our retriability. 19 | switch n := rand.Intn(10); { 20 | case n < 3: 21 | return []int{1, 2, 3, 4, 5}, nil 22 | default: 23 | return nil, errors.New("unexpected error") 24 | } 25 | } 26 | 27 | type PipelineDependencies struct { 28 | API interface { 29 | fetchData(_ context.Context, id string) ([]int, error) 30 | } 31 | } 32 | 33 | type Pipeline struct { 34 | dep PipelineDependencies 35 | 36 | values []int 37 | 38 | Sum int 39 | AVG int 40 | Count int 41 | } 42 | 43 | func newPipeline(dep PipelineDependencies) Pipeline { 44 | return Pipeline{dep: dep} 45 | } 46 | 47 | func (s *Pipeline) fetchValues(id string) pp.Step { 48 | return func(ctx context.Context) (err error) { 49 | s.values, err = s.dep.API.fetchData(ctx, id) 50 | return 51 | } 52 | } 53 | 54 | func (s *Pipeline) calcSum(_ context.Context) (err error) { 55 | for _, v := range s.values { 56 | s.Sum += v 57 | } 58 | return 59 | } 60 | 61 | func (s *Pipeline) calcCount(_ context.Context) (err error) { 62 | s.Count = len(s.values) 63 | return 64 | } 65 | 66 | func (s *Pipeline) calcAverage(_ context.Context) (err error) { 67 | // simple example of aggregation error. 68 | if s.Count == 0 { 69 | return errors.New("cannot calculate average for empty slice") 70 | } 71 | s.AVG = s.Sum / s.Count 72 | return 73 | } 74 | 75 | func main() { 76 | ctx := context.Background() 77 | api := API{} 78 | pipeline := newPipeline(PipelineDependencies{ 79 | API: api, 80 | }) 81 | err := pp.Run(ctx, 82 | retry.Constant(retry.Inf, time.Second, 83 | pipeline.fetchValues("objectID"), 84 | ), 85 | pp.Parallel(2, 86 | pipeline.calcSum, 87 | pipeline.calcCount, 88 | ), 89 | pipeline.calcAverage, 90 | ) 91 | if err != nil { 92 | println("could not execute pipeline: ", err.Error()) 93 | } 94 | // {dep:{API:{}} values:[1 2 3 4 5] Sum:15 AVG:3 Count:5} 95 | } 96 | -------------------------------------------------------------------------------- /retry/retry.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "context" 5 | "math" 6 | "time" 7 | 8 | pp "github.com/sonalys/pipego" 9 | ) 10 | 11 | const Inf = -1 12 | 13 | type Retrier interface { 14 | Retry(retryNumber int) time.Duration 15 | } 16 | 17 | type constantRetry struct { 18 | time.Duration 19 | } 20 | 21 | func (r constantRetry) Retry(n int) time.Duration { 22 | return r.Duration 23 | } 24 | 25 | // Constant is a constant retry implementation. 26 | // It always return the same delay. 27 | // Example: 2s: 2s, 2s, 2s, 2s... 28 | func Constant(n int, delay time.Duration, steps ...pp.Step) pp.Step { 29 | return newRetry(n, constantRetry{delay}, steps...) 30 | } 31 | 32 | type linearRetry struct { 33 | time.Duration 34 | } 35 | 36 | func (r linearRetry) Retry(n int) time.Duration { 37 | return r.Duration * time.Duration(n) 38 | } 39 | 40 | // Linear is a linear retry implementation. 41 | // It returns a linear series for the delay calculation. 42 | // Example: 1s: 1s, 2s, 3s, 4s, ... 43 | func Linear(n int, delay time.Duration, steps ...pp.Step) pp.Step { 44 | return newRetry(n, linearRetry{delay}, steps...) 45 | } 46 | 47 | type expRetry struct { 48 | initial time.Duration 49 | max time.Duration 50 | exp float64 51 | } 52 | 53 | func (r expRetry) Retry(n int) time.Duration { 54 | delay := time.Duration(math.Pow(float64(n), r.exp))*time.Second + r.initial 55 | if delay > r.max { 56 | return r.max 57 | } 58 | return delay 59 | } 60 | 61 | // Exp is a exponential retry implementation. 62 | // Given an initialDelay, it does (initialDelay * n) ^ exp. 63 | // Example: n ^ 2 + 1s = 1s, 3s, 9s... 64 | func Exp(n int, initialDelay, maxDelay time.Duration, exp float64, steps ...pp.Step) pp.Step { 65 | return newRetry(n, expRetry{initialDelay, maxDelay, exp}, steps...) 66 | } 67 | 68 | // Retry implements a pipeline step for retrying all children steps inside. 69 | // If retries = -1, it will retry until it succeeds. 70 | func newRetry(retries int, r Retrier, steps ...pp.Step) pp.Step { 71 | return func(ctx context.Context) (err error) { 72 | for _, step := range steps { 73 | for n := 0; n < retries || retries == -1; n++ { 74 | if err = step(ctx); err == nil { 75 | break 76 | } 77 | time.Sleep(r.Retry(n)) 78 | } 79 | if err != nil { 80 | return err 81 | } 82 | } 83 | return err 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= 6 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 7 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 8 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 9 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 10 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 11 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 12 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 13 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 14 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 15 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 16 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 17 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 18 | github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= 19 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 20 | golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= 21 | golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 22 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 23 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 24 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 25 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 26 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 27 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 28 | -------------------------------------------------------------------------------- /slices_test.go: -------------------------------------------------------------------------------- 1 | package pp 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestDivideSliceInSize(t *testing.T) { 11 | mockFactory := func() (*int, func(i int) Step) { 12 | var count int 13 | return &count, func(_ int) Step { 14 | count++ 15 | return func(ctx context.Context) (err error) { 16 | return nil 17 | } 18 | } 19 | } 20 | t.Run("empty", func(t *testing.T) { 21 | count, empty := mockFactory() 22 | steps := DivideSliceInSize([]int{}, 1, empty) 23 | require.Empty(t, steps) 24 | require.Equal(t, 0, *count) 25 | }) 26 | t.Run("exact size", func(t *testing.T) { 27 | count, empty := mockFactory() 28 | steps := DivideSliceInSize([]int{1, 2, 3, 4}, 2, empty) 29 | require.Len(t, steps, 2) 30 | require.Equal(t, 4, *count) 31 | }) 32 | t.Run("non matching size", func(t *testing.T) { 33 | count, empty := mockFactory() 34 | steps := DivideSliceInSize([]int{1, 2, 3, 4}, 3, empty) 35 | require.Len(t, steps, 2) 36 | require.Equal(t, 4, *count) 37 | }) 38 | t.Run("size bigger than slice", func(t *testing.T) { 39 | count, empty := mockFactory() 40 | steps := DivideSliceInSize([]int{1, 2, 3, 4}, 5, empty) 41 | require.Len(t, steps, 1) 42 | require.Equal(t, 4, *count) 43 | }) 44 | } 45 | 46 | func TestDivideSliceInGroups(t *testing.T) { 47 | mockFactory := func() (*int, func(i int) Step) { 48 | var count int 49 | return &count, func(_ int) Step { 50 | count++ 51 | return func(ctx context.Context) (err error) { 52 | return nil 53 | } 54 | } 55 | } 56 | t.Run("empty", func(t *testing.T) { 57 | count, empty := mockFactory() 58 | steps := DivideSliceInGroups([]int{}, 1, empty) 59 | require.Empty(t, steps) 60 | require.Equal(t, 0, *count) 61 | }) 62 | t.Run("exact size", func(t *testing.T) { 63 | count, empty := mockFactory() 64 | steps := DivideSliceInGroups([]int{1, 2, 3, 4}, 2, empty) 65 | require.Len(t, steps, 2) 66 | require.Equal(t, 4, *count) 67 | }) 68 | t.Run("non matching size", func(t *testing.T) { 69 | count, empty := mockFactory() 70 | steps := DivideSliceInGroups([]int{1, 2, 3, 4}, 3, empty) 71 | require.Len(t, steps, 3) 72 | require.Equal(t, 4, *count) 73 | }) 74 | t.Run("size bigger than slice", func(t *testing.T) { 75 | count, empty := mockFactory() 76 | steps := DivideSliceInGroups([]int{1, 2, 3, 4}, 5, empty) 77 | require.Len(t, steps, 4) 78 | require.Equal(t, 4, *count) 79 | }) 80 | } 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pipego 2 | 3 | Pipego is a robust and type safe pipelining framework, made to improve go's error handling, while also allowing you to load balance, add fail safety, better parallelism and modularization for your code. 4 | 5 | ## Features 6 | 7 | This framework has support for: 8 | 9 | - **Parallelism**: fetch data in parallel, with a cancellable context like in errorGroup implementation. 10 | - **Retriability**: choose from constant, linear and exponential backoffs for retrying any step. 11 | - **Load balance**: you can easily split slices and channels over go-routines using different algorithms. 12 | - **Plug and play api**: you can implement any middleware you want on top of pipego's API. 13 | 14 | ## Functions 15 | 16 | ### Parallel 17 | 18 | With parallel you can run any given steps at `n` parallelism. 19 | 20 | ### Retry 21 | 22 | You can define different retry behaviors for the given steps. 23 | 24 | ### Timeout 25 | 26 | You define a total timeout all the steps inside should take, otherwise cancel them. 27 | 28 | ### WrapErr 29 | 30 | You define a function that will cast any error returned by given steps to a specific error, example: integration error. 31 | 32 | ### DivideSliceInSize 33 | 34 | Divides any given slice in groups of size `n` which can be processed parallel for example. 35 | 36 | ### DivideSliceInGroups 37 | 38 | Does the same thing as DivideSliceInSize, but divide the slice into `n` groups instead. 39 | 40 | ### ChanDivide 41 | 42 | Creates a pool of workers, which takes values from the provided channel `ch` as soon as the worker is available. 43 | 44 | ## Examples 45 | 46 | All examples are under the [examples folder](./examples/) 47 | 48 | - [Simple](./examples/simple/main.go) [Slice, Parallel, Errors] 49 | - [Streaming](./examples/streaming/main.go) [Field, Warnings] 50 | - [Aggregation](./examples/aggregation/main.go) [Slice, Parallel, Errors] 51 | 52 | ### Simple example 53 | 54 | ```go 55 | package main 56 | 57 | import ( 58 | "context" 59 | "fmt" 60 | "time" 61 | 62 | pp "github.com/sonalys/pipego" 63 | "github.com/sonalys/pipego/retry" 64 | ) 65 | 66 | type api struct{} 67 | 68 | func (a api) fetch(ctx context.Context, id string) (int, error) { 69 | return 4, nil 70 | } 71 | 72 | type pipeline struct { 73 | API interface { 74 | fetch(ctx context.Context, id string) (int, error) 75 | } 76 | 77 | input int 78 | sum int 79 | square int 80 | } 81 | 82 | func (p *pipeline) fetchInput(id string) pp.StepFunc { 83 | return func(ctx context.Context) (err error) { 84 | p.input, err = p.API.fetch(ctx, id) 85 | return 86 | } 87 | } 88 | 89 | func (p *pipeline) sumInput(ctx context.Context) (err error) { 90 | p.sum = p.input + p.input 91 | return 92 | } 93 | 94 | func (p *pipeline) sqrInput(ctx context.Context) (err error) { 95 | p.square = p.input * p.input 96 | return 97 | } 98 | 99 | func main() { 100 | ctx := context.Background() 101 | p := pipeline{ 102 | API: api{}, 103 | } 104 | t1 := time.Now() 105 | err := pp.Run(ctx, 106 | retry.Constant(3, time.Second, 107 | p.fetchInput("id"), 108 | ), 109 | pp.Parallel(2, 110 | p.sumInput, 111 | p.sqrInput, 112 | ), 113 | ) 114 | if err != nil { 115 | println(err.Error()) 116 | } 117 | fmt.Printf("Execution took %s.\n%#v\n", time.Since(t1), p) 118 | // Execution took 82.54µs. 119 | // main.pipeline{API:main.api{}, input:4, sum:8, square:16} 120 | } 121 | ``` 122 | 123 | ## Contributions 124 | 125 | With Pipego, I aim to offer a sufficient framework for facilitating any popular pipeline flows. 126 | If you have any ideas, feel free to open an issue 127 | and discuss it's implementation with me. 128 | 129 | Writing more unit tests and fixing any possible bugs would also be nice from any developer. 130 | 131 | ## Disclaimer 132 | 133 | This library is not stable yet, and it's not production ready. 134 | --------------------------------------------------------------------------------