├── .github └── workflows │ ├── linters.yml │ └── tests.yml ├── .gitignore ├── .golangci.yml ├── LICENSE ├── Makefile ├── README.md ├── gen_data_test.go ├── go.mod ├── pipe.go ├── pipe_test.go ├── pipeline.go └── pipeline_test.go /.github/workflows/linters.yml: -------------------------------------------------------------------------------- 1 | name: linters 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | 11 | lint: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v5 18 | with: 19 | go-version-file: go.mod 20 | 21 | - name: Run linters 22 | uses: golangci/golangci-lint-action@v2 -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: tests 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | 11 | test: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | with: 16 | fetch-depth: 2 17 | 18 | - name: Set up Go 19 | uses: actions/setup-go@v5 20 | with: 21 | go-version-file: go.mod 22 | 23 | - name: Build 24 | run: go build -v ./... 25 | 26 | - name: Test 27 | run: go test -v -race -coverprofile=coverage.txt -covermode=atomic ./... 28 | 29 | - name: Upload coverage report 30 | uses: codecov/codecov-action@v5 31 | with: 32 | token: ${{ secrets.CODECOV_TOKEN }} 33 | files: ./coverage.txt 34 | flags: unittests 35 | name: codecov-umbrella 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 1m 3 | allow-parallel-runners: true 4 | 5 | linters-settings: 6 | dupl: 7 | threshold: 100 8 | funlen: 9 | lines: 100 10 | statements: 50 11 | goconst: 12 | min-len: 2 13 | min-occurrences: 2 14 | gocritic: 15 | enabled-tags: 16 | - diagnostic 17 | - experimental 18 | - opinionated 19 | - performance 20 | - style 21 | disabled-checks: 22 | - dupImport # https://github.com/go-critic/go-critic/issues/845 23 | gocyclo: 24 | min-complexity: 11 25 | goimports: 26 | local-prefixes: github.com/golangci/golangci-lint 27 | lll: 28 | line-length: 120 29 | misspell: 30 | locale: US 31 | nolintlint: 32 | allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space) 33 | allow-unused: false # report any unused nolint directives 34 | require-explanation: false # don't require an explanation for nolint directives 35 | require-specific: false # don't require nolint directives to be specific about which linter is being skipped 36 | 37 | linters: 38 | enable: 39 | - depguard 40 | - dogsled 41 | - dupl 42 | - errcheck 43 | - exhaustive 44 | - funlen 45 | - gochecknoinits 46 | - goconst 47 | - gocritic 48 | - gocyclo 49 | - gofmt 50 | - goimports 51 | - goprintffuncname 52 | - gosec 53 | - gosimple 54 | - govet 55 | - ineffassign 56 | - lll 57 | - misspell 58 | - nakedret 59 | - nolintlint 60 | - rowserrcheck 61 | - staticcheck 62 | - stylecheck 63 | - typecheck 64 | - unconvert 65 | - unparam 66 | - unused 67 | - whitespace 68 | - asciicheck 69 | - gochecknoglobals 70 | - gocognit 71 | - godox 72 | - nestif 73 | - prealloc 74 | - revive 75 | - wsl 76 | - cyclop 77 | - durationcheck 78 | - errorlint 79 | - forbidigo 80 | - gci 81 | - gofumpt 82 | - makezero 83 | - predeclared 84 | - tparallel 85 | disable: 86 | - paralleltest 87 | - bodyclose 88 | - godot 89 | - noctx 90 | - sqlclosecheck 91 | - testpackage 92 | issues: 93 | # Excluding configuration per-path, per-linter, per-text and per-source 94 | exclude-rules: 95 | - path: _test\.go 96 | linters: 97 | - gosec 98 | 99 | - path: _test\.go$ 100 | linters: 101 | - revive 102 | text: "empty-block" 103 | - path: _test\.go$ 104 | linters: 105 | - depguard 106 | text: "not allowed from list 'Main'" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 Yuriy Nazarenko 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | test: 2 | golangci-lint run ./... --fix 3 | go test -v -race -timeout 5s 4 | 5 | bench: 6 | go test -benchmem -bench=. -timeout 10s -benchtime 5s -memprofile=mem.out -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Parapipe - paralleling pipeline 2 | =============================== 3 | 4 | [![Mentioned in Awesome Go](https://awesome.re/mentioned-badge-flat.svg)](https://github.com/avelino/awesome-go) 5 | [![tests](https://github.com/nazar256/parapipe/actions/workflows/tests.yml/badge.svg)](https://github.com/nazar256/parapipe/actions/workflows/tests.yml) 6 | [![linters](https://github.com/nazar256/parapipe/actions/workflows/linters.yml/badge.svg)](https://github.com/nazar256/parapipe/actions/workflows/linters.yml) 7 | [![coverage](https://codecov.io/gh/nazar256/parapipe/branch/main/graph/badge.svg?token=N6NI66KPXG)](https://codecov.io/gh/nazar256/parapipe) 8 | [![Go Report Card](https://goreportcard.com/badge/github.com/nazar256/parapipe)](https://goreportcard.com/report/github.com/nazar256/parapipe) 9 | [![GoDoc](https://godoc.org/github.com/nazar256/parapipe?status.svg)](https://godoc.org/github.com/nazar256/parapipe) 10 | 11 | The library provides a zero-dependency non-blocking buffered FIFO-pipeline 12 | for structuring the code and vertically scaling your app. 13 | Unlike regular pipeline examples you may find on the internet - parapipe executes everything on each step concurrently, 14 | yet maintaining the output order. Although, this library does not use any locks or mutexes. Just pure channels. 15 | 16 | When to use 17 | ----------- 18 | 19 | * processed data can be divided in chunks (messages), and the flow may consist of one or more stages 20 | * data should be processed concurrently (scaled vertically) 21 | * the order of processing messages must be maintained 22 | 23 | Installation 24 | ------------ 25 | 26 | ``` 27 | go get -u github.com/nazar256/parapipe@latest 28 | ``` 29 | 30 | Usage 31 | ----- 32 | 33 | 1. Create a pipeline with first step. Processing callback is generic (so as the pipeline). 34 | It may receive and return any type of data, but the second return value should always be a boolean. 35 | 36 | ```go 37 | concurrency := runtime.NumCPU() // how many messages to process concurrently for each pipe 38 | pipeline := parapipe.NewPipeline(concurrency, func(msg YourInputType) (YourOutputType, bool) { 39 | // do something and generate a new value "someValue" 40 | shouldProceedWithNextStep := true 41 | return someValue, shouldProceedWithNextStep 42 | }) 43 | ``` 44 | 45 | 2. Add pipes - call `Attach()` function one or more times to add steps to the pipeline 46 | ```go 47 | p1 := parapipe.NewPipeline(runtime.NumCPU(), func(msg int) (int, bool) { 48 | time.Sleep(30 * time.Millisecond) 49 | return msg + 1000, true 50 | }) 51 | p2 := parapipe.Attach(p1, parapipe.NewPipeline(concurrency, func(msg int) (string, bool) { 52 | time.Sleep(30 * time.Millisecond) 53 | return strconv.Itoa(msg), true 54 | })) 55 | 56 | // final pipeline you are going to work with (push messages and read output) 57 | pipeline := parapipe.Attach(p2, parapipe.NewPipeline(concurrency, func(msg string) (string, bool) { 58 | time.Sleep(30 * time.Millisecond) 59 | return "#" + msg, true 60 | })) 61 | ``` 62 | 63 | 3. Get "out" channel when all pipes are added and read results from it 64 | ```go 65 | for result := range pipeline.Out() { 66 | // do something with the result 67 | } 68 | ``` 69 | It's **important** to drain the pipeline (read everything from "out") even when the pipeline won't produce any viable result. 70 | It could be stuck otherwise. 71 | 72 | 4. Push values for processing into the pipeline: 73 | ```go 74 | pipeline.Push("something") 75 | ``` 76 | 77 | 5. Close pipeline to after the last message. This will cleanup its resources and close its output channel. 78 | It's not recommended closing pipeline using `defer` because you may not want to hang output util defer is executed. 79 | ```go 80 | pipeline.Close() 81 | ``` 82 | 83 | ### Circuit breaking 84 | 85 | In some cases (errors) there could be impossible to process a message, thus there is no way to pass it further. 86 | In such case just return `false` as a second return value from the step processing callback. 87 | The first value will be ignored. 88 | 89 | ```go 90 | pipeline.Pipe(4, func(inputValue InputType) (OutputType, bool) { 91 | someValue, err := someOperation(inputValue) 92 | if err != nil { 93 | // handle the error 94 | // slog.Error("error when calling someOperation", "err", err) 95 | return someValue, false 96 | } 97 | return someValue, true 98 | }) 99 | // ... 100 | for result := range pipeline.Out() { 101 | // do something with the result 102 | } 103 | ``` 104 | 105 | ### Performance 106 | 107 | Parapipe makes use of generics and channels. 108 | Overall it should be performant enough for most of the cases. 109 | It has zero heap allocations in hot code, thus generates little load for garbage collector. 110 | However, it uses channels under the hood and is bottlenecked mostly by the channel operations which are several 111 | writes and reads per each message. 112 | 113 | Examples 114 | -------- 115 | 116 | ### AMQP middleware 117 | 118 | Parapipe can be handful when you need to process messages in the middle concurrently, yet maintaining their order. 119 | 120 | See the [working example of using parapipe in AMQP client](http://github.com/nazar256/go-amqp-sniffer/blob/a5c5db375dc68a2e83c24686e4e57a63cf08c80b/sniffer/sniffer.go#L49-L108). 121 | 122 | ### Other examples 123 | 124 | With parapipe you can: 125 | 126 | * in your API respond a long JSON-feed as stream, retrieve, enrich and marshal each object concurrently, in maintained order and return them to the client 127 | * fetch and merge entries from different sources as one stream 128 | * structure your API controllers or handlers 129 | * processing heavy files in effective way 130 | -------------------------------------------------------------------------------- /gen_data_test.go: -------------------------------------------------------------------------------- 1 | package parapipe_test 2 | 3 | import "github.com/nazar256/parapipe" 4 | 5 | func makeRange(start, end int) []int { 6 | a := make([]int, end-start+1) 7 | for i := range a { 8 | a[i] = start + i 9 | } 10 | 11 | return a 12 | } 13 | 14 | func feedPipeline[T any](pipeline *parapipe.Pipeline[int, T], amount int) { 15 | go func() { 16 | for i := 0; i < amount; i++ { 17 | pipeline.Push(i) 18 | } 19 | 20 | pipeline.Close() 21 | }() 22 | } 23 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/nazar256/parapipe 2 | 3 | go 1.21 4 | 5 | -------------------------------------------------------------------------------- /pipe.go: -------------------------------------------------------------------------------- 1 | package parapipe 2 | 3 | import ( 4 | "log/slog" 5 | "sync" 6 | ) 7 | 8 | type Callback[I any, O any] func(msg I) (output O, proceed bool) 9 | 10 | type pipe[I any, O any] struct { 11 | in chan I 12 | out chan O 13 | callback Callback[I, O] 14 | concurrency int 15 | } 16 | 17 | // newPipe creates a new pipe with the specified concurrency and callback function. 18 | func newPipe[I, O any](concurrency int, callback Callback[I, O]) *pipe[I, O] { 19 | if concurrency < 1 { 20 | slog.Error("concurrency must be greater than 0", slog.Int("concurrency", concurrency)) 21 | concurrency = 1 22 | } 23 | 24 | p := pipe[I, O]{ 25 | in: make(chan I, 1), 26 | out: make(chan O, concurrency+1), 27 | callback: callback, 28 | concurrency: concurrency, 29 | } 30 | 31 | promisesCh := make(chan promise[O], concurrency) 32 | 33 | wp := startWorkerPool[I, O](callback, concurrency, newPromisePool[O]()) 34 | 35 | // queue message processing and output with promises 36 | go func() { 37 | for msg := range p.in { 38 | promisesCh <- wp.push(msg) 39 | } 40 | 41 | close(promisesCh) 42 | }() 43 | 44 | // wait for each promise and close outCh 45 | go func() { 46 | for pr := range promisesCh { 47 | value, ok := pr.await() 48 | if !ok { 49 | continue 50 | } 51 | 52 | p.out <- value 53 | } 54 | 55 | close(p.out) 56 | }() 57 | 58 | return &p 59 | } 60 | 61 | type promiseValue[O any] struct { 62 | value O 63 | ok bool 64 | } 65 | 66 | type promise[O any] struct { 67 | pool *sync.Pool 68 | ch chan promiseValue[O] 69 | } 70 | 71 | type promisePool[O any] struct { 72 | pool *sync.Pool 73 | } 74 | 75 | func newPromisePool[O any]() promisePool[O] { 76 | return promisePool[O]{ 77 | pool: &sync.Pool{ 78 | New: func() any { 79 | return make(chan promiseValue[O], 1) 80 | }, 81 | }, 82 | } 83 | } 84 | 85 | func (pp promisePool[O]) get() promise[O] { 86 | return promise[O]{ 87 | pool: pp.pool, 88 | ch: pp.pool.Get().(chan promiseValue[O]), 89 | } 90 | } 91 | 92 | func (p promise[O]) send(v O) { 93 | p.ch <- promiseValue[O]{ 94 | value: v, 95 | ok: true, 96 | } 97 | } 98 | 99 | func (p promise[O]) cancel() { 100 | p.ch <- promiseValue[O]{ 101 | ok: false, 102 | } 103 | } 104 | 105 | func (p promise[O]) await() (O, bool) { 106 | result := <-p.ch 107 | p.pool.Put(p.ch) 108 | 109 | return result.value, result.ok 110 | } 111 | 112 | type asyncJob[I, O any] struct { 113 | msg I 114 | promise promise[O] 115 | } 116 | 117 | type workerPool[I, O any] struct { 118 | queue chan asyncJob[I, O] 119 | promisePool promisePool[O] 120 | } 121 | 122 | func startWorkerPool[I, O any](job Callback[I, O], concurrency int, promisePool promisePool[O]) *workerPool[I, O] { 123 | wp := &workerPool[I, O]{ 124 | queue: make(chan asyncJob[I, O], concurrency+1), 125 | promisePool: promisePool, 126 | } 127 | 128 | for i := 0; i < concurrency; i++ { 129 | go func() { 130 | for j := range wp.queue { 131 | result, proceed := job(j.msg) 132 | if !proceed { 133 | j.promise.cancel() 134 | continue 135 | } 136 | 137 | j.promise.send(result) 138 | } 139 | }() 140 | } 141 | 142 | return wp 143 | } 144 | 145 | func (wp *workerPool[I, O]) push(msg I) promise[O] { 146 | p := wp.promisePool.get() 147 | 148 | wp.queue <- asyncJob[I, O]{ 149 | msg: msg, 150 | promise: p, 151 | } 152 | 153 | return p 154 | } 155 | -------------------------------------------------------------------------------- /pipe_test.go: -------------------------------------------------------------------------------- 1 | package parapipe 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func genIntMessages(dst chan<- int, amount int) { 10 | go func() { 11 | for i := 0; i < amount; i++ { 12 | dst <- i 13 | } 14 | 15 | close(dst) 16 | }() 17 | } 18 | 19 | func TestPipeMaintainsMessageOrder(t *testing.T) { 20 | p := newPipe[int, int](rand.Intn(4), func(msg int) (int, bool) { 21 | return msg + 1, true 22 | }) 23 | 24 | msgAmount := 100 + rand.Intn(1000) 25 | 26 | go func() { 27 | for input := 0; input < msgAmount; input++ { 28 | p.in <- input 29 | } 30 | 31 | close(p.in) 32 | }() 33 | 34 | for input := 0; input < msgAmount; input++ { 35 | expected := input + 1 36 | actual := <-p.out 37 | 38 | if actual != expected { 39 | t.Errorf("order not maintained, expected %d in sequence of %d, got %d", expected, msgAmount, actual) 40 | t.Fail() 41 | } 42 | } 43 | } 44 | 45 | func TestPipeExecutesJobsConcurrently(t *testing.T) { 46 | concurrency := 100 47 | 48 | p := newPipe[int, int](concurrency, func(msg int) (int, bool) { 49 | time.Sleep(time.Duration(concurrency) * time.Millisecond) 50 | return msg, true 51 | }) 52 | 53 | start := time.Now() 54 | 55 | genIntMessages(p.in, concurrency) 56 | 57 | // wait for everything is processed 58 | for range p.out { 59 | } 60 | 61 | duration := time.Since(start) 62 | if duration > 150*time.Millisecond { 63 | t.Errorf( 64 | "Expected to be executed concurrently in 100ms, actual duration was %dms", 65 | duration/time.Millisecond, 66 | ) 67 | } 68 | } 69 | 70 | func TestPipeCanSkipErrorProcessing(t *testing.T) { 71 | p := newPipe[int, int](rand.Intn(20), func(msg int) (int, bool) { 72 | return msg + 1, false 73 | }) 74 | 75 | go func() { 76 | p.in <- 1 77 | close(p.in) 78 | }() 79 | 80 | outValue := <-p.out 81 | 82 | if outValue != 0 { 83 | t.Error("error processing should skip the job") 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /pipeline.go: -------------------------------------------------------------------------------- 1 | package parapipe 2 | 3 | import "log/slog" 4 | 5 | type Config struct{} 6 | 7 | type Pipeline[I, O any] struct { 8 | inCh chan<- I 9 | outCh <-chan O 10 | closed bool 11 | } 12 | 13 | func NewPipeline[I, O any](concurrency int, callback Callback[I, O]) *Pipeline[I, O] { 14 | p := newPipe[I, O](concurrency, callback) 15 | 16 | return &Pipeline[I, O]{ 17 | inCh: p.in, 18 | outCh: p.out, 19 | } 20 | } 21 | 22 | func (p *Pipeline[I, O]) Push(v I) { 23 | if p.closed { 24 | slog.Error("cannot push after Close is called") 25 | return 26 | } 27 | p.inCh <- v 28 | } 29 | 30 | func (p *Pipeline[I, O]) Out() <-chan O { 31 | return p.outCh 32 | } 33 | 34 | func (p *Pipeline[I, O]) Close() { 35 | if p.closed { 36 | return 37 | } 38 | 39 | p.closed = true 40 | 41 | close(p.inCh) 42 | } 43 | 44 | func Attach[I, T, O any](left *Pipeline[I, T], right *Pipeline[T, O]) *Pipeline[I, O] { 45 | go func() { 46 | for v := range left.Out() { 47 | right.Push(v) 48 | } 49 | 50 | right.Close() 51 | }() 52 | 53 | return &Pipeline[I, O]{ 54 | inCh: left.inCh, 55 | outCh: right.outCh, 56 | closed: left.closed || right.closed, 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /pipeline_test.go: -------------------------------------------------------------------------------- 1 | package parapipe_test 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "strconv" 7 | "testing" 8 | "time" 9 | 10 | "github.com/nazar256/parapipe" 11 | ) 12 | 13 | func TestPipelineExecutesPipesInDefinedOrder(t *testing.T) { 14 | concurrency := rand.Intn(5) + 2 15 | p1 := parapipe.NewPipeline(concurrency, func(msg int) (string, bool) { 16 | return strconv.Itoa(msg), true 17 | }) 18 | pipeline := parapipe.Attach(p1, parapipe.NewPipeline(concurrency, func(msg string) (string, bool) { 19 | return "#" + msg, true 20 | })) 21 | 22 | feedPipeline(pipeline, 100) 23 | 24 | i := 0 25 | 26 | for actualResult := range pipeline.Out() { 27 | expected := fmt.Sprintf("#%d", i) 28 | if actualResult != expected { 29 | t.Errorf("got wrong result from pipeline at iteration %d: \"%s\" instead of \"%s\"", i, actualResult, expected) 30 | t.Fail() 31 | } 32 | 33 | i++ 34 | } 35 | } 36 | 37 | func TestPipelineExecutesConcurrently(t *testing.T) { 38 | inputValuesCount := 100 39 | concurrency := inputValuesCount 40 | p1 := parapipe.NewPipeline(concurrency, func(msg int) (int, bool) { 41 | time.Sleep(30 * time.Millisecond) 42 | return msg + 1000, true 43 | }) 44 | p2 := parapipe.Attach(p1, parapipe.NewPipeline(concurrency, func(msg int) (string, bool) { 45 | time.Sleep(30 * time.Millisecond) 46 | return strconv.Itoa(msg), true 47 | })) 48 | pipeline := parapipe.Attach(p2, parapipe.NewPipeline(concurrency, func(msg string) (string, bool) { 49 | time.Sleep(30 * time.Millisecond) 50 | return "#" + msg, true 51 | })) 52 | 53 | start := time.Now() 54 | 55 | feedPipeline(pipeline, inputValuesCount) 56 | 57 | // wait for all results 58 | for range pipeline.Out() { 59 | } 60 | 61 | if time.Since(start) > 150*time.Millisecond { 62 | t.Errorf( 63 | "Expected to be executed concurrently in 100ms, actual duration was %dms", 64 | time.Since(start)/time.Millisecond, 65 | ) 66 | } 67 | } 68 | 69 | func TestPipelineSkipsMessages(t *testing.T) { 70 | concurrency := rand.Intn(20) 71 | p1 := parapipe.NewPipeline(concurrency, func(msg int) (int, bool) { 72 | return msg + 1, false 73 | }) 74 | pipeline := parapipe.Attach(p1, parapipe.NewPipeline(concurrency, func(msg int) (int, bool) { 75 | t.Error("Error expected to be skipped from processing, but worker has received one") 76 | return msg, true 77 | })) 78 | 79 | pipeline.Push(1) 80 | pipeline.Close() 81 | <-pipeline.Out() 82 | } 83 | 84 | func Benchmark1Pipe1Message(b *testing.B) { 85 | concurrency := 1 86 | pipeline := parapipe.NewPipeline(concurrency, func(msg int) (int, bool) { return msg, true }) 87 | 88 | for n := 0; n < b.N; n++ { 89 | go func() { 90 | pipeline.Push(1) 91 | }() 92 | 93 | <-pipeline.Out() 94 | } 95 | 96 | pipeline.Close() 97 | } 98 | 99 | func Benchmark5Pipes1Message(b *testing.B) { 100 | concurrency := 1 101 | p1 := parapipe.NewPipeline(concurrency, func(msg int) (int, bool) { return msg, true }) 102 | p2 := parapipe.Attach(p1, parapipe.NewPipeline(concurrency, func(msg int) (int, bool) { return msg, true })) 103 | p3 := parapipe.Attach(p2, parapipe.NewPipeline(concurrency, func(msg int) (int, bool) { return msg, true })) 104 | p4 := parapipe.Attach(p3, parapipe.NewPipeline(concurrency, func(msg int) (int, bool) { return msg, true })) 105 | pipeline := parapipe.Attach(p4, parapipe.NewPipeline(concurrency, func(msg int) (int, bool) { return msg, true })) 106 | 107 | for n := 0; n < b.N; n++ { 108 | go func() { 109 | pipeline.Push(1) 110 | }() 111 | 112 | <-pipeline.Out() 113 | } 114 | } 115 | 116 | func Benchmark1Pipe10000Messages(b *testing.B) { 117 | concurrency := 8 118 | pipeline := parapipe.NewPipeline(concurrency, func(msg int) (int, bool) { return msg, true }) 119 | msgCount := 10000 120 | 121 | for n := 0; n < b.N; n++ { 122 | go func() { 123 | for i := 0; i < msgCount; i++ { 124 | pipeline.Push(i) 125 | } 126 | }() 127 | 128 | for i := 0; i < msgCount; i++ { 129 | <-pipeline.Out() 130 | } 131 | } 132 | 133 | pipeline.Close() 134 | } 135 | 136 | func Benchmark1Pipe10000MessagesBatchedBy100(b *testing.B) { 137 | concurrency := 8 138 | pipeline := parapipe.NewPipeline(concurrency, func(msg []int) ([]int, bool) { return msg, true }) 139 | 140 | batchCount := 100 141 | batchSize := 100 142 | 143 | for n := 0; n < b.N; n++ { 144 | go func() { 145 | msg := makeRange(1, batchSize) 146 | 147 | for i := 0; i < batchCount; i++ { 148 | pipeline.Push(msg) 149 | } 150 | }() 151 | 152 | for i := 0; i < batchCount; i++ { 153 | <-pipeline.Out() 154 | } 155 | } 156 | 157 | pipeline.Close() 158 | } 159 | --------------------------------------------------------------------------------